fix(update): stop a timed-out git from leaving .git/index.lock behind

subprocess.run's timeout SIGKILLs only the direct git child, so the lock
is left behind and git's helpers are orphaned. On a slow SD card the same
pull then times out on every retry and recreates the lock, so removing it
by hand does not help.

- run_git starts git in its own session, stops the group with SIGTERM
  before SIGKILL, and clears the lock if nothing owns it. Update Code's
  pull, checkout, reset and stash go through it, with a 600s pull timeout.
- Read-only calls (describe --dirty, status, the update-check fetch) set
  GIT_OPTIONAL_LOCKS=0 so a timed-out poll cannot take the lock.
- A timed-out pull returns a clear error; a rebase left by an interrupted
  update is aborted before pulling.
- The manual-fix hint says to stop the services and kill running git.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

test/test_auto_update.py

@@ -823,6 +823,7 @@ def test_core_update_names_the_requirement_files_that_failed():
     pkg.api_v3.plugin_store_manager = None
     failed_install = subprocess.CompletedProcess([], 1, stdout='', stderr='boom')
     with patch.object(pkg.subprocess, 'run', run), \
+            patch.object(system.git_index_lock, 'run_git', run), \
             patch.object(system, '_pip_install_requirements', return_value=failed_install):
         result = system.perform_core_update()
     assert result['status'] == 'success'

test/test_git_index_lock.py

@@ -11,6 +11,7 @@ import subprocess
 import sys
 import time
 from pathlib import Path
+import contextlib
 from unittest.mock import patch
 
 import pytest
@@ -27,6 +28,13 @@ needs_git = pytest.mark.skipif(shutil.which('git') is None, reason='git not inst
 OLD = 1_000_000.0
 
 
+@contextlib.contextmanager
+def _patched(fake):
+    """Fake git for both the plain subprocess calls and the timeout-safe runner."""
+    with patch.object(mod.subprocess, 'run', fake), patch.object(mod.git_index_lock, 'run_git', fake):
+        yield
+
+
 def _checkout(tmp_path, lock_age=None):
     repo = tmp_path / 'LEDMatrix'
     (repo / '.git').mkdir(parents=True)
@@ -173,7 +181,7 @@ def test_update_code_clears_the_lock_before_running_git(client):
         return gil.REMOVED
     calls = []
     with patch.object(gil, 'clear_stale_index_lock', clear), \
-            patch.object(mod.subprocess, 'run', _fake_git(calls)):
+            _patched(_fake_git(calls)):
         data = _update(client)
     assert seen == [str(mod.PROJECT_ROOT)]
     assert data['status'] == 'success'
@@ -182,7 +190,7 @@ def test_update_code_clears_the_lock_before_running_git(client):
 def test_update_code_waits_for_a_git_that_is_still_running(client):
     calls = []
     with patch.object(gil, 'clear_stale_index_lock', lambda d: gil.IN_USE), \
-            patch.object(mod.subprocess, 'run', _fake_git(calls)):
+            _patched(_fake_git(calls)):
         data = _update(client)
     assert data['status'] == 'error'
     assert 'Another git command is running' in data['message']
@@ -193,8 +201,45 @@ def test_a_lock_it_could_not_clear_gets_the_manual_fix(client):
     err = "fatal: Unable to create '/home/ledpi/LEDMatrix/.git/index.lock': File exists.\n"
     calls = []
     with patch.object(gil, 'clear_stale_index_lock', lambda d: gil.UNKNOWN), \
-            patch.object(mod.subprocess, 'run', _fake_git(calls, pull_rc=128, pull_err=err)):
+            _patched(_fake_git(calls, pull_rc=128, pull_err=err)):
         data = _update(client)
     assert data['status'] == 'error'
     assert data['message'].startswith('Update failed: fatal: Unable to create')
     assert 'rm -f ' in data['message'] and 'index.lock' in data['message']
+
+
+# -- run_git: a timed-out git must not leave a lock or an orphaned helper ------
+
+def _lock_holder_script(tmp_path, lock):
+    """A stand-in for a slow git: takes the lock, spawns a helper, then hangs."""
+    script = tmp_path / 'slow.sh'
+    script.write_text(f'#!/bin/sh\n: > "{lock}"\nsleep 60 &\necho $! > "{tmp_path}/helper.pid"\nwait\n')
+    script.chmod(0o755)
+    return script
+
+
+def test_run_git_returns_a_completed_process(tmp_path):
+    result = gil.run_git([sys.executable, '-c', 'print("hi")'], cwd=str(tmp_path), timeout=30)
+    assert (result.returncode, result.stdout.strip()) == (0, 'hi')
+
+
+@pytest.mark.skipif(not os.path.isdir('/proc'), reason='needs /proc')
+def test_run_git_timeout_stops_the_group_and_clears_the_lock(tmp_path, monkeypatch):
+    (tmp_path / '.git').mkdir()
+    lock = tmp_path / '.git' / 'index.lock'
+    script = _lock_holder_script(tmp_path, lock)
+    monkeypatch.setattr(gil, 'MIN_STALE_AGE_SECONDS', 0)
+    monkeypatch.setattr(gil, 'TERM_GRACE_SECONDS', 1)
+    # The scan only sees processes named git, so the stand-in must not match;
+    # the lock is then provably unowned once the group is gone.
+    with pytest.raises(subprocess.TimeoutExpired):
+        gil.run_git([str(script)], cwd=str(tmp_path), timeout=1)
+    assert not lock.exists()
+    helper = int((tmp_path / 'helper.pid').read_text())
+    time.sleep(0.2)
+    with pytest.raises(ProcessLookupError):
+        os.kill(helper, 0)
+
+
+def test_read_only_env_disables_optional_locks():
+    assert gil.read_only_env()['GIT_OPTIONAL_LOCKS'] == '0'

test/test_update_prompts_restart.py

@@ -9,6 +9,7 @@ restart is owed, and the UI raises the existing restart-pending banner.
 import subprocess
 import sys
 from pathlib import Path
+import contextlib
 from unittest.mock import patch
 
 import pytest
@@ -20,6 +21,13 @@ from web_interface.blueprints import api_v3 as mod  # noqa: E402
 from web_interface.blueprints.api_v3 import api_v3  # noqa: E402
 
 
+@contextlib.contextmanager
+def _patched(fake):
+    """Fake git for both the plain subprocess calls and the timeout-safe runner."""
+    with patch.object(mod.subprocess, 'run', fake), patch.object(mod.git_index_lock, 'run_git', fake):
+        yield
+
+
 @pytest.fixture
 def client():
     app = Flask(__name__)
@@ -62,7 +70,7 @@ def _pull(client):
 
 class TestRestartIsRequestedWhenCodeChanged:
     def test_a_pull_that_moved_head_asks_for_a_restart(self, client):
-        with patch.object(mod.subprocess, 'run', _git(['aaa111', 'bbb222'])):
+        with _patched(_git(['aaa111', 'bbb222'])):
             data = _pull(client)
         assert data['status'] == 'success'
         assert data['restart_required'] is True, (
@@ -70,15 +78,14 @@ class TestRestartIsRequestedWhenCodeChanged:
             "nothing told the user to restart")
 
     def test_already_up_to_date_does_not(self, client):
-        with patch.object(mod.subprocess, 'run',
-                          _git(['aaa111', 'aaa111'], pull_out='Already up to date.\n')):
+        with _patched(_git(['aaa111', 'aaa111'], pull_out='Already up to date.\n')):
             data = _pull(client)
         assert data['status'] == 'success'
         assert data['restart_required'] is False, (
             "prompting after a no-op update trains users to ignore the prompt")
 
     def test_a_failed_pull_does_not(self, client):
-        with patch.object(mod.subprocess, 'run', _git(['aaa111'], pull_rc=1)):
+        with _patched(_git(['aaa111'], pull_rc=1)):
             data = _pull(client)
         assert data['status'] == 'error'
         assert data['restart_required'] is False
@@ -100,19 +107,18 @@ class TestUnitsAreRefreshedWithTheCode:
         return calls
 
     def test_an_update_that_moved_head_refreshes_the_units(self, client, refresh):
-        with patch.object(mod.subprocess, 'run', _git(['aaa111', 'bbb222'])):
+        with _patched(_git(['aaa111', 'bbb222'])):
             data = _pull(client)
         assert refresh == [True]
         assert data['unit_refresh']['status'] == 'refreshed'
         assert 'Service settings updated' in data['message']
 
     def test_nothing_new_touches_no_units(self, client, refresh):
-        with patch.object(mod.subprocess, 'run',
-                          _git(['aaa111', 'aaa111'], pull_out='Already up to date.\n')):
+        with _patched(_git(['aaa111', 'aaa111'], pull_out='Already up to date.\n')):
             data = _pull(client)
         assert refresh == [] and data['unit_refresh'] is None
 
     def test_a_failed_pull_touches_no_units(self, client, refresh):
-        with patch.object(mod.subprocess, 'run', _git(['aaa111'], pull_rc=1)):
+        with _patched(_git(['aaa111'], pull_rc=1)):
             data = _pull(client)
         assert refresh == [] and data['unit_refresh'] is None