fix(update): clear a stale .git/index.lock before Update Code (#801)

* fix(update): clear a stale .git/index.lock before Update Code A git that subprocess.run's timeout kills (SIGKILL) cannot remove .git/index.lock, so every later update failed with "Unable to create .../.git/index.lock: File exists" until the file was removed by hand. Update Code and the automatic update's rollback now remove the lock when no git process is running in the checkout and the file is more than a few seconds old. While another git is running, Update Code says so instead of failing; when the lock could not be checked, the error names the rm command that fixes it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SvFtgthKUNEPhJu6RuX9Sk * fix(update): count a git that names the checkout in its arguments as running git --git-dir/--work-tree can work on the checkout from any working directory, so the cwd check alone could take such a git's live lock for a stale one. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SvFtgthKUNEPhJu6RuX9Sk * fix(update): resolve a git's relative path arguments against its cwd `git -C LEDMatrix ...` or a relative --git-dir started outside the checkout works on it too; resolve each argument against the process's working directory before deciding a lock is stale, and count unreadable arguments as naming the checkout. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SvFtgthKUNEPhJu6RuX9Sk --------- Co-authored-by: Claude <noreply@anthropic.com>
2026-10-27 00:29:13 +00:00 · 2026-10-26 20:13:40 -04:00
parent 4c883ec2c2
commit fa51ebab5f
6 changed files with 389 additions and 2 deletions
@@ -0,0 +1,200 @@
+"""A stale .git/index.lock must not block Update Code forever.
+
+subprocess.run's timeout SIGKILLs git, and a killed git cannot remove its
+index.lock. Every later update then failed with "Unable to create
+'.../.git/index.lock': File exists" until someone removed the file over SSH.
+The update now removes the lock when no git process can still own it.
+"""
+import os
+import shutil
+import subprocess
+import sys
+import time
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+from flask import Flask
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from web_interface import git_index_lock as gil  # noqa: E402
+from web_interface.blueprints import api_v3 as mod  # noqa: E402
+from web_interface.blueprints.api_v3 import api_v3  # noqa: E402
+
+needs_git = pytest.mark.skipif(shutil.which('git') is None, reason='git not installed')
+
+OLD = 1_000_000.0
+
+
+def _checkout(tmp_path, lock_age=None):
+    repo = tmp_path / 'LEDMatrix'
+    (repo / '.git').mkdir(parents=True)
+    if lock_age is not None:
+        lock = repo / '.git' / 'index.lock'
+        lock.write_text('')
+        os.utime(lock, (OLD, OLD))
+    return repo
+
+
+def _proc(tmp_path, *procs):
+    """A fake /proc: ``procs`` are (comm, cwd[, argv]); cwd None means unreadable."""
+    root = tmp_path / 'proc'
+    root.mkdir()
+    (root / 'self').mkdir()  # non-numeric entries are skipped
+    for pid, (comm, cwd, *argv) in enumerate(procs, start=100):
+        entry = root / str(pid)
+        entry.mkdir()
+        (entry / 'comm').write_text(comm + '\n')
+        (entry / 'cmdline').write_bytes(b'\0'.join(a.encode() for a in [comm, *argv]) + b'\0')
+        if cwd is not None:
+            (entry / 'cwd').symlink_to(cwd)
+        else:
+            (entry / 'cwd').mkdir()  # readlink on a directory fails: unreadable
+    return root
+
+
+class TestClearStaleIndexLock:
+    def test_no_lock_is_left_alone(self, tmp_path):
+        repo = _checkout(tmp_path)
+        assert gil.clear_stale_index_lock(repo, proc_root=_proc(tmp_path)) == gil.ABSENT
+
+    def test_a_stale_lock_is_removed(self, tmp_path):
+        repo = _checkout(tmp_path, lock_age=60)
+        proc = _proc(tmp_path, ('python3', str(repo)), ('git', str(tmp_path / 'elsewhere')))
+        assert gil.clear_stale_index_lock(repo, proc_root=proc, now=OLD + 60) == gil.REMOVED
+        assert not (repo / '.git' / 'index.lock').exists()
+
+    @pytest.mark.parametrize('comm', ['git', 'git-remote-http'])
+    def test_a_lock_held_by_a_running_git_is_kept(self, tmp_path, comm):
+        repo = _checkout(tmp_path, lock_age=60)
+        proc = _proc(tmp_path, (comm, str(repo / 'src')))
+        assert gil.clear_stale_index_lock(repo, proc_root=proc, now=OLD + 60) == gil.IN_USE
+        assert (repo / '.git' / 'index.lock').exists()
+
+    def test_a_git_pointed_at_the_checkout_from_elsewhere_is_running(self, tmp_path):
+        repo = _checkout(tmp_path, lock_age=60)
+        proc = _proc(tmp_path, ('git', str(tmp_path), f'--git-dir={repo.resolve()}/.git',
+                                f'--work-tree={repo.resolve()}', 'checkout', 'v3.8.7'))
+        assert gil.clear_stale_index_lock(repo, proc_root=proc, now=OLD + 60) == gil.IN_USE
+        assert (repo / '.git' / 'index.lock').exists()
+
+    @pytest.mark.parametrize('argv', [['-C', 'LEDMatrix', 'status'],
+                                      ['--git-dir=LEDMatrix/.git', '--work-tree=LEDMatrix', 'add', '.']])
+    def test_a_git_pointed_at_the_checkout_by_a_relative_path_is_running(self, tmp_path, argv):
+        repo = _checkout(tmp_path, lock_age=60)
+        proc = _proc(tmp_path, ('git', str(repo.resolve().parent), *argv))
+        assert gil.clear_stale_index_lock(repo, proc_root=proc, now=OLD + 60) == gil.IN_USE
+
+    def test_a_git_working_on_another_repository_is_not(self, tmp_path):
+        repo = _checkout(tmp_path, lock_age=60)
+        proc = _proc(tmp_path, ('git', str(tmp_path), '-C', 'other', 'pull'),
+                     ('git', str(tmp_path), '--git-dir=LEDMatrix-backup/.git', 'fetch'))
+        assert gil.clear_stale_index_lock(repo, proc_root=proc, now=OLD + 60) == gil.REMOVED
+
+    def test_a_git_whose_cwd_cannot_be_read_counts_as_running(self, tmp_path):
+        repo = _checkout(tmp_path, lock_age=60)
+        proc = _proc(tmp_path, ('git', None))
+        assert gil.clear_stale_index_lock(repo, proc_root=proc, now=OLD + 60) == gil.IN_USE
+        assert (repo / '.git' / 'index.lock').exists()
+
+    def test_a_brand_new_lock_is_kept(self, tmp_path):
+        """A git may have created it between the process scan and the removal."""
+        repo = _checkout(tmp_path, lock_age=0)
+        assert gil.clear_stale_index_lock(repo, proc_root=_proc(tmp_path), now=OLD + 1) == gil.IN_USE
+        assert (repo / '.git' / 'index.lock').exists()
+
+    def test_without_proc_nothing_is_removed(self, tmp_path):
+        repo = _checkout(tmp_path, lock_age=60)
+        missing = tmp_path / 'no-proc'
+        assert gil.clear_stale_index_lock(repo, proc_root=missing, now=OLD + 60) == gil.UNKNOWN
+        assert (repo / '.git' / 'index.lock').exists()
+
+    def test_a_checkout_without_a_git_directory_is_left_alone(self, tmp_path):
+        assert gil.clear_stale_index_lock(tmp_path, proc_root=_proc(tmp_path)) == gil.ABSENT
+
+
+@needs_git
+@pytest.mark.skipif(not Path('/proc/self/comm').exists(), reason='needs /proc')
+def test_git_works_again_after_a_real_stale_lock_is_cleared(tmp_path):
+    repo = tmp_path / 'repo'
+    env = {**os.environ, 'GIT_AUTHOR_NAME': 't', 'GIT_AUTHOR_EMAIL': 't@t',
+           'GIT_COMMITTER_NAME': 't', 'GIT_COMMITTER_EMAIL': 't@t'}
+
+    def git(*args):
+        return subprocess.run(['git', *args], cwd=repo, env=env, capture_output=True, text=True)
+    repo.mkdir()
+    git('init', '-q')
+    (repo / 'a.txt').write_text('a')
+    lock = repo / '.git' / 'index.lock'
+    lock.write_text('')
+    old = time.time() - 60
+    os.utime(lock, (old, old))
+    assert 'index.lock' in git('add', 'a.txt').stderr
+
+    assert gil.clear_stale_index_lock(repo) == gil.REMOVED
+    assert git('add', 'a.txt').returncode == 0
+
+
+# -- Update Code ----------------------------------------------------------------
+
+@pytest.fixture
+def client():
+    app = Flask(__name__)
+    app.config['TESTING'] = True
+    app.register_blueprint(api_v3, url_prefix='/api/v3')
+    api_v3.plugin_store_manager = None
+    api_v3.config_manager = None
+    return app.test_client()
+
+
+def _fake_git(calls, pull_rc=0, pull_err=''):
+    def run(args, **kwargs):
+        calls.append(list(args))
+        if args[:2] == ['git', 'rev-parse'] and args[-1] == 'HEAD':
+            return subprocess.CompletedProcess(args, 0, stdout='aaa111\n', stderr='')
+        if 'symbolic-full-name' in args or '@{u}' in args:
+            return subprocess.CompletedProcess(args, 0, stdout='origin/main\n', stderr='')
+        if args[:2] == ['git', 'pull']:
+            return subprocess.CompletedProcess(args, pull_rc, stdout='', stderr=pull_err)
+        return subprocess.CompletedProcess(args, 0, stdout='', stderr='')
+    return run
+
+
+def _update(client):
+    return client.post('/api/v3/system/action', json={'action': 'git_pull'}).get_json()
+
+
+def test_update_code_clears_the_lock_before_running_git(client):
+    seen = []
+
+    def clear(project_dir):
+        seen.append(project_dir)
+        return gil.REMOVED
+    calls = []
+    with patch.object(gil, 'clear_stale_index_lock', clear), \
+            patch.object(mod.subprocess, 'run', _fake_git(calls)):
+        data = _update(client)
+    assert seen == [str(mod.PROJECT_ROOT)]
+    assert data['status'] == 'success'
+
+
+def test_update_code_waits_for_a_git_that_is_still_running(client):
+    calls = []
+    with patch.object(gil, 'clear_stale_index_lock', lambda d: gil.IN_USE), \
+            patch.object(mod.subprocess, 'run', _fake_git(calls)):
+        data = _update(client)
+    assert data['status'] == 'error'
+    assert 'Another git command is running' in data['message']
+    assert calls == [], "nothing may touch the checkout while another git owns it"
+
+
+def test_a_lock_it_could_not_clear_gets_the_manual_fix(client):
+    err = "fatal: Unable to create '/home/ledpi/LEDMatrix/.git/index.lock': File exists.\n"
+    calls = []
+    with patch.object(gil, 'clear_stale_index_lock', lambda d: gil.UNKNOWN), \
+            patch.object(mod.subprocess, 'run', _fake_git(calls, pull_rc=128, pull_err=err)):
+        data = _update(client)
+    assert data['status'] == 'error'
+    assert data['message'].startswith('Update failed: fatal: Unable to create')
+    assert 'rm -f ' in data['message'] and 'index.lock' in data['message']