Merge audit16-core/autoupdate into the September 16 core audit fixes

test/test_auto_update.py

@@ -72,7 +72,7 @@ class Repo:
 
 
 def real_pull(device, **extra):
-    def core_update():
+    def core_update(**kwargs):
         before = git(device, 'rev-parse', 'HEAD')
         r = subprocess.run(['git', 'pull', '-q', '--rebase', '--autostash'],
                            cwd=str(device), capture_output=True, text=True)
@@ -339,6 +339,140 @@ class TestPreflightRefuses:
         assert repo.head() == old and h.sudo == []
 
 
+# -- what counts as a local change ---------------------------------------------
+
+def _device_with_plugin_and_script(tmp_path):
+    """A clone carrying a bundled plugin and an executable script, as LEDMatrix does."""
+    repo = Repo(tmp_path)
+    for rel, text in (('plugin-repos/web-ui-info/manager.py', 'x = 1\n'),
+                      ('web_interface/static/v3/js/plugins/store.js', 'var a;\n'),
+                      ('scripts/run.sh', 'echo hi\n')):
+        (repo.seed / rel).parent.mkdir(parents=True, exist_ok=True)
+        (repo.seed / rel).write_text(text)
+    git(repo.seed, 'add', '.')
+    git(repo.seed, 'update-index', '--chmod=+x', 'scripts/run.sh')
+    git(repo.seed, 'commit', '-qm', 'layout')
+    git(repo.seed, 'push', '-q', 'origin', 'main')
+    git(repo.device, 'pull', '-q')
+    return repo
+
+
+def _chmod_like_the_installer(device):
+    """first_time_install.sh sets tracked 100755 files to 644: a mode-only change.
+
+    Windows has no exec bit, so there core.fileMode=true alone shows it."""
+    import os
+    git(device, 'config', 'core.fileMode', 'true')
+    os.chmod(device / 'scripts' / 'run.sh', 0o644)
+    assert 'mode change 100755 => 100644 scripts/run.sh' in git(device, 'diff', '--summary')
+
+
+@needs_git
+class TestLocalChangesAreCountedOnce:
+    """The preflight promises the pull will not stash. That holds only if both
+    count local changes the same way (auto_update.local_changes)."""
+
+    def test_mode_changes_and_plugin_folders_do_not_count(self, tmp_path):
+        repo = _device_with_plugin_and_script(tmp_path)
+        _chmod_like_the_installer(repo.device)
+        (repo.device / 'plugin-repos/web-ui-info/manager.py').write_text('x = 2  # store update\n')
+        assert au.local_changes(repo.device) == []
+
+    def test_a_core_folder_named_plugins_still_counts(self, tmp_path):
+        """The old filter matched 'plugins/' anywhere in the line."""
+        repo = _device_with_plugin_and_script(tmp_path)
+        (repo.device / 'web_interface/static/v3/js/plugins/store.js').write_text('var mine;\n')
+        assert au.local_changes(repo.device) == ['web_interface/static/v3/js/plugins/store.js']
+
+    def test_a_staged_rename_is_reported_by_its_new_name(self, tmp_path):
+        repo = Repo(tmp_path)
+        git(repo.device, 'mv', 'app.py', 'main.py')
+        assert au.local_changes(repo.device) == ['main.py']
+
+    def test_the_preflight_refuses_a_core_edit_under_a_plugins_folder(self, tmp_path):
+        repo = _device_with_plugin_and_script(tmp_path)
+        old = repo.head()
+        repo.publish()
+        (repo.device / 'web_interface/static/v3/js/plugins/store.js').write_text('var mine;\n')
+        result = Harness(tmp_path, repo).updater.run()
+        assert result['core_outcome'] == 'blocked'
+        assert 'store.js' in result['core_message']
+        assert repo.head() == old
+
+    @pytest.fixture
+    def core_update(self, monkeypatch):
+        """The real perform_core_update, pointed at a test clone."""
+        from web_interface.blueprints import api_v3 as pkg
+        from web_interface.blueprints.api_v3 import system
+
+        def point_at(device):
+            monkeypatch.setattr(system, 'PROJECT_ROOT', device)
+            monkeypatch.setattr(pkg.api_v3, 'plugin_store_manager', None, raising=False)
+            monkeypatch.setattr(system, '_pip_install_requirements',
+                                lambda *a, **k: pytest.fail('no requirements changed'))
+            return system.perform_core_update
+        return point_at
+
+    @pytest.mark.parametrize('change', [
+        'bundled_plugin_edit',
+        pytest.param('installer_chmod', marks=pytest.mark.skipif(
+            sys.platform == 'win32',
+            reason='no exec bit: the reset inside --autostash cannot clear a mode change, '
+                   'so git will not reapply it (on Linux it reapplies cleanly)')),
+    ])
+    def test_an_update_that_passed_the_preflight_leaves_no_stash(self, tmp_path, core_update, change):
+        """Found by audit: a bundled-plugin edit or the installer's chmods
+        passed the preflight, then perform_core_update stashed them for good.
+        Now --autostash carries them across the pull and puts them back."""
+        repo = _device_with_plugin_and_script(tmp_path)
+        new = repo.publish()
+        if change == 'installer_chmod':
+            _chmod_like_the_installer(repo.device)
+        else:
+            (repo.device / 'plugin-repos/web-ui-info/manager.py').write_text('x = 2  # store update\n')
+        h = Harness(tmp_path, repo, core_update=core_update(repo.device))
+        assert h.updater.preflight({})[0] == 'ready'
+
+        result = h.updater.run()
+        assert result['core_outcome'] == 'verifying', result['core_message']
+        assert repo.head() == new
+        assert git(repo.device, 'stash', 'list') == ''
+        if change == 'installer_chmod':
+            assert 'mode change 100755 => 100644 scripts/run.sh' in git(repo.device, 'diff', '--summary')
+        else:
+            assert 'store update' in (repo.device / 'plugin-repos/web-ui-info/manager.py').read_text()
+
+    def test_edits_made_after_the_preflight_are_refused_not_stashed(self, tmp_path, core_update):
+        repo = Repo(tmp_path)
+        old = repo.head()
+        repo.publish()
+        perform = core_update(repo.device)
+
+        def edit_then_update(**kwargs):
+            (repo.device / 'app.py').write_text('mine\n')
+            return perform(**kwargs)
+        result = Harness(tmp_path, repo, core_update=edit_then_update).updater.run()
+        assert result['core_outcome'] == 'blocked'
+        assert 'app.py' in result['core_message'] and 'will not stash' in result['core_message']
+        assert repo.head() == old
+        assert git(repo.device, 'stash', 'list') == ''
+        assert (repo.device / 'app.py').read_text() == 'mine\n'
+
+    def test_update_code_still_stashes_core_edits_but_not_plugin_folders(self, tmp_path, core_update):
+        repo = _device_with_plugin_and_script(tmp_path)
+        new = repo.publish()
+        (repo.device / 'scripts/run.sh').write_text('echo mine\n')
+        (repo.device / 'plugin-repos/web-ui-info/manager.py').write_text('x = 2  # store update\n')
+        payload = core_update(repo.device)()
+        assert payload['status'] == 'success', payload['message']
+        assert 'stashed' in payload['message']
+        assert repo.head() == new
+        assert 'LEDMatrix auto-stash before update' in git(repo.device, 'stash', 'list')
+        stashed = git(repo.device, 'stash', 'show', '--name-only', 'stash@{0}')
+        assert stashed.split() == ['scripts/run.sh']
+        assert 'store update' in (repo.device / 'plugin-repos/web-ui-info/manager.py').read_text()
+
+
 # -- the update and its hand-off to the health check ---------------------------
 
 @needs_git
@@ -395,7 +529,7 @@ class TestUpdateIsVerified:
         old = repo.head()
         repo.publish()
         h = Harness(tmp_path, repo,
-                    core_update=lambda: {'status': 'error', 'message': 'Update failed: network'})
+                    core_update=lambda **kw: {'status': 'error', 'message': 'Update failed: network'})
         result = h.updater.run()
         assert result['core_outcome'] == 'error'
         assert repo.head() == old
@@ -407,13 +541,43 @@ class TestUpdateIsVerified:
         repo.publish()
         pull = real_pull(repo.device)
 
-        def half_failed():
+        def half_failed(**kwargs):
             pull()
             return {'status': 'error', 'message': 'Update failed: interrupted'}
         result = Harness(tmp_path, repo, core_update=half_failed).updater.run()
         assert repo.head() == old
         assert 'rolled back' in result['core_message']
 
+    @pytest.mark.parametrize('how', ['partial_pull', 'check_never_started'])
+    def test_a_failed_rollback_leaves_plugins_and_the_display_alone(self, tmp_path, how):
+        """update_core returns rollback_failed itself on two paths. The device
+        is then in an unknown state, exactly as when the health check reports
+        rollback_failed, so no plugin updates and no restart may follow."""
+        repo = Repo(tmp_path)
+        repo.publish()
+        store = FakeStore(tmp_path / 'plugins', {'clock': '1.0.0'}, bump={'clock'})
+        pull = real_pull(repo.device)
+
+        def half_failed(**kwargs):
+            pull()
+            return {'status': 'error', 'message': 'Update failed: interrupted'}
+        h = Harness(tmp_path, repo, store=store, pickup=how != 'check_never_started',
+                    core_update=half_failed if how == 'partial_pull' else None)
+        real_run = h.updater.run_command
+
+        def reset_fails(args, **kwargs):
+            if args[:3] == ['git', 'reset', '--hard']:
+                return subprocess.CompletedProcess(args, 1, stdout='', stderr='index.lock exists')
+            return real_run(args, **kwargs)
+        h.updater.run_command = reset_fails
+
+        result = h.updater.run()
+        assert result['core_outcome'] == 'rollback_failed'
+        assert store.updated_calls == [], "plugins must not pile onto a core in an unknown state"
+        assert h.restarts == []
+        assert h.state['plugins_pending'] is False and result['plugins_deferred'] is False
+        assert h.state['alert']
+
 
 # -- reporting the health check's outcome --------------------------------------
 

test/test_auto_update_verify.py

@@ -69,6 +69,8 @@ class FakeHost:
         self.running_head = None  # not restarted yet: still the old, healthy code
         self.restarts = []        # (unit, commit it was restarted onto)
         self.pip_installs = []
+        self.pip_timeouts = []
+        self.pip = None  # optional (args, host) -> result, or raises, instead of pip_ok
         self.now = 0.0
         self.nrestarts = 0
 
@@ -90,6 +92,9 @@ class FakeHost:
             return done(args, f'{self.nrestarts}\n')
         if args[0] == 'sudo' and any(a.endswith('safe_pip_install.sh') for a in args):
             self.pip_installs.append(args[-1])
+            self.pip_timeouts.append(kwargs.get('timeout'))
+            if self.pip:
+                return self.pip(args, self)
             return done(args, rc=0 if self.pip_ok else 1)
         if args[:4] == ['sudo', '-n', 'systemctl', 'restart']:
             if self.restart_failures:
@@ -171,6 +176,77 @@ def test_a_failed_dependency_reinstall_is_reported(tmp_path):
     assert 'Install Base Requirements' in result['detail']
 
 
+def _rollback_with_pip(tmp_path, pip, web_requirements_too=False):
+    """Roll back an update that changed the requirements, with pip faked by ``pip``."""
+    repo, old, new = updated_repo(tmp_path, new_requirements=True)
+    if web_requirements_too:
+        (repo / 'web_interface').mkdir()
+        (repo / 'web_interface' / 'requirements.txt').write_text('flask\n')
+        git(repo, 'add', '.')
+        git(repo, 'commit', '-qm', 'web reqs')
+        (repo / 'web_interface' / 'requirements.txt').write_text('flask\nnew\n')
+        (repo / 'requirements.txt').write_text('requests\nnewer\n')
+        git(repo, 'commit', '-qam', 'bump both')
+        old, new = git(repo, 'rev-parse', 'HEAD~1'), git(repo, 'rev-parse', 'HEAD')
+    host = FakeHost(repo, new)
+    host.pip = pip
+    ok, detail = host.verifier().rollback({'old_head': old, 'new_head': new})
+    return ok, detail, host
+
+
+def test_a_failed_pip_is_not_run_again_with_the_other_bash(tmp_path):
+    """Retrying after pip itself ran only repeats it, and every repeat can
+    take PIP_TIMEOUT_SECONDS of the unit's time limit."""
+    ok, detail, host = _rollback_with_pip(
+        tmp_path, lambda args, h: subprocess.CompletedProcess(args, 1, '', 'ERROR: No matching distribution'))
+    assert ok and 'requirements.txt' in detail
+    assert len(host.pip_installs) == 1
+
+
+def test_a_pip_timeout_is_not_retried(tmp_path):
+    def slow(args, h):
+        h.now += h.pip_timeouts[-1]
+        raise subprocess.TimeoutExpired(args, h.pip_timeouts[-1])
+    ok, detail, host = _rollback_with_pip(tmp_path, slow)
+    assert ok and 'Install Base Requirements' in detail
+    assert len(host.pip_installs) == 1
+
+
+def test_a_sudo_refusal_tries_the_next_bash(tmp_path):
+    def refused_once(args, h):
+        if len(h.pip_installs) == 1:
+            return subprocess.CompletedProcess(args, 1, '', 'sudo: a password is required')
+        return done(args)
+    ok, detail, host = _rollback_with_pip(tmp_path, refused_once)
+    assert ok and detail == ''
+    assert len(host.pip_installs) == 2
+
+
+def test_reinstalls_share_one_time_budget(tmp_path):
+    def hangs(args, h):
+        h.now += h.pip_timeouts[-1]
+        raise subprocess.TimeoutExpired(args, h.pip_timeouts[-1])
+    ok, detail, host = _rollback_with_pip(tmp_path, hangs, web_requirements_too=True)
+    assert ok and 'requirements.txt' in detail and 'web_interface/requirements.txt' in detail
+    assert sum(host.pip_timeouts) <= av.PIP_BUDGET_SECONDS
+    assert len(host.pip_installs) == 1, "the first file used the whole budget"
+
+
+def test_the_worst_case_fits_the_unit_time_limit():
+    """systemd kills the check at TimeoutStartSec, mid-rollback, and the update
+    then sits in "verifying" until the web interface calls it lost."""
+    from web_interface import auto_update as au
+    service = (ROOT / 'systemd' / 'ledmatrix-update-verify.service').read_text(encoding='utf-8')
+    minutes = int(re.search(r'^TimeoutStartSec=(\d+)min$', service, re.M).group(1))
+    assert av.WORST_CASE_SECONDS < minutes * 60
+    assert minutes * 60 < au.VERIFY_LOST_SECONDS, "the web UI must not call a running check lost"
+
+
+def test_sudo_refusal_wording_matches_permission_utils():
+    from src.common import permission_utils
+    assert set(av.SUDO_REFUSAL_PHRASES) == set(permission_utils.SUDO_REFUSAL_PHRASES)
+
+
 def test_still_broken_after_rolling_back_is_rollback_failed(tmp_path):
     code, result, host, head, old, new = check(tmp_path, 'always')
     assert code == 1 and result['status'] == 'rollback_failed'

test/test_safe_pip_install_allowlist.py

@@ -0,0 +1,128 @@
+"""scripts/fix_perms/safe_pip_install.sh must accept what the updaters install.
+
+Update Code and the automatic update's health check install the core's own
+requirement files through that root wrapper, and the wrapper refuses any path
+it does not list. It used to list only requirements.txt, so an update that
+changed web_interface/requirements.txt failed its dependency install -- and
+the automatic updater rolls back any update whose dependencies did not
+install, on every device, every week.
+
+The real script runs here with only its final pip command swapped for an
+echo, so the path checks under test are the shipped ones.
+"""
+import importlib.util
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+WRAPPER_REL = Path('scripts') / 'fix_perms' / 'safe_pip_install.sh'
+PIP_LINE = 'exec "$PYTHON_PATH" -m pip install'
+
+
+def _bash():
+    """A bash whose realpath understands --canonicalize-missing, or None."""
+    candidates = [shutil.which('bash')]
+    if os.name == 'nt':
+        candidates.insert(0, r'C:\Program Files\Git\bin\bash.exe')
+    for bash in candidates:
+        if not bash or not os.path.exists(bash):
+            continue
+        try:
+            probe = subprocess.run([bash, '-c', 'realpath --canonicalize-missing /no/such/x'],
+                                   capture_output=True, text=True, timeout=30)
+        except (OSError, subprocess.SubprocessError):
+            continue
+        if probe.returncode == 0:
+            return bash
+    return None
+
+
+BASH = _bash()
+pytestmark = pytest.mark.skipif(BASH is None, reason='no bash with GNU realpath')
+
+
+@pytest.fixture
+def project(tmp_path):
+    """A project tree holding the real wrapper, with pip stubbed out."""
+    root = tmp_path / 'LEDMatrix'
+    wrapper = root / WRAPPER_REL
+    wrapper.parent.mkdir(parents=True)
+    text = (ROOT / WRAPPER_REL).read_text(encoding='utf-8').replace('\r\n', '\n')
+    lines = text.split('\n')
+    pip = [i for i, line in enumerate(lines) if line.startswith(PIP_LINE)]
+    assert len(pip) == 1, 'the wrapper no longer ends in the pip install this test stubs'
+    lines[pip[0]] = 'echo "WOULD INSTALL $RESOLVED_TARGET"; exit 0'
+    wrapper.write_text('\n'.join(lines), encoding='utf-8', newline='\n')
+    for rel in ('requirements.txt', 'web_interface/requirements.txt',
+                'plugin-repos/clock/requirements.txt', 'src/requirements.txt',
+                'web_interface/extra/requirements.txt'):
+        (root / rel).parent.mkdir(parents=True, exist_ok=True)
+        (root / rel).write_text('requests\n', encoding='utf-8')
+    return root
+
+
+def run_wrapper(root, rel):
+    """Run the wrapper on ``rel``, with the path spelled the way bash sees the tree."""
+    return subprocess.run(
+        [BASH, '-c', 'cd "$1" && bash scripts/fix_perms/safe_pip_install.sh "$PWD/$2"',
+         '_', str(root), rel],
+        capture_output=True, text=True, timeout=60)
+
+
+def _core_requirement_files():
+    from web_interface.blueprints.api_v3 import system
+    spec = importlib.util.spec_from_file_location(
+        'auto_update_verify_for_allowlist', ROOT / 'scripts' / 'utils' / 'auto_update_verify.py')
+    verifier = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(verifier)
+    assert set(verifier.REQUIREMENT_FILES) == set(system.CORE_REQUIREMENT_FILES), (
+        'the rollback must reinstall the same files Update Code installs')
+    return system.CORE_REQUIREMENT_FILES
+
+
+@pytest.mark.parametrize('rel', _core_requirement_files())
+def test_every_core_requirement_file_the_updaters_install_is_allowed(project, rel):
+    result = run_wrapper(project, rel)
+    assert result.returncode == 0, result.stderr
+    assert 'WOULD INSTALL' in result.stdout
+
+
+def test_plugin_requirements_are_still_allowed(project):
+    assert run_wrapper(project, 'plugin-repos/clock/requirements.txt').returncode == 0
+
+
+@pytest.mark.parametrize('rel', [
+    'src/requirements.txt',                    # repo-owned folder, but not listed
+    'web_interface/extra/requirements.txt',    # below an allowed file's folder
+    'web_interface/requirements.txt.bak',      # not a requirements.txt
+])
+def test_other_paths_are_still_refused(project, rel):
+    if rel.endswith('.bak'):
+        (project / rel).write_text('requests\n', encoding='utf-8')
+    result = run_wrapper(project, rel)
+    assert result.returncode == 2, result.stdout + result.stderr
+    assert 'DENIED' in result.stderr
+
+
+@pytest.mark.parametrize('rel', ['requirements.txt', 'web_interface/requirements.txt'])
+def test_a_core_requirement_file_symlinked_out_of_the_project_is_refused(project, tmp_path, rel):
+    """Only the allowed files' folders are resolved, so the link's target is
+    compared, and refused, rather than allowed as the file itself."""
+    outside = tmp_path / 'elsewhere' / 'requirements.txt'
+    outside.parent.mkdir()
+    outside.write_text('evil\n', encoding='utf-8')
+    link = project / rel
+    link.unlink()
+    try:
+        link.symlink_to(outside)
+    except (OSError, NotImplementedError):
+        pytest.skip('symlinks unavailable')
+    result = run_wrapper(project, rel)
+    assert result.returncode == 2, result.stdout + result.stderr