LEDMatrix/test/test_auto_update_verify.py

"""The automatic update's health check and rollback (scripts/utils/auto_update_verify.py).

This is what stands between an unattended update and a device that no longer
works, so each way an update can fail is exercised against a real git repo:
the new commit is checked out, the services are "restarted", and whether they
come up healthy depends on which commit they were restarted onto. Only
systemd, sudo and the HTTP check are faked.
"""
import importlib.util
import re
import shutil
import subprocess
import sys
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))

_spec = importlib.util.spec_from_file_location(
    'auto_update_verify', ROOT / 'scripts' / 'utils' / 'auto_update_verify.py')
av = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(av)

pytestmark = pytest.mark.skipif(shutil.which('git') is None, reason='git not installed')


def git(cwd, *args):
    result = subprocess.run(['git', '-c', 'user.name=t', '-c', 'user.email=t@example.com', *args],
                            cwd=str(cwd), capture_output=True, text=True)
    assert result.returncode == 0, result.stderr
    return result.stdout.strip()


def done(args, stdout='', rc=0):
    return subprocess.CompletedProcess(args, rc, stdout=stdout, stderr='' if rc == 0 else 'failed')


def updated_repo(tmp_path, new_requirements=False):
    """A checkout on a new commit, with the commit it came from."""
    repo = tmp_path / 'LEDMatrix'
    repo.mkdir()
    git(repo, 'init', '-q', '-b', 'main')
    (repo / 'app.py').write_text('v = 1\n')
    (repo / 'requirements.txt').write_text('requests\n')
    git(repo, 'add', '.')
    git(repo, 'commit', '-qm', 'old')
    old = git(repo, 'rev-parse', 'HEAD')
    (repo / 'app.py').write_text('v = 2\n')
    if new_requirements:
        (repo / 'requirements.txt').write_text('requests\nnewthing\n')
    git(repo, 'commit', '-qam', 'new')
    return repo, old, git(repo, 'rev-parse', 'HEAD')


class FakeHost:
    """systemd, sudo and the web interface.

    Services run whatever commit was checked out when they were last
    restarted; ``failure`` says how they misbehave on the new commit
    ("display_down", "web_down", "crash_loop") or on any commit ("always").
    """

    def __init__(self, repo, bad_head, failure=None, pip_ok=True, restart_failures=0, count_readable=True):
        self.repo, self.bad_head, self.failure, self.pip_ok = repo, bad_head, failure, pip_ok
        self.restart_failures = restart_failures  # how many restart commands fail, first to last
        self.count_readable = count_readable
        self.running_head = None  # not restarted yet: still the old, healthy code
        self.restarts = []        # (unit, commit it was restarted onto)
        self.pip_installs = []
        self.pip_timeouts = []
        self.pip = None  # optional (args, host) -> result, or raises, instead of pip_ok
        self.now = 0.0
        self.nrestarts = 0

    def broken(self, kind):
        if self.running_head is None:
            return False
        return self.failure == 'always' or (self.running_head == self.bad_head and self.failure == kind)

    def run(self, args, **kwargs):
        if args[0] == 'git':
            return subprocess.run(args, **kwargs)
        if args[:2] == ['systemctl', 'is-active']:
            return done(args, 'failed\n' if self.broken('display_down') else 'active\n')
        if args[:2] == ['systemctl', 'show']:
            if not self.count_readable:
                return done(args, rc=1)
            if self.broken('crash_loop'):
                self.nrestarts += 1
            return done(args, f'{self.nrestarts}\n')
        if args[0] == 'sudo' and any(a.endswith('safe_pip_install.sh') for a in args):
            self.pip_installs.append(args[-1])
            self.pip_timeouts.append(kwargs.get('timeout'))
            if self.pip:
                return self.pip(args, self)
            return done(args, rc=0 if self.pip_ok else 1)
        if args[:4] == ['sudo', '-n', 'systemctl', 'restart']:
            if self.restart_failures:
                self.restart_failures -= 1
                return done(args, rc=1)  # the old process keeps running
            self.running_head = git(self.repo, 'rev-parse', 'HEAD')
            self.restarts.append((args[4], self.running_head))
            return done(args)
        raise AssertionError(f'unexpected command: {args}')

    def web_responds(self):
        return not self.broken('web_down')

    def sleep(self, seconds):
        self.now += seconds

    def verifier(self):
        return av.Verifier(self.repo, run=self.run, sleep=self.sleep, clock=lambda: self.now,
                           web_responds=self.web_responds, log=lambda msg: None)


def check(tmp_path, failure=None, new_requirements=False, pip_ok=True, restart_failures=0,
          count_readable=True, **pending):
    repo, old, new = updated_repo(tmp_path, new_requirements)
    fields = {'status': 'pending', 'old_head': old, 'new_head': new,
              'display_was_active': True, 'dependency_failures': []}
    fields.update(pending)
    av.write_pending(av.pending_path(repo), fields)
    host = FakeHost(repo, new, failure, pip_ok, restart_failures, count_readable)
    code = host.verifier().verify()
    result = av.read_pending(av.pending_path(repo))
    return code, result, host, git(repo, 'rev-parse', 'HEAD'), old, new


def test_a_healthy_update_is_kept(tmp_path):
    code, result, host, head, old, new = check(tmp_path)
    assert code == 0 and result['status'] == 'success'
    assert head == new
    assert host.restarts == [('ledmatrix.service', new), ('ledmatrix-web.service', new)]


@pytest.mark.parametrize('failure, reason', [
    ('display_down', 'the display service did not stay running'),
    ('web_down', 'the web interface did not respond'),
    ('crash_loop', 'the display service kept restarting'),
])
def test_an_unhealthy_update_is_rolled_back(tmp_path, failure, reason):
    code, result, host, head, old, new = check(tmp_path, failure)
    assert code == 0
    assert result['status'] == 'rolled_back' and result['reason'] == reason
    assert head == old
    assert host.restarts[-2:] == [('ledmatrix.service', old), ('ledmatrix-web.service', old)]


def test_a_stopped_display_is_not_started(tmp_path):
    code, result, host, head, old, new = check(tmp_path, display_was_active=False)
    assert result['status'] == 'success'
    assert [unit for unit, _ in host.restarts] == ['ledmatrix-web.service']


def test_failed_dependencies_roll_back_before_anything_restarts_onto_them(tmp_path):
    code, result, host, head, old, new = check(tmp_path, dependency_failures=['requirements.txt'])
    assert result['status'] == 'rolled_back'
    assert 'requirements.txt' in result['reason']
    assert head == old
    assert all(commit == old for _, commit in host.restarts), host.restarts


def test_rollback_reinstalls_the_previous_dependencies(tmp_path):
    code, result, host, head, old, new = check(tmp_path, 'display_down', new_requirements=True)
    assert result['status'] == 'rolled_back' and result['detail'] is None
    assert host.pip_installs == [str(tmp_path / 'LEDMatrix' / 'requirements.txt')]


def test_a_failed_dependency_reinstall_is_reported(tmp_path):
    code, result, host, head, old, new = check(tmp_path, 'display_down', new_requirements=True,
                                               pip_ok=False)
    assert result['status'] == 'rolled_back' and head == old
    assert 'Install Base Requirements' in result['detail']


def _rollback_with_pip(tmp_path, pip, web_requirements_too=False):
    """Roll back an update that changed the requirements, with pip faked by ``pip``."""
    repo, old, new = updated_repo(tmp_path, new_requirements=True)
    if web_requirements_too:
        (repo / 'web_interface').mkdir()
        (repo / 'web_interface' / 'requirements.txt').write_text('flask\n')
        git(repo, 'add', '.')
        git(repo, 'commit', '-qm', 'web reqs')
        (repo / 'web_interface' / 'requirements.txt').write_text('flask\nnew\n')
        (repo / 'requirements.txt').write_text('requests\nnewer\n')
        git(repo, 'commit', '-qam', 'bump both')
        old, new = git(repo, 'rev-parse', 'HEAD~1'), git(repo, 'rev-parse', 'HEAD')
    host = FakeHost(repo, new)
    host.pip = pip
    ok, detail = host.verifier().rollback({'old_head': old, 'new_head': new})
    return ok, detail, host


def test_a_failed_pip_is_not_run_again_with_the_other_bash(tmp_path):
    """Retrying after pip itself ran only repeats it, and every repeat can
    take PIP_TIMEOUT_SECONDS of the unit's time limit."""
    ok, detail, host = _rollback_with_pip(
        tmp_path, lambda args, h: subprocess.CompletedProcess(args, 1, '', 'ERROR: No matching distribution'))
    assert ok and 'requirements.txt' in detail
    assert len(host.pip_installs) == 1


def test_a_pip_timeout_is_not_retried(tmp_path):
    def slow(args, h):
        h.now += h.pip_timeouts[-1]
        raise subprocess.TimeoutExpired(args, h.pip_timeouts[-1])
    ok, detail, host = _rollback_with_pip(tmp_path, slow)
    assert ok and 'Install Base Requirements' in detail
    assert len(host.pip_installs) == 1


def test_a_sudo_refusal_tries_the_next_bash(tmp_path):
    def refused_once(args, h):
        if len(h.pip_installs) == 1:
            return subprocess.CompletedProcess(args, 1, '', 'sudo: a password is required')
        return done(args)
    ok, detail, host = _rollback_with_pip(tmp_path, refused_once)
    assert ok and detail == ''
    assert len(host.pip_installs) == 2


def test_reinstalls_share_one_time_budget(tmp_path):
    def hangs(args, h):
        h.now += h.pip_timeouts[-1]
        raise subprocess.TimeoutExpired(args, h.pip_timeouts[-1])
    ok, detail, host = _rollback_with_pip(tmp_path, hangs, web_requirements_too=True)
    assert ok and 'requirements.txt' in detail and 'web_interface/requirements.txt' in detail
    assert sum(host.pip_timeouts) <= av.PIP_BUDGET_SECONDS
    assert len(host.pip_installs) == 1, "the first file used the whole budget"


def test_the_worst_case_fits_the_unit_time_limit():
    """systemd kills the check at TimeoutStartSec, mid-rollback, and the update
    then sits in "verifying" until the web interface calls it lost."""
    from web_interface import auto_update as au
    service = (ROOT / 'systemd' / 'ledmatrix-update-verify.service').read_text(encoding='utf-8')
    minutes = int(re.search(r'^TimeoutStartSec=(\d+)min$', service, re.M).group(1))
    assert av.WORST_CASE_SECONDS < minutes * 60
    assert minutes * 60 < au.VERIFY_LOST_SECONDS, "the web UI must not call a running check lost"


def test_sudo_refusal_wording_matches_permission_utils():
    from src.common import permission_utils
    assert set(av.SUDO_REFUSAL_PHRASES) == set(permission_utils.SUDO_REFUSAL_PHRASES)


def test_still_broken_after_rolling_back_is_rollback_failed(tmp_path):
    code, result, host, head, old, new = check(tmp_path, 'always')
    assert code == 1 and result['status'] == 'rollback_failed'
    assert 'still unhealthy after rolling back' in result['detail']


def test_a_failed_restart_is_not_mistaken_for_a_healthy_update(tmp_path):
    """When the restart command itself fails the old process keeps answering,
    so checking it would pass an update that never started."""
    code, result, host, head, old, new = check(tmp_path, restart_failures=1)
    assert result['status'] == 'rolled_back'
    assert result['reason'] == 'restarting the services failed'
    assert head == old


def test_restarts_that_keep_failing_after_rollback_are_rollback_failed(tmp_path):
    code, result, host, head, old, new = check(tmp_path, restart_failures=100)
    assert code == 1 and result['status'] == 'rollback_failed'
    assert 'restarting the services failed' in result['detail']


def test_an_unreadable_restart_count_is_never_called_stable(tmp_path):
    """With no restart count a crash loop looks healthy between attempts."""
    code, result, host, head, old, new = check(tmp_path, count_readable=False)
    assert result['status'] != 'success'
    assert 'restart count could not be read' in result['reason']


def test_nothing_pending_does_nothing(tmp_path):
    repo, _, _ = updated_repo(tmp_path)
    host = FakeHost(repo, None)
    assert host.verifier().verify() == 0
    assert host.restarts == []


def test_a_crash_is_recorded_not_left_as_running(tmp_path, monkeypatch):
    repo, old, new = updated_repo(tmp_path)
    av.write_pending(av.pending_path(repo), {'status': 'pending', 'old_head': old, 'new_head': new})

    def boom(self):
        raise RuntimeError('boom')
    monkeypatch.setattr(av.Verifier, 'verify', boom)
    assert av.main(['auto_update_verify.py', str(repo)]) == 1
    result = av.read_pending(av.pending_path(repo))
    assert result['status'] == 'rollback_failed' and result['detail'] == 'boom'


def test_units_installers_and_updater_agree():
    """The request file, the copied verifier, the unit names and the places
    that install them must all line up, or the health check silently never
    starts on real devices -- and code updates stay paused."""
    from web_interface import auto_update as au
    from src import auto_update_setup as aus
    from src.startup_validator import StartupValidator

    service = (ROOT / 'systemd' / aus.SERVICE_UNIT).read_text(encoding='utf-8')
    path = (ROOT / 'systemd' / aus.PATH_UNIT).read_text(encoding='utf-8')
    request = f'__PROJECT_ROOT_DIR__/{au.REQUEST_REL.as_posix()}'
    assert f'PathExists={request}' in path
    assert f'Unit={aus.SERVICE_UNIT}' in path
    assert f'ExecStartPre=/bin/rm -f "{request}"' in service, "the request must be consumed or the path unit re-fires"
    assert f'"__PROJECT_ROOT_DIR__/{au.VERIFIER_COPY_REL.as_posix()}" "__PROJECT_ROOT_DIR__"' in service
    assert au.PENDING_REL.name == av.PENDING_NAME
    assert au.PATH_UNIT == aus.PATH_UNIT and au.SETUP_RESULT_REL == aus.RESULT_REL

    for installer in ('scripts/install/install_web_service.sh', 'scripts/install/install_service.sh'):
        text = (ROOT / installer).read_text(encoding='utf-8')
        assert 'ledmatrix-update-verify.service ledmatrix-update-verify.path' in text, installer
        assert 'enable --now ledmatrix-update-verify.path' in text, installer
    for unit in aus.UNITS:
        assert (f'systemd/{unit}', f'/etc/systemd/system/{unit}') in StartupValidator._UNITS

    # Triggering takes no privilege any more; no sudoers rule should linger.
    for sudoers in ('scripts/install/configure_web_sudo.sh', 'first_time_install.sh'):
        assert not re.search(r'NOPASSWD:.*update-verify', (ROOT / sudoers).read_text(encoding='utf-8')), sudoers