feat(display): systemd watchdog and heartbeat for a frozen render loop (#687)

If the render loop gets stuck inside a plugin's display(), ledmatrix.service
stays active and the panel stays frozen. This adds a way to detect that.

- src/display_watchdog.py (standard library only) sends sd_notify over
  $NOTIFY_SOCKET and writes /run/ledmatrix/display-heartbeat.json. Only the
  render thread counts: beats from other threads are ignored.
- ledmatrix.service: WatchdogSec=120, NotifyAccess=main,
  RuntimeDirectory=ledmatrix (0755), RestartSteps=4 and
  RestartMaxDelaySec=2min. It stays Type=simple. run.py widens the watchdog
  to 15 min for start-up, and load_plugin() does the same on the render
  thread. The loop arms after its first frame.
- /api/v3/health adds checks.display_loop: running, stalled (no heartbeat
  for over 60s, which makes the status degraded) or not_reported. With web
  login on, a caller who is not logged in still gets only healthy/degraded,
  and a stall degrades that answer.
- The update verifier requires a fresh heartbeat from the restarted display
  when the display it replaced was writing one. A frozen panel is rolled
  back.
- Existing installs get the systemd watchdog only after install_service.sh
  is re-run. The heartbeat works right away.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

test/conftest.py

@@ -301,6 +301,20 @@ def emulator_mode(monkeypatch):
     return True
 
 
+@pytest.fixture(autouse=True)
+def _hermetic_display_watchdog(monkeypatch):
+    """Keep the render loop's watchdog off the host.
+
+    Every test that runs DisplayController.run() arms the process-wide
+    watchdog, which would ping a real $NOTIFY_SOCKET and write a heartbeat
+    into /run/ledmatrix -- the live display's, when the suite runs as root on
+    a device. Each test gets a fresh instance that does neither.
+    """
+    from src import display_watchdog
+    monkeypatch.setattr(display_watchdog, 'watchdog',
+                        display_watchdog.RenderWatchdog(environ={}, heartbeat_dir=None))
+
+
 @pytest.fixture(autouse=True)
 def reset_logging():
     """Reset logging configuration before each test."""

test/test_api_v3_health.py

@@ -5,8 +5,10 @@ PluginManager does not have; a hasattr guard turned that into a permanent 0.
 Each check that fails answers "see logs for details", so it has to log.
 """
 
+import json
 import logging
 import sys
+import time
 from pathlib import Path
 from types import SimpleNamespace
 
@@ -25,6 +27,28 @@ def _no_systemctl(monkeypatch):
                         lambda: {"active": True})
 
 
+@pytest.fixture(autouse=True)
+def heartbeat(tmp_path, monkeypatch):
+    """The display's heartbeat file, somewhere private; absent until written."""
+    from src import display_watchdog
+    path = tmp_path / "display-heartbeat.json"
+    monkeypatch.setattr(display_watchdog, "HEARTBEAT_PATH", str(path))
+
+    def write(age):
+        path.write_text(json.dumps({"pid": 1, "mono": time.monotonic() - age,
+                                    "wall": time.time() - age}))
+    return write
+
+
+@pytest.fixture
+def fresh_preview(tmp_path, monkeypatch):
+    """A just-written preview frame, so only the heartbeat decides the verdict."""
+    from web_interface import display_preview
+    snapshot = tmp_path / "preview.png"
+    snapshot.write_bytes(b"png")
+    monkeypatch.setattr(display_preview, "SNAPSHOT_PATH", str(snapshot))
+
+
 def _checks(client):
     response = client.get(URL)
     assert response.status_code == 200, response.get_json()
@@ -88,3 +112,59 @@ def test_a_failed_hardware_check_is_logged(api_v3_client, api_v3_module, caplog,
     assert check["status"] == "unknown"
     logged = [r for r in caplog.records if "snapshot" in r.getMessage()]
     assert logged and logged[0].exc_info
+
+
+# -- the display's render-loop heartbeat -----------------------------------------
+#
+# The preview frame's age said nothing about a panel frozen by a render thread
+# stuck in a plugin; the heartbeat is written by that thread itself.
+
+def _health(client):
+    response = client.get(URL)
+    assert response.status_code == 200, response.get_json()
+    return response.get_json()["data"]
+
+
+def test_a_fresh_heartbeat_is_a_running_display_loop(api_v3_client, heartbeat, fresh_preview):
+    heartbeat(age=3)
+
+    data = _health(api_v3_client)
+
+    assert data["checks"]["display_loop"]["status"] == "running"
+    assert 2 <= data["checks"]["display_loop"]["heartbeat_age_seconds"] < 10
+    assert data["status"] == "healthy"
+
+
+def test_a_stale_heartbeat_is_a_stalled_display_loop(api_v3_client, heartbeat, fresh_preview):
+    """Service active, preview recent, and still the panel is frozen."""
+    heartbeat(age=300)
+
+    data = _health(api_v3_client)
+
+    assert data["checks"]["display_loop"]["status"] == "stalled"
+    assert data["checks"]["display_loop"]["heartbeat_age_seconds"] >= 299
+    assert data["status"] == "degraded"
+
+
+def test_no_heartbeat_falls_back_to_the_older_checks(api_v3_client, fresh_preview):
+    """The dev server, the emulator, Windows, or a display without the
+    feature: absence is not a failure, and the verdict is what it was."""
+    data = _health(api_v3_client)
+
+    assert data["checks"]["display_loop"]["status"] == "not_reported"
+    assert data["checks"]["hardware"]["status"] == "connected"
+    assert data["status"] == "healthy"
+
+
+def test_an_unreadable_heartbeat_is_reported_not_raised(api_v3_client, monkeypatch, caplog):
+    from src import display_watchdog
+
+    def boom(_path):
+        raise RuntimeError("bad heartbeat")
+    monkeypatch.setattr(display_watchdog, "read_heartbeat", boom)
+
+    with caplog.at_level(logging.WARNING):
+        check = _checks(api_v3_client)["display_loop"]
+
+    assert check["status"] == "unknown"
+    assert any("heartbeat" in r.getMessage() for r in caplog.records)

test/test_auto_update_verify.py

@@ -59,10 +59,15 @@ class FakeHost:
 
     Services run whatever commit was checked out when they were last
     restarted; ``failure`` says how they misbehave on the new commit
-    ("display_down", "web_down", "crash_loop") or on any commit ("always").
+    ("display_down", "web_down", "crash_loop", "frozen": active but the
+    render loop stuck after its first frame) or on any commit ("always").
+
+    ``heartbeat`` is whether the display writes one: never (``None``, code
+    from before the heartbeat), or on every commit (``"always"``).
     """
 
-    def __init__(self, repo, bad_head, failure=None, pip_ok=True, restart_failures=0, count_readable=True):
+    def __init__(self, repo, bad_head, failure=None, pip_ok=True, restart_failures=0, count_readable=True,
+                 heartbeat=None):
         self.repo, self.bad_head, self.failure, self.pip_ok = repo, bad_head, failure, pip_ok
         self.restart_failures = restart_failures  # how many restart commands fail, first to last
         self.count_readable = count_readable
@@ -73,6 +78,8 @@ class FakeHost:
         self.pip = None  # optional (args, host) -> result, or raises, instead of pip_ok
         self.now = 0.0
         self.nrestarts = 0
+        self.heartbeat = heartbeat
+        self.display_started_at = -1000.0  # the pre-update display, long running
 
     def broken(self, kind):
         if self.running_head is None:
@@ -102,28 +109,43 @@ class FakeHost:
                 return done(args, rc=1)  # the old process keeps running
             self.running_head = git(self.repo, 'rev-parse', 'HEAD')
             self.restarts.append((args[4], self.running_head))
+            if args[4] == 'ledmatrix.service':
+                self.display_started_at = self.now
             return done(args)
         raise AssertionError(f'unexpected command: {args}')
 
     def web_responds(self):
         return not self.broken('web_down')
 
+    def read_heartbeat(self):
+        if self.heartbeat is None:
+            return None
+        first_frame = self.display_started_at + 10  # plugins load, then it draws
+        if self.now < first_frame:
+            # Nothing from this process yet. A display whose unit predates
+            # RuntimeDirectory= leaves its predecessor's file behind.
+            return {'mono': self.display_started_at - 1}
+        if self.broken('frozen'):
+            return {'mono': first_frame}  # drew once, then stuck
+        return {'mono': self.now}
+
     def sleep(self, seconds):
         self.now += seconds
 
     def verifier(self):
         return av.Verifier(self.repo, run=self.run, sleep=self.sleep, clock=lambda: self.now,
-                           web_responds=self.web_responds, log=lambda msg: None)
+                           web_responds=self.web_responds, log=lambda msg: None,
+                           read_heartbeat=self.read_heartbeat)
 
 
 def check(tmp_path, failure=None, new_requirements=False, pip_ok=True, restart_failures=0,
-          count_readable=True, **pending):
+          count_readable=True, heartbeat=None, **pending):
     repo, old, new = updated_repo(tmp_path, new_requirements)
     fields = {'status': 'pending', 'old_head': old, 'new_head': new,
               'display_was_active': True, 'dependency_failures': []}
     fields.update(pending)
     av.write_pending(av.pending_path(repo), fields)
-    host = FakeHost(repo, new, failure, pip_ok, restart_failures, count_readable)
+    host = FakeHost(repo, new, failure, pip_ok, restart_failures, count_readable, heartbeat)
     code = host.verifier().verify()
     result = av.read_pending(av.pending_path(repo))
     return code, result, host, git(repo, 'rev-parse', 'HEAD'), old, new
@@ -323,3 +345,63 @@ def test_units_installers_and_updater_agree():
     for sudoers in ('scripts/install/configure_web_sudo.sh', 'first_time_install.sh',
                     'scripts/install/lib_sudoers.sh'):
         assert not re.search(r'NOPASSWD:.*update-verify', (ROOT / sudoers).read_text(encoding='utf-8')), sudoers
+
+
+# -- the display's heartbeat -------------------------------------------------------
+#
+# "Service active" plus one HTTP 200 passed a panel frozen by a render loop
+# stuck in a plugin. Where the display writes a heartbeat, the restarted
+# display has to keep it fresh too.
+
+FROZEN_REASON = ('the display service is running but its panel is not being drawn '
+                 '(no fresh heartbeat)')
+
+
+def test_a_display_that_keeps_drawing_passes(tmp_path):
+    code, result, host, head, old, new = check(tmp_path, heartbeat='always')
+    assert result['status'] == 'success' and head == new
+
+
+def test_a_frozen_panel_is_rolled_back(tmp_path):
+    code, result, host, head, old, new = check(tmp_path, 'frozen', heartbeat='always')
+    assert result['status'] == 'rolled_back' and result['reason'] == FROZEN_REASON
+    assert head == old
+
+
+def test_the_previous_processs_heartbeat_does_not_count(tmp_path):
+    """Under a unit without RuntimeDirectory= the old file outlives the old
+    process; a restarted display that never draws must not pass on it."""
+    repo, old, new = updated_repo(tmp_path)
+    host = FakeHost(repo, new, heartbeat='always')
+    verifier = host.verifier()
+    verifier.expect_heartbeat = True
+    host.display_started_at = host.now = 100.0
+    verifier.display_restarted_at = 100.0
+    host.now = 101.0  # the new process has not drawn yet
+    assert verifier.display_drawing() is False
+    host.now = 115.0
+    assert verifier.display_drawing() is True
+
+
+def test_without_a_heartbeat_the_check_is_what_it_was(tmp_path):
+    """Code from before the heartbeat (or a display that cannot write one)
+    never wrote one, so it cannot be asked for -- a frozen panel then passes,
+    exactly as it did."""
+    code, result, host, head, old, new = check(tmp_path, 'frozen', heartbeat=None)
+    assert result['status'] == 'success'
+
+
+def test_a_stopped_display_is_not_asked_for_a_heartbeat(tmp_path):
+    code, result, host, head, old, new = check(tmp_path, 'frozen', heartbeat='always',
+                                               display_was_active=False)
+    assert result['status'] == 'success'
+
+
+def test_the_heartbeat_location_and_freshness_match_the_display():
+    """A copy, not an import: the verifier must not depend on the code it checks."""
+    from src import display_watchdog
+    assert av.HEARTBEAT_PATH == display_watchdog.HEARTBEAT_PATH
+    # A display frozen right after its first frame must go stale inside the
+    # window it has to stay healthy for.
+    assert av.HEARTBEAT_FRESH_SECONDS + av.POLL_SECONDS < av.STABLE_SECONDS
+    assert av.HEARTBEAT_FRESH_SECONDS > display_watchdog.BEAT_INTERVAL_SECONDS * 2

test/test_display_watchdog.py

@@ -0,0 +1,660 @@
+"""The render loop's liveness signals: systemd watchdog pings and the heartbeat.
+
+A panel can freeze while ledmatrix.service stays "active" -- a render thread
+stuck inside a plugin's display(). src/display_watchdog.py lets only the
+render thread vouch for itself, to systemd (sd_notify WATCHDOG=1) and to the
+web interface (a heartbeat file under /run/ledmatrix). These tests pin:
+
+* the sd_notify wire format, including abstract-namespace sockets;
+* that nothing is armed until the first frame, so start-up keeps its allowance;
+* that beats from any other thread are ignored, so a stuck render thread
+  goes quiet even while the update worker and Vegas's tick thread carry on;
+* the places the render loop checks in from (dwell sleeps, per-frame
+  display, Vegas's own loop, a plugin load's longer allowance).
+"""
+import json
+import os
+import socket
+import sys
+import threading
+import time
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+os.environ.setdefault("EMULATOR", "true")  # display_controller imports without hardware
+
+from src import display_watchdog  # noqa: E402
+from src.display_watchdog import (  # noqa: E402
+    RenderWatchdog, heartbeat_age, notify, read_heartbeat, watchdog_usec)
+
+WATCHDOG_120 = {'NOTIFY_SOCKET': '/run/systemd/notify', 'WATCHDOG_USEC': '120000000'}
+
+
+class FakeSocket:
+    """Records what notify() does with the socket it creates."""
+
+    def __init__(self, record, fail_connect=False):
+        self.record = record
+        self.fail_connect = fail_connect
+        self.closed = False
+
+    def connect(self, address):
+        self.record['address'] = address
+        if self.fail_connect:
+            raise ConnectionRefusedError('nobody listening')
+
+    def sendall(self, data):
+        self.record.setdefault('sent', []).append(data)
+
+    def close(self):
+        self.closed = True
+        self.record['closed'] = True
+
+
+def fake_factory(record, fail_connect=False):
+    def factory(family, kind):
+        record['family'], record['type'] = family, kind
+        return FakeSocket(record, fail_connect)
+    return factory
+
+
+@pytest.fixture
+def af_unix(monkeypatch):
+    """AF_UNIX for the fake-socket tests, even on a Python built without it."""
+    monkeypatch.setattr(socket, 'AF_UNIX', getattr(socket, 'AF_UNIX', 1), raising=False)
+    return socket.AF_UNIX
+
+
+# -- sd_notify -------------------------------------------------------------
+
+class TestNotify:
+    def test_sends_one_datagram_to_the_socket_path(self, af_unix):
+        record = {}
+        assert notify('WATCHDOG=1', {'NOTIFY_SOCKET': '/run/systemd/notify'},
+                      socket_factory=fake_factory(record)) is True
+        assert record['family'] == af_unix
+        assert record['type'] & socket.SOCK_DGRAM == socket.SOCK_DGRAM
+        assert record['address'] == '/run/systemd/notify'
+        assert record['sent'] == [b'WATCHDOG=1']
+        assert record['closed']
+
+    def test_an_at_sign_means_the_abstract_namespace(self, af_unix):
+        record = {}
+        notify('READY=1\nSTATUS=Rendering', {'NOTIFY_SOCKET': '@/org/freedesktop/systemd1/notify'},
+               socket_factory=fake_factory(record))
+        assert record['address'] == '\0/org/freedesktop/systemd1/notify'
+        assert record['sent'] == [b'READY=1\nSTATUS=Rendering']
+
+    @pytest.mark.parametrize('address', [None, '', 'relative/path', 'vsock:2:1234'])
+    def test_no_usable_socket_sends_nothing(self, af_unix, address):
+        record = {}
+        env = {} if address is None else {'NOTIFY_SOCKET': address}
+        assert notify('WATCHDOG=1', env, socket_factory=fake_factory(record)) is False
+        assert record == {}
+
+    def test_a_failed_send_is_false_not_an_exception(self, af_unix):
+        record = {}
+        assert notify('WATCHDOG=1', {'NOTIFY_SOCKET': '/nope'},
+                      socket_factory=fake_factory(record, fail_connect=True)) is False
+        assert record['closed']
+
+    @pytest.mark.skipif(not hasattr(socket, 'AF_UNIX') or os.name != 'posix',
+                        reason='needs AF_UNIX datagram sockets')
+    def test_a_real_socket_receives_the_message(self, tmp_path):
+        path = str(tmp_path / 'notify')
+        server = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+        try:
+            server.bind(path)
+            server.settimeout(2)
+            assert notify('WATCHDOG=1', {'NOTIFY_SOCKET': path}) is True
+            assert server.recv(4096) == b'WATCHDOG=1'
+        finally:
+            server.close()
+
+    @pytest.mark.skipif(not sys.platform.startswith('linux'),
+                        reason='abstract sockets are Linux-only')
+    def test_a_real_abstract_socket_receives_the_message(self):
+        name = f'ledmatrix-test-{os.getpid()}-{time.monotonic_ns()}'
+        server = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+        try:
+            server.bind('\0' + name)
+            server.settimeout(2)
+            assert notify('READY=1', {'NOTIFY_SOCKET': '@' + name}) is True
+            assert server.recv(4096) == b'READY=1'
+        finally:
+            server.close()
+
+
+class TestWatchdogUsec:
+    def test_reads_the_units_value(self):
+        assert watchdog_usec({'WATCHDOG_USEC': '120000000'}) == 120_000_000
+
+    def test_meant_for_another_process(self):
+        assert watchdog_usec({'WATCHDOG_USEC': '120000000',
+                              'WATCHDOG_PID': str(os.getpid() + 1)}) is None
+
+    def test_meant_for_this_process(self):
+        assert watchdog_usec({'WATCHDOG_USEC': '5000000',
+                              'WATCHDOG_PID': str(os.getpid())}) == 5_000_000
+
+    @pytest.mark.parametrize('value', [None, '', 'abc', '0', '-5'])
+    def test_no_watchdog(self, value):
+        env = {} if value is None else {'WATCHDOG_USEC': value}
+        assert watchdog_usec(env) is None
+
+
+# -- the render loop's side ----------------------------------------------------
+
+class Clock:
+    def __init__(self, now=1000.0):
+        self.now = now
+
+    def __call__(self):
+        return self.now
+
+
+def make(environ=WATCHDOG_120, heartbeat_dir=None, clock=None):
+    sent = []
+    wd = RenderWatchdog(environ=environ, send=lambda m: sent.append(m) or True,
+                        clock=clock or Clock(), wall_clock=lambda: 1_700_000_000.0,
+                        heartbeat_dir=heartbeat_dir, enable_faulthandler=False)
+    return wd, sent
+
+
+def pings(sent):
+    return [m for m in sent if 'WATCHDOG=1' in m.split('\n')]
+
+
+def on_other_thread(fn):
+    t = threading.Thread(target=fn)
+    t.start()
+    t.join()
+
+
+class TestStartup:
+    def test_start_up_widens_the_limit(self):
+        wd, sent = make()
+        wd.begin_startup()
+        assert sent == [f'WATCHDOG_USEC={int(display_watchdog.STARTUP_ALLOWANCE_SECONDS * 1e6)}'
+                        '\nSTATUS=Starting: loading plugins']
+
+    def test_start_up_never_shortens_a_longer_unit_limit(self):
+        wd, sent = make({'NOTIFY_SOCKET': '/x', 'WATCHDOG_USEC': str(3600 * 10**6)})
+        wd.begin_startup()
+        assert sent[0].startswith(f'WATCHDOG_USEC={3600 * 10**6}\n')
+
+    def test_without_a_watchdog_start_up_sends_nothing(self):
+        wd, sent = make({'NOTIFY_SOCKET': '/x'})
+        wd.begin_startup()
+        assert sent == []
+
+
+class TestArming:
+    def test_nothing_before_the_render_loop_starts(self, tmp_path):
+        wd, sent = make(heartbeat_dir=str(tmp_path))
+        wd.note_frame()  # a start-up screen
+        wd.beat()
+        wd.loop_pass()
+        assert sent == [] and not wd.armed
+        assert not (tmp_path / display_watchdog.HEARTBEAT_NAME).exists()
+
+    def test_nothing_before_the_first_frame(self, tmp_path):
+        wd, sent = make(heartbeat_dir=str(tmp_path))
+        wd.bind_render_thread()
+        wd.loop_pass()   # the first pass has begun...
+        wd.beat()        # ...and is, say, composing Vegas content
+        assert sent == [] and not wd.armed
+        assert not (tmp_path / display_watchdog.HEARTBEAT_NAME).exists()
+
+    def test_the_first_frame_arms_it(self, tmp_path):
+        wd, sent = make(heartbeat_dir=str(tmp_path))
+        wd.bind_render_thread()
+        wd.note_frame()
+        assert wd.armed
+        assert sent == ['READY=1\nWATCHDOG_USEC=120000000\nWATCHDOG=1\nSTATUS=Rendering']
+        heartbeat = json.loads((tmp_path / display_watchdog.HEARTBEAT_NAME).read_text())
+        assert heartbeat == {'pid': os.getpid(), 'mono': 1000.0, 'wall': 1_700_000_000.0}
+
+    def test_a_first_frame_pushed_by_another_thread_arms_on_the_next_render_beat(self):
+        """A screen's first display() runs on PluginExecutor's thread."""
+        wd, sent = make()
+        wd.bind_render_thread()
+        on_other_thread(wd.note_frame)
+        assert not wd.armed and sent == []
+        wd.beat()
+        assert wd.armed and sent[0].startswith('READY=1\n')
+
+    def test_a_full_pass_with_nothing_drawn_arms_it(self):
+        """No plugins enabled, or every screen empty: the loop is still alive."""
+        wd, sent = make()
+        wd.bind_render_thread()
+        wd.loop_pass()
+        assert not wd.armed
+        wd.loop_pass()
+        assert wd.armed and sent[0].startswith('READY=1\n')
+
+    def test_without_a_watchdog_it_still_says_ready_and_writes_the_heartbeat(self, tmp_path):
+        wd, sent = make({'NOTIFY_SOCKET': '/x'}, heartbeat_dir=str(tmp_path))
+        wd.bind_render_thread()
+        wd.note_frame()
+        assert sent == ['READY=1\nSTATUS=Rendering']
+        assert (tmp_path / display_watchdog.HEARTBEAT_NAME).exists()
+        sent_before = len(sent)
+        wd.beat()
+        assert len(sent) == sent_before  # no WATCHDOG=1 without a watchdog
+
+
+class TestBeats:
+    def test_pings_are_rate_limited(self, tmp_path):
+        clock = Clock()
+        wd, sent = make(heartbeat_dir=str(tmp_path), clock=clock)
+        wd.bind_render_thread()
+        wd.note_frame()
+        del sent[:]
+        for _ in range(100):  # a burst of frames
+            wd.beat()
+        assert sent == []
+        clock.now += display_watchdog.BEAT_INTERVAL_SECONDS
+        wd.beat()
+        assert sent == ['WATCHDOG=1']
+        heartbeat = read_heartbeat(str(tmp_path / display_watchdog.HEARTBEAT_NAME))
+        assert heartbeat['mono'] == clock.now
+
+    def test_a_short_unit_limit_pings_more_often(self):
+        clock = Clock()
+        wd, sent = make({'NOTIFY_SOCKET': '/x', 'WATCHDOG_USEC': '3000000'}, clock=clock)
+        wd.bind_render_thread()
+        wd.note_frame()
+        del sent[:]
+        clock.now += 1.0  # a third of 3s
+        wd.beat()
+        assert sent == ['WATCHDOG=1']
+
+    def test_beats_from_other_threads_are_ignored(self, tmp_path):
+        """The update worker, Vegas's tick thread and the prefetcher keep
+        running while the render thread is stuck; they must not keep the
+        watchdog fed on its behalf."""
+        clock = Clock()
+        wd, sent = make(heartbeat_dir=str(tmp_path), clock=clock)
+        wd.bind_render_thread()
+        wd.note_frame()
+        del sent[:]
+        before = (tmp_path / display_watchdog.HEARTBEAT_NAME).read_text()
+        clock.now += 60
+        on_other_thread(wd.beat)
+        on_other_thread(wd.note_frame)
+        on_other_thread(wd.loop_pass)
+        assert sent == []
+        assert (tmp_path / display_watchdog.HEARTBEAT_NAME).read_text() == before
+
+    def test_the_module_shortcuts_reach_the_process_instance(self, monkeypatch):
+        wd, sent = make()
+        monkeypatch.setattr(display_watchdog, 'watchdog', wd)
+        wd.bind_render_thread()
+        display_watchdog.note_frame()
+        assert wd.armed
+        with display_watchdog.extended(600, 'x'):
+            pass
+        display_watchdog.beat()
+        assert any('WATCHDOG_USEC=600000000' in m for m in sent)
+
+
+class TestExtended:
+    def test_a_long_job_gets_the_longer_limit_then_the_units_back(self):
+        wd, sent = make()
+        wd.bind_render_thread()
+        wd.note_frame()
+        del sent[:]
+        with wd.extended(900, 'loading plugin weather'):
+            assert sent == ['WATCHDOG_USEC=900000000\nWATCHDOG=1\nSTATUS=Busy: loading plugin weather']
+        assert sent[-1] == 'WATCHDOG_USEC=120000000\nWATCHDOG=1\nSTATUS=Rendering'
+
+    def test_nested_jobs_restore_once(self):
+        wd, sent = make()
+        wd.bind_render_thread()
+        wd.note_frame()
+        del sent[:]
+        with wd.extended(900):
+            with wd.extended(900):
+                pass
+            assert len(sent) == 1  # the inner one neither re-extends nor restores
+        assert len(sent) == 2 and sent[-1].startswith('WATCHDOG_USEC=120000000\n')
+
+    def test_restored_even_when_the_job_fails(self):
+        wd, sent = make()
+        wd.bind_render_thread()
+        wd.note_frame()
+        with pytest.raises(RuntimeError):
+            with wd.extended(900):
+                raise RuntimeError('pip failed')
+        assert sent[-1].startswith('WATCHDOG_USEC=120000000\n')
+
+    def test_off_the_render_thread_or_before_arming_it_does_nothing(self):
+        """Start-up loads run on a thread pool under the start-up allowance."""
+        wd, sent = make()
+        wd.bind_render_thread()
+        with wd.extended(900):
+            pass
+        wd.note_frame()
+        del sent[:]
+        on_other_thread(lambda: wd.extended(900).__enter__())
+        assert sent == []
+
+
+class TestStopping:
+    def test_a_clean_stop_removes_the_heartbeat(self, tmp_path):
+        wd, sent = make(heartbeat_dir=str(tmp_path))
+        wd.bind_render_thread()
+        wd.note_frame()
+        wd.stopping()
+        assert sent[-1] == 'STOPPING=1'
+        assert not (tmp_path / display_watchdog.HEARTBEAT_NAME).exists()
+
+    def test_stopping_outside_systemd_is_harmless(self):
+        wd, sent = make({})
+        wd.stopping()
+        assert sent == []
+
+
+class TestHeartbeatFile:
+    def test_the_directory_is_created_when_missing(self, tmp_path):
+        """An install whose unit predates RuntimeDirectory=; the display is root."""
+        target = tmp_path / 'run' / 'ledmatrix'
+        wd, _ = make(heartbeat_dir=str(target))
+        wd.bind_render_thread()
+        wd.note_frame()
+        assert (target / display_watchdog.HEARTBEAT_NAME).is_file()
+        assert [p.name for p in target.iterdir()] == [display_watchdog.HEARTBEAT_NAME]
+
+    @pytest.mark.skipif(os.name != 'posix', reason='POSIX permissions')
+    def test_other_users_can_read_it(self, tmp_path):
+        wd, _ = make(heartbeat_dir=str(tmp_path))
+        wd.bind_render_thread()
+        wd.note_frame()
+        mode = (tmp_path / display_watchdog.HEARTBEAT_NAME).stat().st_mode & 0o777
+        assert mode == 0o644
+
+    def test_nowhere_to_write_is_not_an_error(self, tmp_path):
+        blocker = tmp_path / 'not-a-dir'
+        blocker.write_text('x')
+        clock = Clock()
+        wd, sent = make(heartbeat_dir=str(blocker / 'ledmatrix'), clock=clock)
+        wd.bind_render_thread()
+        wd.note_frame()
+        clock.now += 10
+        wd.beat()
+        assert wd.armed and pings(sent)  # the watchdog works regardless
+
+    def test_windows_gets_no_heartbeat_by_default(self, monkeypatch):
+        monkeypatch.setattr(display_watchdog.os, 'name', 'nt')
+        wd = RenderWatchdog(environ={}, send=lambda m: True)
+        assert wd._heartbeat_path() is None
+
+
+class TestHeartbeatAge:
+    def test_monotonic_is_preferred(self):
+        assert heartbeat_age({'mono': 100.0, 'wall': 0.0}, now_mono=112.5, now_wall=9e9) == 12.5
+
+    def test_the_wall_clock_is_the_fallback(self):
+        assert heartbeat_age({'wall': 50.0}, now_mono=1.0, now_wall=80.0) == 30.0
+
+    def test_a_monotonic_stamp_from_the_future_is_not_trusted(self):
+        """Not the same clock -- fall back rather than report a fresh heartbeat."""
+        assert heartbeat_age({'mono': 500.0, 'wall': 50.0}, now_mono=100.0, now_wall=170.0) == 120.0
+
+    def test_no_time_at_all(self):
+        assert heartbeat_age({'pid': 1}) is None
+        assert heartbeat_age({'mono': True}) is None
+
+    def test_reading_a_missing_or_broken_file(self, tmp_path):
+        assert read_heartbeat(str(tmp_path / 'absent.json')) is None
+        (tmp_path / 'broken.json').write_text('{not json')
+        assert read_heartbeat(str(tmp_path / 'broken.json')) is None
+        (tmp_path / 'list.json').write_text('[1, 2]')
+        assert read_heartbeat(str(tmp_path / 'list.json')) is None
+
+
+# -- where the render loop checks in -------------------------------------------
+
+@pytest.fixture
+def armed(monkeypatch):
+    """A process watchdog bound to this thread and armed, with a clock to advance."""
+    clock = Clock()
+    wd, sent = make(clock=clock)
+    monkeypatch.setattr(display_watchdog, 'watchdog', wd)
+    wd.bind_render_thread()
+    wd.note_frame()
+    del sent[:]
+    return SimpleNamespace(wd=wd, sent=sent, clock=clock)
+
+
+def _tick_clock(armed):
+    """Advance the watchdog's clock past the rate limit on every beat check."""
+    original = armed.wd._clock
+
+    def advancing():
+        armed.clock.now += display_watchdog.BEAT_INTERVAL_SECONDS
+        return original()
+    armed.wd._clock = advancing
+
+
+class TestCheckInPoints:
+    def test_the_dwell_sleep_checks_in(self, armed):
+        from src.display_controller import DisplayController
+        dc = object.__new__(DisplayController)
+        dc.current_display_mode = 'm'
+        dc.is_display_active = True
+        dc.on_demand_active = False
+        dc._tick_plugin_updates = lambda: None
+        dc._service_pending_changes = lambda: None
+        _tick_clock(armed)
+        dc._sleep_with_plugin_updates(0.05, tick_interval=0.01)
+        assert len(pings(armed.sent)) >= 3
+
+    def test_every_frame_of_a_screen_checks_in(self, armed):
+        from src.display_controller import DisplayController
+        dc = object.__new__(DisplayController)
+        dc.plugin_manager = None
+        plugin = MagicMock(plugin_id='p')
+        _tick_clock(armed)
+        for _ in range(3):
+            dc._display_once(plugin, 'm', accepts_display_mode=False)
+        assert len(pings(armed.sent)) == 3
+
+    def test_vegas_checks_in_every_frame_of_its_own_loop(self, armed):
+        """An iteration runs for minutes without returning to run()."""
+        import threading as _threading
+        from src.vegas_mode.config import VegasModeConfig
+        from src.vegas_mode.coordinator import VegasModeCoordinator
+        coord = VegasModeCoordinator.__new__(VegasModeCoordinator)
+        coord.vegas_config = VegasModeConfig.from_config({'display': {'vegas_scroll': {
+            'enabled': True, 'max_cycle_duration': 60}}})
+        coord.render_pipeline = MagicMock(frame_interval=0.0, target_fps=90)
+        coord.display_manager = MagicMock()
+        coord._state_lock = _threading.Lock()
+        coord._is_active = True
+        coord._is_paused = False
+        coord._should_stop = False
+        coord._live_priority_active = False
+        coord._fps_last_health_log = 0.0
+        coord._fps_was_degraded = False
+        coord._interrupt_check = None
+        coord._interrupt_check_interval = 10
+        coord._update_callback = None
+        coord._update_tick_running = False
+        coord._check_static_plugin_trigger = lambda: None
+        frames = []
+
+        def run_frame():
+            frames.append(1)
+            if len(frames) == 5:
+                coord._should_stop = True
+                return False
+            return True
+        coord.run_frame = run_frame
+        _tick_clock(armed)
+        coord.run_iteration()
+        assert len(pings(armed.sent)) == 5
+
+    def test_each_plugin_fetched_for_a_vegas_cycle_checks_in(self, armed):
+        from src.vegas_mode.stream_manager import StreamManager
+        sm = StreamManager.__new__(StreamManager)
+        sm.plugin_manager = SimpleNamespace(plugins={})
+        _tick_clock(armed)
+        for plugin_id in ('a', 'b'):
+            sm._fetch_plugin_content(plugin_id)
+        assert len(pings(armed.sent)) == 2
+
+    def test_loading_a_plugin_gets_the_longer_limit(self, armed):
+        from src.plugin_system.plugin_manager import PluginManager
+        pm = PluginManager.__new__(PluginManager)
+        seen = []
+        pm._load_plugin = lambda plugin_id, force_enabled=False: seen.append(list(armed.sent)) or True
+        assert pm.load_plugin('weather') is True
+        allowance = int(display_watchdog.PLUGIN_LOAD_ALLOWANCE_SECONDS * 1e6)
+        assert seen[0] and seen[0][-1].startswith(f'WATCHDOG_USEC={allowance}\n')
+        assert armed.sent[-1].startswith('WATCHDOG_USEC=120000000\n')
+
+
+class TestStuckRenderThread:
+    def test_a_render_thread_stuck_in_display_stops_the_pings(self, test_display_controller,
+                                                             monkeypatch):
+        """End to end through DisplayController.run(): pings flow while frames
+        do, stop while display() is stuck even though other threads keep
+        calling beat(), and nothing else in the process keeps them alive."""
+        sent = []
+        lock = threading.Lock()
+
+        def record(message):
+            with lock:
+                sent.append(message)
+            return True
+        # 0.3s limit -> a ping at most every 0.1s.
+        wd = RenderWatchdog(environ={'NOTIFY_SOCKET': '/x', 'WATCHDOG_USEC': '300000'},
+                            send=record, heartbeat_dir=None, enable_faulthandler=False)
+        monkeypatch.setattr(display_watchdog, 'watchdog', wd)
+
+        stuck, release = threading.Event(), threading.Event()
+
+        class Plugin:
+            plugin_id = 'stuck-plugin'
+            needs_high_fps = True
+            enabled = True
+            calls = 0
+
+            def display(self, force_clear=False):
+                Plugin.calls += 1
+                display_watchdog.note_frame()  # DisplayManager is mocked here
+                if Plugin.calls >= 60:  # about half a second of frames
+                    stuck.set()
+                    release.wait(10)
+                    raise KeyboardInterrupt  # ends run() the way SIGTERM does
+
+        controller = test_display_controller
+        controller.available_modes = ['stuck-mode']
+        controller.plugin_modes = {'stuck-mode': Plugin()}
+        controller.mode_to_plugin_id = {'stuck-mode': 'stuck-plugin'}
+        controller.current_mode_index = 0
+
+        runner = threading.Thread(target=controller.run, daemon=True)
+        runner.start()
+        assert stuck.wait(10), 'the render loop never reached the plugin'
+        with lock:
+            before = len(pings(sent))
+        assert wd.armed and any(m.startswith('READY=1') for m in sent)
+        assert before >= 2, sent
+
+        # Other threads carry on while the render thread is stuck.
+        stop_others = threading.Event()
+
+        def busy_other_thread():
+            while not stop_others.is_set():
+                display_watchdog.beat()
+                display_watchdog.note_frame()
+                time.sleep(0.01)
+        other = threading.Thread(target=busy_other_thread, daemon=True)
+        other.start()
+        time.sleep(0.8)  # well past the 0.3s limit
+        with lock:
+            after = len(pings(sent))
+        stop_others.set()
+        release.set()
+        other.join(5)
+        runner.join(10)
+        assert after == before, 'something other than the render thread fed the watchdog'
+        assert not runner.is_alive()
+        assert sent[-1] == 'STOPPING=1'
+
+
+# -- the unit and the entry point ----------------------------------------------
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def _service_directives():
+    with open(os.path.join(ROOT, 'systemd', 'ledmatrix.service'), encoding='utf-8') as f:
+        lines = [line.strip() for line in f]
+    return dict(line.split('=', 1) for line in lines
+                if line and not line.startswith(('#', '[')) and '=' in line
+                and not line.startswith('Environment='))
+
+
+def _seconds(value):
+    value = value.strip()
+    for suffix, factor in (('min', 60), ('s', 1)):
+        if value.endswith(suffix):
+            return float(value[:-len(suffix)]) * factor
+    return float(value)
+
+
+class TestUnit:
+    def test_the_display_unit_has_a_watchdog_the_process_can_feed(self):
+        d = _service_directives()
+        # Type=notify would block "systemctl start/restart" until READY=1 --
+        # after plugins load -- and the web UI and the update check call those
+        # with short timeouts.
+        assert d['Type'] == 'simple'
+        assert d['NotifyAccess'] == 'main'
+        assert 'WatchdogSec' in d
+
+    def test_the_watchdog_outlasts_the_loops_longest_healthy_gap(self):
+        from src.plugin_system.plugin_executor import PluginExecutor
+        limit = _seconds(_service_directives()['WatchdogSec'])
+        executor_timeout = PluginExecutor().default_timeout
+        assert limit >= 3 * executor_timeout, (
+            "a screen's first display() may legitimately take the executor's "
+            f"{executor_timeout}s timeout; WatchdogSec={limit:.0f}s leaves too little margin")
+        assert limit < display_watchdog.STARTUP_ALLOWANCE_SECONDS
+        assert limit < display_watchdog.PLUGIN_LOAD_ALLOWANCE_SECONDS
+        assert display_watchdog.BEAT_INTERVAL_SECONDS * 4 <= limit
+        assert limit > display_watchdog.HEARTBEAT_STALE_SECONDS >= 2 * executor_timeout
+
+    def test_the_heartbeat_directory_is_created_readable_by_the_web_user(self):
+        d = _service_directives()
+        assert d['RuntimeDirectory'] == 'ledmatrix'
+        assert display_watchdog.HEARTBEAT_DIR == '/run/' + d['RuntimeDirectory']
+        assert d['RuntimeDirectoryMode'] == '0755'
+
+    def test_a_crash_loop_backs_off_instead_of_stopping_for_good(self):
+        """A tripped start limit leaves the panel dark and refuses the web UI's
+        Start button and the update rollback's restart."""
+        d = _service_directives()
+        assert d['Restart'] == 'always'
+        assert 'StartLimitBurst' not in d
+        assert int(d['RestartSteps']) > 0
+        assert _seconds(d['RestartMaxDelaySec']) > _seconds(d['RestartSec'])
+
+    def test_run_py_widens_the_watchdog_before_importing_anything_heavy(self):
+        with open(os.path.join(ROOT, 'run.py'), encoding='utf-8') as f:
+            text = f.read()
+        call = text.index('display_watchdog.watchdog.begin_startup()')
+        assert call < text.index('from src.logging_config')
+        assert call < text.index('from src.display_controller')
+
+    def test_the_module_imports_nothing_else_from_src(self):
+        """run.py loads it first thing; importing it must stay cheap."""
+        with open(os.path.join(ROOT, 'src', 'display_watchdog.py'), encoding='utf-8') as f:
+            imports = [line for line in f if line.startswith(('import ', 'from '))]
+        assert not [line for line in imports if 'src' in line], imports

test/test_web_auth.py

@@ -503,6 +503,42 @@ class TestExemptions:
         assert set(r.get_json()['data']) == {'status'}
         assert 'checks' in admin.get('/api/v3/health').get_json()['data']
 
+    def test_a_stalled_render_loop_reaches_the_minimal_answer(
+            self, config_manager, api_v3_module, tmp_path, monkeypatch):
+        """The display's heartbeat (checks.display_loop) feeds the one word a
+        caller who is not logged in gets, without its detail."""
+        import time
+        from src import display_watchdog
+        from web_interface import display_preview
+        from web_interface.blueprints.api_v3 import misc
+        # Every other check healthy, so the heartbeat alone decides.
+        monkeypatch.setattr(misc, '_get_display_service_status', lambda: {'active': True})
+        monkeypatch.setattr(misc, '_discovered_plugin_manifests', lambda: {})
+        monkeypatch.setattr(api_v3_module.api_v3, 'plugin_catalog', object(), raising=False)
+        preview = tmp_path / 'preview.png'
+        preview.write_bytes(b'png')
+        monkeypatch.setattr(display_preview, 'SNAPSHOT_PATH', str(preview))
+        heartbeat = tmp_path / 'display-heartbeat.json'
+        monkeypatch.setattr(display_watchdog, 'HEARTBEAT_PATH', str(heartbeat))
+
+        def beat(age):
+            heartbeat.write_text(json.dumps({'pid': 1, 'mono': time.monotonic() - age,
+                                             'wall': time.time() - age}))
+
+        app = build(config_manager, api_v3_module)
+        admin = enable(app)
+        stranger = lan_client(app)
+
+        beat(age=2)
+        assert admin.get('/api/v3/health').get_json()['data']['checks']['display_loop']['status'] == 'running'
+        assert stranger.get('/api/v3/health').get_json()['data'] == {'status': 'healthy'}
+
+        beat(age=300)
+        full = admin.get('/api/v3/health').get_json()['data']
+        assert full['checks']['display_loop']['status'] == 'stalled'
+        assert full['status'] == 'degraded'
+        assert stranger.get('/api/v3/health').get_json()['data'] == {'status': 'degraded'}
+
 
 # --- The hash never leaves ------------------------------------------------------