feat(display): systemd watchdog and heartbeat for a frozen render loop (#687)

If the render loop gets stuck inside a plugin's display(), ledmatrix.service
stays active and the panel stays frozen. This adds a way to detect that.

- src/display_watchdog.py (standard library only) sends sd_notify over
  $NOTIFY_SOCKET and writes /run/ledmatrix/display-heartbeat.json. Only the
  render thread counts: beats from other threads are ignored.
- ledmatrix.service: WatchdogSec=120, NotifyAccess=main,
  RuntimeDirectory=ledmatrix (0755), RestartSteps=4 and
  RestartMaxDelaySec=2min. It stays Type=simple. run.py widens the watchdog
  to 15 min for start-up, and load_plugin() does the same on the render
  thread. The loop arms after its first frame.
- /api/v3/health adds checks.display_loop: running, stalled (no heartbeat
  for over 60s, which makes the status degraded) or not_reported. With web
  login on, a caller who is not logged in still gets only healthy/degraded,
  and a stall degrades that answer.
- The update verifier requires a fresh heartbeat from the restarted display
  when the display it replaced was writing one. A frozen panel is rolled
  back.
- Existing installs get the systemd watchdog only after install_service.sh
  is re-run. The heartbeat works right away.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

test/test_api_v3_health.py

@@ -5,8 +5,10 @@ PluginManager does not have; a hasattr guard turned that into a permanent 0.
 Each check that fails answers "see logs for details", so it has to log.
 """
 
+import json
 import logging
 import sys
+import time
 from pathlib import Path
 from types import SimpleNamespace
 
@@ -25,6 +27,28 @@ def _no_systemctl(monkeypatch):
                         lambda: {"active": True})
 
 
+@pytest.fixture(autouse=True)
+def heartbeat(tmp_path, monkeypatch):
+    """The display's heartbeat file, somewhere private; absent until written."""
+    from src import display_watchdog
+    path = tmp_path / "display-heartbeat.json"
+    monkeypatch.setattr(display_watchdog, "HEARTBEAT_PATH", str(path))
+
+    def write(age):
+        path.write_text(json.dumps({"pid": 1, "mono": time.monotonic() - age,
+                                    "wall": time.time() - age}))
+    return write
+
+
+@pytest.fixture
+def fresh_preview(tmp_path, monkeypatch):
+    """A just-written preview frame, so only the heartbeat decides the verdict."""
+    from web_interface import display_preview
+    snapshot = tmp_path / "preview.png"
+    snapshot.write_bytes(b"png")
+    monkeypatch.setattr(display_preview, "SNAPSHOT_PATH", str(snapshot))
+
+
 def _checks(client):
     response = client.get(URL)
     assert response.status_code == 200, response.get_json()
@@ -88,3 +112,59 @@ def test_a_failed_hardware_check_is_logged(api_v3_client, api_v3_module, caplog,
     assert check["status"] == "unknown"
     logged = [r for r in caplog.records if "snapshot" in r.getMessage()]
     assert logged and logged[0].exc_info
+
+
+# -- the display's render-loop heartbeat -----------------------------------------
+#
+# The preview frame's age said nothing about a panel frozen by a render thread
+# stuck in a plugin; the heartbeat is written by that thread itself.
+
+def _health(client):
+    response = client.get(URL)
+    assert response.status_code == 200, response.get_json()
+    return response.get_json()["data"]
+
+
+def test_a_fresh_heartbeat_is_a_running_display_loop(api_v3_client, heartbeat, fresh_preview):
+    heartbeat(age=3)
+
+    data = _health(api_v3_client)
+
+    assert data["checks"]["display_loop"]["status"] == "running"
+    assert 2 <= data["checks"]["display_loop"]["heartbeat_age_seconds"] < 10
+    assert data["status"] == "healthy"
+
+
+def test_a_stale_heartbeat_is_a_stalled_display_loop(api_v3_client, heartbeat, fresh_preview):
+    """Service active, preview recent, and still the panel is frozen."""
+    heartbeat(age=300)
+
+    data = _health(api_v3_client)
+
+    assert data["checks"]["display_loop"]["status"] == "stalled"
+    assert data["checks"]["display_loop"]["heartbeat_age_seconds"] >= 299
+    assert data["status"] == "degraded"
+
+
+def test_no_heartbeat_falls_back_to_the_older_checks(api_v3_client, fresh_preview):
+    """The dev server, the emulator, Windows, or a display without the
+    feature: absence is not a failure, and the verdict is what it was."""
+    data = _health(api_v3_client)
+
+    assert data["checks"]["display_loop"]["status"] == "not_reported"
+    assert data["checks"]["hardware"]["status"] == "connected"
+    assert data["status"] == "healthy"
+
+
+def test_an_unreadable_heartbeat_is_reported_not_raised(api_v3_client, monkeypatch, caplog):
+    from src import display_watchdog
+
+    def boom(_path):
+        raise RuntimeError("bad heartbeat")
+    monkeypatch.setattr(display_watchdog, "read_heartbeat", boom)
+
+    with caplog.at_level(logging.WARNING):
+        check = _checks(api_v3_client)["display_loop"]
+
+    assert check["status"] == "unknown"
+    assert any("heartbeat" in r.getMessage() for r in caplog.records)