fix(status): runtime status agrees with the heartbeat; current-status republishes on wake (#726)

The runtime status snapshot now agrees with the display heartbeat, and current-status is republished when the display wakes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

test/test_display_controller_shutdown_and_state.py

@@ -3,6 +3,8 @@
 - systemd stops the service with SIGTERM; it must run the same cleanup as Ctrl-C.
 - The current mode is republished while it stays on screen, so the web UI's
   "Now showing" does not turn into "unknown" after its 120 s max_age.
+- Waking from scheduled-off, blanking, or on-demand starting or ending with
+  the mode name unchanged is republished at once, not up to 30 s later.
 - Switching Vegas on in the web UI works when Vegas was off at startup.
 """
 
@@ -78,6 +80,67 @@ def test_unchanged_mode_is_republished_after_the_refresh_interval():
     assert dc.cache_manager.set.call_count == 2
 
 
+def _published(dc):
+    return dc.cache_manager.set.call_args[0][1]
+
+
+@pytest.mark.parametrize("attr,before,after", [
+    ("is_display_active", False, True),    # woke from scheduled-off
+    ("is_display_active", True, False),    # blanked for scheduled-off
+    ("on_demand_active", False, True),     # on-demand started on this mode
+    ("on_demand_active", True, False),     # ...and ended on it
+])
+def test_a_flag_change_with_the_same_mode_is_republished_at_once(attr, before, after):
+    """ledpi: current-status showed is_display_active stale for up to 30 s
+    after a wake from scheduled-off, because the mode name had not changed."""
+    dc = _publisher()
+    setattr(dc, attr, before)
+    now = [1000.0]
+    with patch.object(dc_module.time, "monotonic", lambda: now[0]):
+        dc._publish_current_mode_state()            # e.g. the blank's publish
+        assert _published(dc)[attr] is before
+        now[0] += 1                                 # well inside the refresh
+        setattr(dc, attr, after)
+        dc._publish_current_mode_state_if_changed()  # the next publish point
+    assert dc.cache_manager.set.call_count == 2
+    assert _published(dc)[attr] is after
+    assert _published(dc)["mode"] == "mlb_live"
+
+
+def test_wake_inside_the_blank_sleep_is_published_by_the_service_helper():
+    """The blank sleeps through _service_pending_changes; the schedule turning
+    the display back on there reaches the web UI on that same call."""
+    dc = _publisher()
+    dc.is_display_active = False
+    dc._last_pending_service = None
+    dc.PENDING_CHANGES_INTERVAL = 0.25
+    for name in ("_poll_on_demand_requests", "_check_on_demand_expiration",
+                 "_apply_brightness_target"):
+        setattr(dc, name, MagicMock())
+
+    def wake():
+        dc.is_display_active = True
+    dc._evaluate_schedule = MagicMock(side_effect=wake)
+    now = [5000.0]
+    with patch.object(dc_module.time, "monotonic", lambda: now[0]):
+        dc._publish_current_mode_state()            # published while blank
+        now[0] += 2
+        dc._service_pending_changes()
+    assert dc.cache_manager.set.call_count == 2
+    assert _published(dc)["is_display_active"] is True
+
+
+def test_unchanged_flags_do_not_add_writes():
+    dc = _publisher()
+    now = [1000.0]
+    with patch.object(dc_module.time, "monotonic", lambda: now[0]):
+        dc._publish_current_mode_state_if_changed()
+        for _ in range(5):
+            now[0] += 1
+            dc._publish_current_mode_state_if_changed()
+    assert dc.cache_manager.set.call_count == 1
+
+
 def test_refresh_interval_is_well_inside_the_web_max_age():
     # api_v3/display.py reads display_current_state with max_age=120.
     assert dc_module.CURRENT_STATE_REFRESH_SECONDS < 120 / 2

test/test_plugin_runtime_snapshot.py

@@ -11,7 +11,9 @@ The cross-process tests use two CacheManagers over one temporary directory,
 the arrangement of the real services (which share /var/cache/ledmatrix).
 """
 import json
+import os
 import sys
+import time
 from pathlib import Path
 from unittest.mock import MagicMock
 
@@ -19,6 +21,7 @@ import pytest
 
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
+from src import display_watchdog  # noqa: E402
 from src.cache_manager import CacheManager  # noqa: E402
 from src.plugin_system import plugin_runtime as rt  # noqa: E402
 from src.plugin_system.plugin_manager import PluginManager  # noqa: E402
@@ -305,6 +308,74 @@ class TestReader:
         eager = _snapshot(self.NOW - 20, stale_after=0)
         assert view_from_snapshot(eager, now=self.NOW).status == "live"
 
+    # -- render-loop liveness (the heartbeat /api/v3/health reads) --
+
+    MONO = 50_000.0
+
+    def _beat(self, age, pid=4242):
+        return {"pid": pid, "mono": self.MONO - age, "wall": self.NOW - age}
+
+    def test_a_stale_heartbeat_with_a_live_snapshot_is_stalled(self):
+        """The publisher thread keeps writing while the render loop is
+        hung; the heartbeat says so, and the runtime status must agree with
+        /api/v3/health's display_loop: stalled."""
+        snapshot = _snapshot(self.NOW - 5, plugins={"clock": self.CLOCK}, pid=4242)
+        view = view_from_snapshot(
+            snapshot, now=self.NOW, now_mono=self.MONO,
+            heartbeat=self._beat(display_watchdog.HEARTBEAT_STALE_SECONDS + 1))
+        assert view.status == "stalled"
+        assert view.plugin("clock")["loaded"] is None  # no frozen truth passed on
+        described = view.describe()
+        assert described["status"] == "stalled"
+        assert described["heartbeat_age_seconds"] == display_watchdog.HEARTBEAT_STALE_SECONDS + 1
+
+    def test_the_threshold_is_the_health_checks(self):
+        snapshot = _snapshot(self.NOW - 5, pid=4242)
+        limit = display_watchdog.HEARTBEAT_STALE_SECONDS
+        fresh = view_from_snapshot(snapshot, now=self.NOW, now_mono=self.MONO,
+                                   heartbeat=self._beat(limit - 0.5))
+        assert fresh.status == "live"
+        assert fresh.describe()["heartbeat_age_seconds"] == limit - 0.5
+        assert view_from_snapshot(snapshot, now=self.NOW, now_mono=self.MONO,
+                                  heartbeat=self._beat(limit)).status == "stalled"
+
+    @pytest.mark.parametrize("heartbeat", [
+        None,                                   # dev server, Windows, starting up
+        {"pid": 9999, "mono": 0.0},             # another process: a restarted display
+        {"pid": "4242", "mono": 0.0},           # unparseable pid
+        {"pid": 4242},                          # no time in it
+    ])
+    def test_a_heartbeat_that_says_nothing_leaves_it_live(self, heartbeat):
+        snapshot = _snapshot(self.NOW - 5, plugins={"clock": self.CLOCK}, pid=4242)
+        view = view_from_snapshot(snapshot, now=self.NOW, now_mono=self.MONO,
+                                  heartbeat=heartbeat)
+        assert view.status == "live"
+        assert view.plugin("clock")["loaded"] is True
+
+    def test_a_stale_snapshot_stays_stale_whatever_the_heartbeat(self):
+        snapshot = _snapshot(self.NOW - 181, pid=4242)
+        assert view_from_snapshot(snapshot, now=self.NOW, now_mono=self.MONO,
+                                  heartbeat=self._beat(500)).status == "stale"
+
+    def test_a_snapshot_from_a_dead_process_is_stale_at_once(self):
+        """After a watchdog kill systemd removes the heartbeat's directory, so
+        nothing goes stale; the publisher's pid being gone is the signal."""
+        snapshot = _snapshot(self.NOW - 5, plugins={"clock": self.CLOCK}, pid=4242)
+        dead = view_from_snapshot(snapshot, now=self.NOW,
+                                  process_alive=lambda pid: False)
+        assert dead.status == "stale"
+        assert dead.plugin("clock")["loaded"] is None
+        for answer in (True, None):  # alive, or this platform cannot tell
+            assert view_from_snapshot(snapshot, now=self.NOW,
+                                      process_alive=lambda pid, a=answer: a).status == "live"
+
+    def test_process_exists(self):
+        if os.name != "posix":
+            assert rt.process_exists(os.getpid()) is None
+        else:
+            assert rt.process_exists(os.getpid()) is True
+        assert rt.process_exists(0) is None
+
     def test_no_cache_manager_or_a_failing_one_is_unknown(self):
         assert read_plugin_runtime(None).status == "unknown"
         cache = MagicMock()
@@ -424,6 +495,36 @@ class TestInstalledPluginsRoute:
         assert runtime["age_seconds"] > rt.STALE_AFTER
         assert plugins["clock"]["loaded"] is None
 
+    def _write_heartbeat(self, tmp_path, monkeypatch, age):
+        path = tmp_path / "display-heartbeat.json"
+        path.write_text(json.dumps({"pid": os.getpid(), "mono": time.monotonic() - age,
+                                    "wall": time.time() - age}), encoding="utf-8")
+        monkeypatch.setattr(display_watchdog, "HEARTBEAT_PATH", str(path))
+
+    def test_hung_render_loop_is_stalled_not_live(self, web_listing, tmp_path, monkeypatch):
+        """The publisher's own thread still ticks while the render loop is
+        stuck; the heartbeat it shares a process with has gone stale."""
+        display_cache, get = web_listing
+        self._display(display_cache).tick()   # a fresh snapshot, this pid
+        self._write_heartbeat(tmp_path, monkeypatch,
+                              display_watchdog.HEARTBEAT_STALE_SECONDS + 30)
+
+        plugins, runtime = get()
+
+        assert runtime["status"] == "stalled"
+        assert runtime["heartbeat_age_seconds"] >= display_watchdog.HEARTBEAT_STALE_SECONDS
+        assert plugins["clock"]["loaded"] is None and plugins["clock"]["state"] is None
+
+    def test_fresh_heartbeat_keeps_it_live(self, web_listing, tmp_path, monkeypatch):
+        display_cache, get = web_listing
+        self._display(display_cache).tick()
+        self._write_heartbeat(tmp_path, monkeypatch, 1)
+
+        plugins, runtime = get()
+
+        assert runtime["status"] == "live"
+        assert plugins["clock"]["loaded"] is True
+
 
 class TestDisplayControllerStopsThePublisher:
     def test_cleanup_publishes_stopped(self, monkeypatch):