rick/LEDMatrix
1
0
Fork 0
mirror of https://github.com/ChuckBuilds/LEDMatrix.git synced 2026-10-04 14:25:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ipc): control socket stage 3 - a state stream replaces polled cache keys (#735)

Browse Source 
Adds state.get / state.subscribe to the display's control socket (StateHub in src/ipc/server.py). The web interface holds one subscription per process (web_interface/display_state.py) and reads current-status, on-demand status, plugin runtime and /health's display_loop from it, falling back to the cache keys and heartbeat file. While the socket serves readers, display_current_state and plugin_runtime_snapshot are written less often (about 1.5 instead of 5 cache writes a minute for 15 s screens).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chuck
2026-10-03 12:56:47 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 6bf7c3fa51
commit 515248b34e
16 changed files with 2487 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plugin_system/plugin_runtime.py
+118 -10
View File
@@ -36,6 +36,13 @@ tmpfs. A missing heartbeat (dev server, emulator, Windows, a display still
starting up) or one from another process (a display restarted after a
watchdog kill) says nothing, and the snapshot is judged on its own.
The control socket. Where the display serves its state stream (stage 3,
docs/IPC_CONTROL_SOCKET.md), every tick also hands the snapshot to it, in
memory, and the web interface reads it there first
(``view_from_socket_state``, judged by the same rules). While the socket
serves those readers, the cache copy is their fallback and an unchanged
snapshot is rewritten every ``RELAXED_REFRESH_INTERVAL`` instead.
A dead publisher. systemd removes the heartbeat's directory when the
service stops, so after a watchdog kill there is no heartbeat to go stale.
The reader then asks whether the snapshot's ``pid`` still exists (POSIX
@@ -48,7 +55,7 @@ import math
import os
import threading
import time
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from typing import Any, Callable, Dict, Optional
from src import display_watchdog
@@ -75,6 +82,15 @@ TICK_INTERVAL = 5.0
#: A snapshot older than this is stale: three missed refreshes.
STALE_AFTER = 3 * REFRESH_INTERVAL
#: The refresh while the control socket serves the web interface's readers
#: (``StateHub.readers_active``). The cache copy is then only their fallback,
#: so an unchanged snapshot is rewritten half as often; the snapshot says so
#: in its own ``refresh_interval`` and ``stale_after``.
RELAXED_REFRESH_INTERVAL = 2 * REFRESH_INTERVAL
#: The control socket's section for this snapshot (``state.plugins``).
STATE_SECTION = "plugins"
#: Bounds on a published ``stale_after``, so a corrupt value can make a
#: reader neither trust a dead display for hours nor distrust a live one.
_STALE_AFTER_MIN = 30.0
@@ -140,7 +156,8 @@ def summarize_error(error_info: Optional[Dict[str, Any]]) -> Optional[Dict[str,
def build_runtime_snapshot(state_manager: Any, *, started_at: float,
now: Optional[float] = None,
running: bool = True) -> Dict[str, Any]:
running: bool = True,
refresh_interval: float = REFRESH_INTERVAL) -> Dict[str, Any]:
"""The snapshot for ``state_manager`` (a plugin_state.PluginStateManager).
A stopped snapshot (``running=False``) lists no plugins: nothing is
@@ -162,8 +179,8 @@ def build_runtime_snapshot(state_manager: Any, *, started_at: float,
"running": running,
"published_at": time.time() if now is None else now,
"started_at": started_at,
"refresh_interval": REFRESH_INTERVAL,
"stale_after": STALE_AFTER,
"refresh_interval": refresh_interval,
"stale_after": 3 * refresh_interval,
"pid": os.getpid(),
"plugins": plugins,
}
@@ -197,30 +214,87 @@ class PluginRuntimePublisher:
self._tick_lock = threading.Lock()
self._stop = threading.Event()
self._thread: Optional[threading.Thread] = None
# The control socket's state stream (src/ipc/server.StateHub), when
# the display serves one: every tick also hands it the snapshot, in
# memory, and the cache refresh relaxes while it has readers.
self._hub: Any = None
self._hub_change: Optional[int] = None
self._hub_snapshot: Optional[Dict[str, Any]] = None
self.relaxed_refresh_interval = RELAXED_REFRESH_INTERVAL
def _write(self, running: bool) -> None:
snapshot = build_runtime_snapshot(self.state_manager, started_at=self.started_at,
now=self._wall_clock(), running=running)
def attach_hub(self, hub: Any) -> None:
"""Also publish to the control socket's state hub, starting now."""
with self._tick_lock:
self._hub = hub
self._hub_change = None
self._hub_snapshot = None
try:
self._push_to_hub(self.state_manager.change_count)
except Exception as err: # never let reporting break the display
logger.debug("Could not publish the plugin runtime state: %s", err,
exc_info=True)
def _push_to_hub(self, change: int) -> None:
"""The snapshot to the state hub: rebuilt when the state machine
changed, otherwise the last one with a new ``published_at``, which
the hub does not count as a new version. In memory, every tick, so
the socket's copy is never more than a tick old."""
hub = self._hub
if hub is None:
return
now = self._wall_clock()
if self._hub_snapshot is None or change != self._hub_change:
snapshot = build_runtime_snapshot(self.state_manager, started_at=self.started_at,
now=now)
else:
snapshot = dict(self._hub_snapshot, published_at=now)
hub.publish(STATE_SECTION, snapshot, volatile=("published_at",))
self._hub_snapshot = snapshot
self._hub_change = change
def _cache_refresh_interval(self) -> float:
"""The cache refresh: relaxed while the socket serves the readers."""
hub = self._hub
try:
if hub is not None and hub.readers_active():
return self.relaxed_refresh_interval
except Exception: # pylint: disable=broad-except
# The normal interval is the safe answer: it only writes more.
logger.debug("State hub readers_active() failed; using the normal refresh", exc_info=True)
return self.refresh_interval
def _write(self, running: bool, refresh_interval: Optional[float] = None) -> None:
snapshot = build_runtime_snapshot(
self.state_manager, started_at=self.started_at, now=self._wall_clock(),
running=running,
refresh_interval=self.refresh_interval if refresh_interval is None
else refresh_interval)
self.cache_manager.set(PLUGIN_RUNTIME_KEY, snapshot)
def tick(self) -> bool:
"""Publish if something changed (throttled) or the refresh is due.
True if a snapshot was written."""
True if a snapshot was written to the cache."""
with self._tick_lock:
try:
change = self.state_manager.change_count
try:
self._push_to_hub(change)
except Exception as err: # the cache copy still goes out below
logger.debug("Could not publish the plugin runtime state: %s", err,
exc_info=True)
now = self._clock()
refresh = self._cache_refresh_interval()
since = None if self._last_attempt is None else now - self._last_attempt
if since is not None:
if change == self._published_change:
if since < self.refresh_interval:
if since < refresh:
return False
elif since < self.min_interval:
return False
# Stamp the attempt before writing: a cache that keeps failing
# is retried at the throttled rate, not on every tick.
self._last_attempt = now
self._write(running=True)
self._write(running=True, refresh_interval=refresh)
self._published_change = change
return True
except Exception as err: # never let reporting break the display
@@ -317,6 +391,9 @@ class PluginRuntimeView:
plugins: Dict[str, Dict[str, Any]] = field(default_factory=dict)
#: Age of the render loop's heartbeat, when it was taken into account.
heartbeat_age_seconds: Optional[float] = None
#: Where the snapshot came from: ``cache`` (the shared cache file and the
#: heartbeat file) or ``socket`` (the control socket's state stream).
source: str = "cache"
@property
def live(self) -> bool:
@@ -348,6 +425,7 @@ class PluginRuntimeView:
"stale_after": self.stale_after,
"heartbeat_age_seconds": (None if self.heartbeat_age_seconds is None
else round(self.heartbeat_age_seconds, 1)),
"source": self.source,
}
@@ -445,6 +523,36 @@ def view_from_snapshot(snapshot: Any, now: Optional[float] = None,
)
def view_from_socket_state(snapshot: Any, now: Optional[float] = None,
now_mono: Optional[float] = None) -> Optional[PluginRuntimeView]:
"""Judge the ``plugins`` section of a control-socket state snapshot by
the same rules as the cache copy; None when it has none (an older
display, or a snapshot too large to carry it), so the caller reads the
cache instead.
The display measured its render loop's heartbeat age when it answered
(``state.loop``); that is the heartbeat here, aged by the time since the
answer arrived. A live snapshot with a stalled loop is ``stalled``, and a
snapshot older than its ``stale_after`` (the publisher thread stopped)
is ``stale``, exactly as for the cache. The display answered, so its
process is alive: there is no pid check.
"""
from src.ipc.client import snapshot_loop_age # stdlib-only module
if not isinstance(snapshot, dict):
return None
state = snapshot.get("state")
plugins = state.get(STATE_SECTION) if isinstance(state, dict) else None
if not isinstance(plugins, dict):
return None
now_mono = time.monotonic() if now_mono is None else now_mono
beat_age = snapshot_loop_age(snapshot, now_mono=now_mono)
heartbeat = None
if beat_age is not None:
heartbeat = {"pid": plugins.get("pid"), "mono": now_mono - beat_age}
view = view_from_snapshot(plugins, now=now, heartbeat=heartbeat, now_mono=now_mono)
return replace(view, source="socket")
def read_plugin_runtime(cache_manager: Any, now: Optional[float] = None,
heartbeat_path: Optional[str] = None) -> PluginRuntimeView:
"""The display's latest snapshot, judged for staleness and against the