rick/LEDMatrix
1
0
Fork 0
mirror of https://github.com/ChuckBuilds/LEDMatrix.git synced 2026-10-06 15:25:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(plugins): a busy-lock update skip is report-only, never a breaker failure

Browse Source 
When the update worker gives up waiting for a plugin's lock
(PLUGIN_LOCK_TIMEOUT, 5s) the skip was recorded as a hang, so three in a
row opened the circuit breaker. Vegas prefetch holds a plugin's lock for
its whole content render, which on a slow Pi can outlast 5s, so a healthy
plugin could be pulled from rotation.

The skip is now report-only: still logged (rate-limited) and still left as
PluginBusyError state error info with last-update stamped, but in health it
is counted as a busy skip (busy_skip_count / last_busy_skip, via the new
PluginHealthTracker.record_busy_skip) and never touches the failure streak,
last_error or the breaker. Only real hangs -- display() or update() running
past the executor timeout -- still count toward the breaker.

Busy-skip persistence shares the slow-call throttle (first one saved at
once so the web process sees it, then at most once a minute per plugin).
Tests prove repeated busy skips never open the breaker and repeated real
hangs still do, with busy skips interleaved.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chuck
2026-09-29 17:57:42 -04:00
co-authored by Claude Opus 5.5
parent 931557bc2e
commit 089a177c4b
5 changed files with 193 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plugin_system/plugin_health.py
+51 -11
View File
@@ -256,7 +256,7 @@ class PluginHealthTracker:
def record_hang(self, plugin_id: str, operation: str, seconds: float,
error: Optional[Exception] = None) -> None:
"""Record a call that ran past its limit, or held the plugin's lock past it.
"""Record a display() or update() call that ran past its limit.
Counts as a failure, so the ordinary circuit breaker handles a plugin
that keeps hanging: after ``failure_threshold`` in a row it is skipped
@@ -264,12 +264,14 @@ class PluginHealthTracker:
cooldown ends. The hang itself is kept alongside (``hang_count``,
``last_hang``) so the health API can tell "hung" from "raised".
Not for an update skipped because the plugin's lock stayed held: the
holder may be a healthy but long render (Vegas prefetch). That is
:meth:`record_busy_skip`, which never touches the breaker.
Args:
plugin_id: Plugin identifier
operation: What hung: ``"display"``, ``"update"``, or the lock
wait that found one of them still running.
seconds: How long it had been running, or how long the lock
was waited on, when this was recorded.
operation: What hung: ``"display"`` or ``"update"``.
seconds: How long it had been running when this was recorded.
error: The error to store as ``last_error``; one is built from
the other arguments when omitted.
"""
@@ -287,9 +289,9 @@ class PluginHealthTracker:
# record_failure saves the record, hang fields included.
self.record_failure(plugin_id, error)
#: Minimum seconds between persisting a plugin's slow-call counters. The
#: in-memory record is updated on every slow call; a plugin that is slow
#: on every frame must not become an SD-card write per frame.
#: Minimum seconds between persisting a plugin's slow-call or busy-skip
#: counters. The in-memory record is updated every time; a plugin that is
#: slow on every frame must not become an SD-card write per frame.
SLOW_CALL_PERSIST_INTERVAL = 60.0
def record_slow_call(self, plugin_id: str, operation: str, seconds: float) -> None:
@@ -308,10 +310,46 @@ class PluginHealthTracker:
'seconds': round(float(seconds), 3),
'time': now,
}
saved_at = self.__dict__.setdefault('_slow_call_saved_at', {})
last = saved_at.get(plugin_id)
self._save_reporting_throttled('slow', plugin_id, state, now)
def record_busy_skip(self, plugin_id: str, operation: str, seconds: float) -> None:
"""Note a call skipped because the plugin's lock stayed held. Reporting only.
The update worker gives up on a plugin's lock after
``PluginManager.PLUGIN_LOCK_TIMEOUT``. Whatever held it may be healthy
-- Vegas prefetch holds the lock for a plugin's whole content render,
which on a slow Pi can take longer than that -- so like
:meth:`record_slow_call` this never touches the circuit breaker, the
failure streak or ``last_error``. A real hang is recorded by
:meth:`record_hang` where it is measured.
Args:
plugin_id: Plugin identifier
operation: What was skipped, e.g. ``"update lock wait"``.
seconds: How long the lock was waited on.
"""
state = self.get_health_state(plugin_id)
count = state.get('busy_skip_count')
state['busy_skip_count'] = (count if isinstance(count, int) and not isinstance(count, bool)
else 0) + 1
now = time.time()
state['last_busy_skip'] = {
'operation': operation,
'seconds': round(float(seconds), 3),
'time': now,
}
self._save_reporting_throttled('busy', plugin_id, state, now)
def _save_reporting_throttled(self, kind: str, plugin_id: str,
state: Dict[str, Any], now: float) -> None:
"""Persist a reporting-only change at most once per
SLOW_CALL_PERSIST_INTERVAL per plugin and ``kind``. The first one is
saved at once, so the web process (which reads the persisted record)
sees it; repeats in between stay in memory until the next save."""
saved_at = self.__dict__.setdefault('_reporting_saved_at', {})
last = saved_at.get((kind, plugin_id))
if last is None or now - last >= self.SLOW_CALL_PERSIST_INTERVAL:
saved_at[plugin_id] = now
saved_at[(kind, plugin_id)] = now
self._save_health_state(plugin_id, state)
def set_degraded(self, plugin_id: str, reason: Optional[str]) -> None:
@@ -410,6 +448,8 @@ class PluginHealthTracker:
'last_hang': state.get('last_hang'),
'slow_call_count': state.get('slow_call_count', 0),
'last_slow_call': state.get('last_slow_call'),
'busy_skip_count': state.get('busy_skip_count', 0),
'last_busy_skip': state.get('last_busy_skip'),
}
def get_all_health_summaries(self) -> Dict[str, Dict[str, Any]]: