mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-10-06 15:25:08 +00:00
fix(plugins): one hung plugin no longer stops every plugin from updating
The single update worker took each plugin's lock with a blocking acquire(), and the render thread holds that lock while it runs the plugin's display(). A display() that never returned -- or a first frame still running on the executor's lingering thread after its 30s timeout -- parked the worker for good, and no plugin updated again. - The worker waits at most PLUGIN_LOCK_TIMEOUT (5s, the bound unload_plugin already uses), skips the busy plugin and records it through the normal update-failure path as a hang (PluginBusyError), so repeats open its circuit breaker. Log lines about it are rate-limited per plugin. - display() is timed on every frame (two monotonic reads). Calls of 2s or more are logged once a minute and counted in plugin health (slow_call_count, last_slow_call); calls past the executor timeout, and a first frame still running at it, are recorded as hangs (hang_count, last_hang) and no longer as successes. An update() still running after its timeout is recorded as a hang too. - on_config_change() runs under the plugin's lock via PluginManager.apply_config_change(); if the lock stays busy the latest change is deferred to the update worker, applied as soon as the lock frees and before the plugin's next update() at the latest. The plugin API is unchanged. Which thread runs each hook is documented in PluginManager.__init__. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -19,8 +19,23 @@ class PluginTimeoutError(Exception):
|
||||
"""Raised when a plugin operation times out."""
|
||||
|
||||
|
||||
class PluginBusyError(PluginTimeoutError):
|
||||
"""A plugin's lock stayed held past its bound.
|
||||
|
||||
Not raised; recorded. The lock is held by the plugin's own display(),
|
||||
update() or on_config_change() -- one that is hung or far slower than it
|
||||
should be -- so the caller skipped the plugin rather than wait on it.
|
||||
"""
|
||||
|
||||
|
||||
class PluginExecutor:
|
||||
"""Handles plugin execution with timeout and error isolation."""
|
||||
|
||||
#: A display() call at least this long is logged and counted as slow.
|
||||
#: A frame is milliseconds; two seconds is a plugin doing I/O in display().
|
||||
SLOW_DISPLAY_SECONDS = 2.0
|
||||
#: An update() call at least this long is logged as slow.
|
||||
SLOW_UPDATE_SECONDS = 5.0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -117,15 +132,15 @@ class PluginExecutor:
|
||||
True if update succeeded, False otherwise
|
||||
"""
|
||||
try:
|
||||
start_time = time.time()
|
||||
start_time = time.monotonic()
|
||||
self.execute_with_timeout(
|
||||
lambda: plugin.update(),
|
||||
timeout=timeout,
|
||||
plugin_id=plugin_id
|
||||
)
|
||||
duration = time.time() - start_time
|
||||
duration = time.monotonic() - start_time
|
||||
|
||||
if duration > 5.0: # Warn if update takes more than 5 seconds
|
||||
if duration > self.SLOW_UPDATE_SECONDS:
|
||||
self.logger.warning(
|
||||
"Plugin %s update() took %.2fs (consider optimizing)",
|
||||
plugin_id,
|
||||
@@ -175,7 +190,7 @@ class PluginExecutor:
|
||||
True if display succeeded, False otherwise
|
||||
"""
|
||||
try:
|
||||
start_time = time.time()
|
||||
start_time = time.monotonic()
|
||||
|
||||
# Does display() take a display_mode keyword? The caller usually
|
||||
# knows and caches the answer, so prefer what it passed.
|
||||
@@ -206,9 +221,9 @@ class PluginExecutor:
|
||||
plugin_id=plugin_id
|
||||
)
|
||||
|
||||
duration = time.time() - start_time
|
||||
duration = time.monotonic() - start_time
|
||||
|
||||
if duration > 2.0: # Warn if display takes more than 2 seconds
|
||||
if duration > self.SLOW_DISPLAY_SECONDS:
|
||||
self.logger.warning(
|
||||
"Plugin %s display() took %.2fs (consider optimizing)",
|
||||
plugin_id,
|
||||
|
||||
Reference in New Issue
Block a user