fix(plugins): one hung plugin no longer stops every plugin from updating (#677)

The update worker no longer blocks forever on a plugin whose display()
never returns. It waits at most PLUGIN_LOCK_TIMEOUT (5s) for a plugin's
lock, then skips that plugin's update (a report-only "busy skip" in
health) and keeps updating every other plugin. display() frames are timed
(slow calls logged and counted; calls past the executor timeout recorded as
hangs), a hung update() is recorded, and on_config_change() now runs under
the plugin lock or is deferred to the worker. The plugin-facing API is
unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chuck
2026-09-30 09:10:41 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent e3c85cece6
commit c0d97e4867
10 changed files with 1077 additions and 28 deletions
+23 -6
View File
@@ -19,8 +19,25 @@ class PluginTimeoutError(Exception):
"""Raised when a plugin operation times out."""
class PluginBusyError(PluginTimeoutError):
"""A plugin's lock stayed held past its bound.
Not raised; recorded. The lock is held by the plugin's own display(),
update(), on_config_change() or a Vegas content render -- slow, or hung
-- so the caller skipped the plugin rather than wait on it. Report-only:
it is kept as the plugin's state error info and counted as a busy skip in
health, never as a failure, so it cannot open the circuit breaker.
"""
class PluginExecutor:
"""Handles plugin execution with timeout and error isolation."""
#: A display() call at least this long is logged and counted as slow.
#: A frame is milliseconds; two seconds is a plugin doing I/O in display().
SLOW_DISPLAY_SECONDS = 2.0
#: An update() call at least this long is logged as slow.
SLOW_UPDATE_SECONDS = 5.0
def __init__(
self,
@@ -117,15 +134,15 @@ class PluginExecutor:
True if update succeeded, False otherwise
"""
try:
start_time = time.time()
start_time = time.monotonic()
self.execute_with_timeout(
lambda: plugin.update(),
timeout=timeout,
plugin_id=plugin_id
)
duration = time.time() - start_time
duration = time.monotonic() - start_time
if duration > 5.0: # Warn if update takes more than 5 seconds
if duration > self.SLOW_UPDATE_SECONDS:
self.logger.warning(
"Plugin %s update() took %.2fs (consider optimizing)",
plugin_id,
@@ -175,7 +192,7 @@ class PluginExecutor:
True if display succeeded, False otherwise
"""
try:
start_time = time.time()
start_time = time.monotonic()
# Does display() take a display_mode keyword? The caller usually
# knows and caches the answer, so prefer what it passed.
@@ -206,9 +223,9 @@ class PluginExecutor:
plugin_id=plugin_id
)
duration = time.time() - start_time
duration = time.monotonic() - start_time
if duration > 2.0: # Warn if display takes more than 2 seconds
if duration > self.SLOW_DISPLAY_SECONDS:
self.logger.warning(
"Plugin %s display() took %.2fs (consider optimizing)",
plugin_id,