mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-08-01 16:58:06 +00:00
perf(plugins): run scheduled updates off the render thread
plugin update() executed inline in the render loop — execute_update's internal thread.join(timeout=30) blocked it, so one slow plugin HTTP fetch froze scrolling for the whole fetch (up to 30s; DNS-retry storms made this a regular occurrence on flaky networks). Scheduling stays on the render thread and keeps every existing gate (enabled, circuit breaker, can_execute, interval); due updates are now enqueued to a single background worker (serialized — same one-at-a-time execution as before, no thundering herd). RUNNING is set at enqueue so can_execute blocks re-entry alongside the pending-set dedup. Per-plugin locks make the old implicit update/display no-overlap guarantee explicit: the worker holds the plugin's lock through its update; the display side try-locks and, when the plugin is mid-update, holds the last frame for that iteration — reported as success so a mid-update skip never advances the rotation. Unlike before, the guarantee now also holds across the post-timeout window (previously the lingering update thread overlapped display()). Deadlock-free by construction: the worker takes one lock; display never blocks. Timeout semantics unchanged (lingering daemon thread documented). Kill switch: plugin_system.synchronous_updates: true restores the inline path. 8 new concurrency tests (non-blocking scheduler, overlap assertion under a hammering display loop, lock release on failure/timeout paths, dedup, kill switch); 4-min devpi soak clean (updates completing, rotation advancing, no stuck RUNNING states). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FqzC1nzTWL4kaqgMaQZFam
This commit is contained in:
Chuck
+68
-28
@@ -23,6 +23,7 @@ Entry point: :func:`main` — instantiates :class:`DisplayController` and calls
|
||||
import time
|
||||
import os
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional, Callable
|
||||
from datetime import datetime
|
||||
@@ -839,6 +840,30 @@ class DisplayController:
|
||||
except Exception: # pylint: disable=broad-except
|
||||
logger.exception("Error running scheduled plugin updates")
|
||||
|
||||
@contextmanager
|
||||
def _display_lock_or_skip(self, plugin_id):
|
||||
"""Try-lock guard keeping a plugin's display() off its in-flight update().
|
||||
|
||||
Yields True when display may run (lock held, released on exit) or
|
||||
when no lock support exists (older plugin manager). Yields False when
|
||||
the plugin's update() is currently executing on the background
|
||||
worker — the caller should treat the frame as displayed (the panel
|
||||
holds the last pushed frame) rather than as a plugin failure, so a
|
||||
mid-update skip never advances the rotation.
|
||||
"""
|
||||
pm = self.plugin_manager
|
||||
if not pm or not hasattr(pm, 'get_plugin_lock') or not plugin_id:
|
||||
yield True
|
||||
return
|
||||
lock = pm.get_plugin_lock(plugin_id)
|
||||
if not lock.acquire(blocking=False):
|
||||
yield False
|
||||
return
|
||||
try:
|
||||
yield True
|
||||
finally:
|
||||
lock.release()
|
||||
|
||||
_FOLLOWER_SEND_INTERVAL = 1.0 / 90 # raw bytes are cheap; 90fps > follower render rate
|
||||
|
||||
def _follower_rebuild_scroll_image(self) -> None:
|
||||
@@ -1836,25 +1861,30 @@ class DisplayController:
|
||||
)
|
||||
_accepts_display_mode = self._plugin_accepts_display_mode[_cache_key]
|
||||
|
||||
# Use PluginExecutor for safe execution with timeout
|
||||
if self.plugin_manager and hasattr(self.plugin_manager, 'plugin_executor'):
|
||||
result = self.plugin_manager.plugin_executor.execute_display(
|
||||
manager_to_display,
|
||||
plugin_id,
|
||||
force_clear=self.force_change,
|
||||
display_mode=active_mode if _accepts_display_mode else None
|
||||
)
|
||||
# execute_display returns bool, convert to expected format
|
||||
if result:
|
||||
result = True # Success
|
||||
with self._display_lock_or_skip(plugin_id) as can_display:
|
||||
if not can_display:
|
||||
# update() in flight on the worker — hold
|
||||
# the last frame; not a plugin failure
|
||||
result = True
|
||||
elif self.plugin_manager and hasattr(self.plugin_manager, 'plugin_executor'):
|
||||
# Use PluginExecutor for safe execution with timeout
|
||||
result = self.plugin_manager.plugin_executor.execute_display(
|
||||
manager_to_display,
|
||||
plugin_id,
|
||||
force_clear=self.force_change,
|
||||
display_mode=active_mode if _accepts_display_mode else None
|
||||
)
|
||||
# execute_display returns bool, convert to expected format
|
||||
if result:
|
||||
result = True # Success
|
||||
else:
|
||||
result = False # Failed
|
||||
else:
|
||||
result = False # Failed
|
||||
else:
|
||||
# Fallback to direct call if executor not available
|
||||
if _accepts_display_mode:
|
||||
result = manager_to_display.display(display_mode=active_mode, force_clear=self.force_change)
|
||||
else:
|
||||
result = manager_to_display.display(force_clear=self.force_change)
|
||||
# Fallback to direct call if executor not available
|
||||
if _accepts_display_mode:
|
||||
result = manager_to_display.display(display_mode=active_mode, force_clear=self.force_change)
|
||||
else:
|
||||
result = manager_to_display.display(force_clear=self.force_change)
|
||||
|
||||
logger.debug(f"display() returned: {result} (type: {type(result)})")
|
||||
# Check if display() returned a boolean (new behavior)
|
||||
@@ -2139,11 +2169,16 @@ class DisplayController:
|
||||
|
||||
while True:
|
||||
try:
|
||||
# Pass display_mode to maintain sticky manager state
|
||||
if _accepts_display_mode:
|
||||
result = manager_to_display.display(display_mode=active_mode, force_clear=False)
|
||||
else:
|
||||
result = manager_to_display.display(force_clear=False)
|
||||
with self._display_lock_or_skip(plugin_id) as can_display:
|
||||
if can_display:
|
||||
# Pass display_mode to maintain sticky manager state
|
||||
if _accepts_display_mode:
|
||||
result = manager_to_display.display(display_mode=active_mode, force_clear=False)
|
||||
else:
|
||||
result = manager_to_display.display(force_clear=False)
|
||||
else:
|
||||
# update() in flight — hold the last frame
|
||||
result = True
|
||||
if isinstance(result, bool) and not result:
|
||||
logger.debug("Display returned False, breaking early")
|
||||
break
|
||||
@@ -2203,11 +2238,16 @@ class DisplayController:
|
||||
break
|
||||
|
||||
try:
|
||||
# Pass display_mode to maintain sticky manager state
|
||||
if _accepts_display_mode:
|
||||
result = manager_to_display.display(display_mode=active_mode, force_clear=False)
|
||||
else:
|
||||
result = manager_to_display.display(force_clear=False)
|
||||
with self._display_lock_or_skip(plugin_id) as can_display:
|
||||
if can_display:
|
||||
# Pass display_mode to maintain sticky manager state
|
||||
if _accepts_display_mode:
|
||||
result = manager_to_display.display(display_mode=active_mode, force_clear=False)
|
||||
else:
|
||||
result = manager_to_display.display(force_clear=False)
|
||||
else:
|
||||
# update() in flight — hold the last frame
|
||||
result = True
|
||||
if isinstance(result, bool) and not result:
|
||||
# For dynamic duration plugins, don't exit on False - keep looping
|
||||
# until cycle is complete or max duration is reached
|
||||
|
||||
Reference in New Issue
Block a user