mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-08-01 16:58:06 +00:00
ChuckandClaude Fable 5
c32aaf02e5
plugin update() executed inline in the render loop — execute_update's internal thread.join(timeout=30) blocked it, so one slow plugin HTTP fetch froze scrolling for the whole fetch (up to 30s; DNS-retry storms made this a regular occurrence on flaky networks). Scheduling stays on the render thread and keeps every existing gate (enabled, circuit breaker, can_execute, interval); due updates are now enqueued to a single background worker (serialized — same one-at-a-time execution as before, no thundering herd). RUNNING is set at enqueue so can_execute blocks re-entry alongside the pending-set dedup. Per-plugin locks make the old implicit update/display no-overlap guarantee explicit: the worker holds the plugin's lock through its update; the display side try-locks and, when the plugin is mid-update, holds the last frame for that iteration — reported as success so a mid-update skip never advances the rotation. Unlike before, the guarantee now also holds across the post-timeout window (previously the lingering update thread overlapped display()). Deadlock-free by construction: the worker takes one lock; display never blocks. Timeout semantics unchanged (lingering daemon thread documented). Kill switch: plugin_system.synchronous_updates: true restores the inline path. 8 new concurrency tests (non-blocking scheduler, overlap assertion under a hammering display loop, lock release on failure/timeout paths, dedup, kill switch); 4-min devpi soak clean (updates completing, rotation advancing, no stuck RUNNING states). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FqzC1nzTWL4kaqgMaQZFam
255 lines
11 KiB
Python
255 lines
11 KiB
Python
import time
|
|
from unittest.mock import MagicMock, patch
|
|
from pathlib import Path
|
|
from src.plugin_system.plugin_manager import PluginManager
|
|
from src.plugin_system.plugin_state import PluginState
|
|
from src.plugin_system.resource_monitor import PluginResourceMonitor
|
|
|
|
class TestPluginManager:
|
|
"""Test PluginManager functionality."""
|
|
|
|
def test_init(self, mock_config_manager, mock_display_manager, mock_cache_manager):
|
|
"""Test PluginManager initialization."""
|
|
with patch('src.plugin_system.plugin_manager.ensure_directory_permissions'):
|
|
pm = PluginManager(
|
|
plugins_dir="plugins",
|
|
config_manager=mock_config_manager,
|
|
display_manager=mock_display_manager,
|
|
cache_manager=mock_cache_manager
|
|
)
|
|
assert pm.plugins_dir == Path("plugins")
|
|
assert pm.config_manager == mock_config_manager
|
|
assert pm.display_manager == mock_display_manager
|
|
assert pm.cache_manager == mock_cache_manager
|
|
assert pm.plugins == {}
|
|
|
|
def test_discover_plugins(self, test_plugin_manager):
|
|
"""Test plugin discovery."""
|
|
pm = test_plugin_manager
|
|
# Mock _scan_directory_for_plugins since we can't easily create real files in fixture
|
|
pm._scan_directory_for_plugins = MagicMock(return_value=["plugin1", "plugin2"])
|
|
|
|
# We need to call the real discover_plugins method, not the mock from the fixture
|
|
# But the fixture mocks the whole class instance.
|
|
# Let's create a real instance with mocked dependencies for this test
|
|
pass # Handled by separate test below
|
|
|
|
def test_load_plugin_success(self, mock_config_manager, mock_display_manager, mock_cache_manager):
|
|
"""Test successful plugin loading."""
|
|
with patch('src.plugin_system.plugin_manager.ensure_directory_permissions'), \
|
|
patch('src.plugin_system.plugin_manager.PluginManager._scan_directory_for_plugins'), \
|
|
patch('src.plugin_system.plugin_manager.PluginLoader') as MockLoader, \
|
|
patch('src.plugin_system.plugin_manager.SchemaManager'):
|
|
|
|
pm = PluginManager(
|
|
plugins_dir="plugins",
|
|
config_manager=mock_config_manager,
|
|
display_manager=mock_display_manager,
|
|
cache_manager=mock_cache_manager
|
|
)
|
|
|
|
# Setup mocks
|
|
pm.plugin_manifests = {"test_plugin": {"id": "test_plugin", "name": "Test Plugin"}}
|
|
|
|
mock_loader = MockLoader.return_value
|
|
mock_loader.find_plugin_directory.return_value = Path("plugins/test_plugin")
|
|
mock_loader.load_plugin.return_value = (MagicMock(), MagicMock())
|
|
|
|
# Test loading
|
|
result = pm.load_plugin("test_plugin")
|
|
|
|
assert result is True
|
|
assert "test_plugin" in pm.plugin_modules
|
|
# PluginManager sets state to ENABLED after successful load
|
|
assert pm.state_manager.get_state("test_plugin") == PluginState.ENABLED
|
|
|
|
def test_load_plugin_missing_manifest(self, mock_config_manager, mock_display_manager, mock_cache_manager):
|
|
"""Test loading plugin with missing manifest."""
|
|
with patch('src.plugin_system.plugin_manager.ensure_directory_permissions'):
|
|
pm = PluginManager(
|
|
plugins_dir="plugins",
|
|
config_manager=mock_config_manager,
|
|
display_manager=mock_display_manager,
|
|
cache_manager=mock_cache_manager
|
|
)
|
|
|
|
# No manifest in pm.plugin_manifests
|
|
result = pm.load_plugin("non_existent_plugin")
|
|
|
|
assert result is False
|
|
assert pm.state_manager.get_state("non_existent_plugin") == PluginState.ERROR
|
|
|
|
def test_run_scheduled_updates_calls_update_with_resource_monitor(
|
|
self, mock_config_manager, mock_display_manager, mock_cache_manager
|
|
):
|
|
"""Regression test: run_scheduled_updates() must actually call a
|
|
plugin's update() when self.resource_monitor is set (as it is in
|
|
every real deployment -- display_controller.py and web_interface/
|
|
app.py both assign a real PluginResourceMonitor after construction).
|
|
|
|
Previously, the resource_monitor branch wrapped the call in a
|
|
function stored as a *class* attribute on a dynamically-built type
|
|
(`type('obj', (object,), {'update': monitored_update})()`), which
|
|
the descriptor protocol turns into a bound method on access --
|
|
silently passing the synthetic instance as an implicit first
|
|
argument to monitored_update(), which takes none. Every plugin's
|
|
scheduled update failed with "monitored_update() takes 0 positional
|
|
arguments but 1 was given" and was silently swallowed into a
|
|
circuit-breaker retry loop that never succeeded, so plugin data
|
|
(scores, odds, etc.) never refreshed.
|
|
"""
|
|
with patch('src.plugin_system.plugin_manager.ensure_directory_permissions'):
|
|
pm = PluginManager(
|
|
plugins_dir="plugins",
|
|
config_manager=mock_config_manager,
|
|
display_manager=mock_display_manager,
|
|
cache_manager=mock_cache_manager
|
|
)
|
|
|
|
plugin_instance = MagicMock()
|
|
plugin_instance.enabled = True
|
|
plugin_instance.update = MagicMock()
|
|
|
|
pm.plugins["test_plugin"] = plugin_instance
|
|
pm.plugin_manifests["test_plugin"] = {"update_interval": 10}
|
|
pm.state_manager.set_state("test_plugin", PluginState.ENABLED)
|
|
# Plain MagicMock, not the mock_cache_manager fixture: this test
|
|
# is about run_scheduled_updates() actually invoking update()
|
|
# through the resource-monitor wrapper, not about
|
|
# PluginResourceMonitor's own cache-backed metrics persistence
|
|
# (which calls cache_manager.get(..., memory_ttl=...) --
|
|
# a kwarg the fixture's mock_get() doesn't accept).
|
|
pm.resource_monitor = PluginResourceMonitor(MagicMock())
|
|
|
|
pm.run_scheduled_updates(current_time=time.time())
|
|
|
|
# Updates now execute on the background worker (the scheduler
|
|
# returns immediately) — wait for completion before asserting.
|
|
deadline = time.time() + 5
|
|
while (plugin_instance.update.call_count == 0
|
|
and time.time() < deadline):
|
|
time.sleep(0.02)
|
|
pm.stop_update_worker()
|
|
|
|
plugin_instance.update.assert_called_once()
|
|
assert "test_plugin" in pm.plugin_last_update
|
|
assert pm.state_manager.get_state("test_plugin") == PluginState.ENABLED
|
|
|
|
|
|
class TestPluginLoader:
|
|
"""Test PluginLoader functionality."""
|
|
|
|
def test_dependency_check(self):
|
|
"""Test dependency checking logic."""
|
|
# Covered by test_plugin_loader.py's install_dependencies tests,
|
|
# which exercise requirements_has_real_deps/requirements_are_satisfied
|
|
# and the pip subprocess fallback.
|
|
|
|
|
|
class TestPluginExecutor:
|
|
"""Test PluginExecutor functionality."""
|
|
|
|
def test_execute_display_success(self):
|
|
"""Test successful display execution."""
|
|
from src.plugin_system.plugin_executor import PluginExecutor
|
|
executor = PluginExecutor()
|
|
|
|
mock_plugin = MagicMock()
|
|
mock_plugin.display.return_value = True
|
|
|
|
result = executor.execute_display(mock_plugin, "test_plugin")
|
|
|
|
assert result is True
|
|
mock_plugin.display.assert_called_once()
|
|
|
|
def test_execute_display_exception(self):
|
|
"""Test display execution with exception."""
|
|
from src.plugin_system.plugin_executor import PluginExecutor
|
|
executor = PluginExecutor()
|
|
|
|
mock_plugin = MagicMock()
|
|
mock_plugin.display.side_effect = Exception("Test error")
|
|
|
|
result = executor.execute_display(mock_plugin, "test_plugin")
|
|
|
|
assert result is False
|
|
|
|
def test_execute_update_timeout(self):
|
|
"""Test update execution timeout."""
|
|
# Using a very short timeout for testing
|
|
from src.plugin_system.plugin_executor import PluginExecutor
|
|
executor = PluginExecutor(default_timeout=0.01)
|
|
|
|
mock_plugin = MagicMock()
|
|
def slow_update():
|
|
time.sleep(0.05)
|
|
mock_plugin.update.side_effect = slow_update
|
|
|
|
result = executor.execute_update(mock_plugin, "test_plugin")
|
|
|
|
assert result is False
|
|
|
|
|
|
class TestPluginHealth:
|
|
"""Test plugin health monitoring."""
|
|
|
|
def test_circuit_breaker(self, mock_cache_manager):
|
|
"""Test circuit breaker activation."""
|
|
from src.plugin_system.plugin_health import PluginHealthTracker
|
|
tracker = PluginHealthTracker(cache_manager=mock_cache_manager, failure_threshold=3, cooldown_period=60)
|
|
|
|
plugin_id = "test_plugin"
|
|
|
|
# Initial state
|
|
assert tracker.should_skip_plugin(plugin_id) is False
|
|
|
|
# Failures
|
|
tracker.record_failure(plugin_id, Exception("Error 1"))
|
|
assert tracker.should_skip_plugin(plugin_id) is False
|
|
|
|
tracker.record_failure(plugin_id, Exception("Error 2"))
|
|
assert tracker.should_skip_plugin(plugin_id) is False
|
|
|
|
tracker.record_failure(plugin_id, Exception("Error 3"))
|
|
# Should trip now
|
|
assert tracker.should_skip_plugin(plugin_id) is True
|
|
|
|
# Recovery (simulate timeout - need to update health state correctly)
|
|
if plugin_id in tracker._health_state:
|
|
tracker._health_state[plugin_id]["last_failure"] = time.time() - 61
|
|
tracker._health_state[plugin_id]["circuit_state"] = "closed"
|
|
assert tracker.should_skip_plugin(plugin_id) is False
|
|
|
|
|
|
class TestBasePlugin:
|
|
"""Test BasePlugin functionality."""
|
|
|
|
def test_dynamic_duration_defaults(self, mock_display_manager, mock_cache_manager):
|
|
"""Test default dynamic duration behavior."""
|
|
from src.plugin_system.base_plugin import BasePlugin
|
|
|
|
# Concrete implementation for testing
|
|
class ConcretePlugin(BasePlugin):
|
|
def update(self): pass
|
|
def display(self, force_clear=False): pass
|
|
|
|
config = {"enabled": True}
|
|
plugin = ConcretePlugin("test", config, mock_display_manager, mock_cache_manager, None)
|
|
|
|
assert plugin.supports_dynamic_duration() is False
|
|
assert plugin.get_dynamic_duration_cap() is None
|
|
assert plugin.is_cycle_complete() is True
|
|
|
|
def test_live_priority_config(self, mock_display_manager, mock_cache_manager):
|
|
"""Test live priority configuration."""
|
|
from src.plugin_system.base_plugin import BasePlugin
|
|
|
|
class ConcretePlugin(BasePlugin):
|
|
def update(self): pass
|
|
def display(self, force_clear=False): pass
|
|
|
|
config = {"enabled": True, "live_priority": True}
|
|
plugin = ConcretePlugin("test", config, mock_display_manager, mock_cache_manager, None)
|
|
|
|
assert plugin.has_live_priority() is True
|