Merge remote-tracking branch 'origin/main' into claude/codebase-audit-cleanup-s28o2e

# Conflicts:
#	.github/workflows/test.yml

test/test_plugin_update_reservation.py

@@ -0,0 +1,335 @@
+"""Plugin update scheduling must be atomic (issue #401).
+
+`run_scheduled_updates()` decided whether to update a plugin with a
+check-then-act sequence: `can_execute()` and `set_state(RUNNING)` were separate
+calls with nothing between them, so two scheduler threads could both observe
+ENABLED and both go on to call the same plugin's `update()`.
+
+Two schedulers really do run at once. The render loop calls
+`_tick_plugin_updates()`, and Vegas mode fires its own `vegas-plugin-tick`
+daemon thread every few seconds; that thread is never joined when
+`VegasModeCoordinator.play()` returns, so a slow `update()` still in flight can
+overlap the next tick from the main loop.
+
+A plugin whose `update()` runs twice at once is unsafe unless it happens to be
+reentrant — shared mutable state, a non-thread-safe HTTP session or cache all
+break. These tests pin the reservation that closes it, and the state
+bookkeeping that has to survive it: a plugin reserved but never dispatched must
+not be stranded in RUNNING, because `can_execute()` would then refuse it
+forever.
+"""
+
+import os
+import sys
+import threading
+import time
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.plugin_system.plugin_manager import PluginManager  # noqa: E402
+from src.plugin_system.plugin_state import PluginState  # noqa: E402
+
+
+class OverlapDetectingPlugin:
+    """Records the high-water mark of concurrent update() calls."""
+
+    def __init__(self, update_seconds=0.25):
+        self.enabled = True
+        self.update_seconds = update_seconds
+        self.update_calls = 0
+        self.max_concurrent = 0
+        self._active = 0
+        self._guard = threading.Lock()
+
+    def update(self):
+        with self._guard:
+            self._active += 1
+            self.update_calls += 1
+            self.max_concurrent = max(self.max_concurrent, self._active)
+        try:
+            time.sleep(self.update_seconds)
+        finally:
+            with self._guard:
+                self._active -= 1
+        return True
+
+    def display(self, force_clear=False):
+        return True
+
+
+@pytest.fixture
+def pm(tmp_path):
+    manager = PluginManager(plugins_dir=str(tmp_path), config_manager=None,
+                            display_manager=None, cache_manager=None)
+    yield manager
+    manager.stop_update_worker()
+
+
+def _install(pm, plugin, plugin_id="racy-plugin", interval=0.01):
+    pm.plugins[plugin_id] = plugin
+    pm._update_interval_cache[plugin_id] = interval
+    pm.state_manager.set_state(plugin_id, PluginState.ENABLED)
+    return plugin_id
+
+
+def _widen_check_then_act_window(pm, delay=0.02):
+    """Hold every scheduler thread inside the eligibility check at once.
+
+    The real gap between can_execute() and the RUNNING transition is a couple
+    of bytecodes wide, so a plain thread race under the GIL almost never lands
+    in it — an unfixed scheduler looks correct in a test that just hammers it.
+    Delaying the check reproduces the interleaving that Vegas's tick thread and
+    the render loop actually produce when a slow update() overlaps the next
+    tick, and it is what makes these tests fail without the reservation.
+
+    Once the check and the transition are under one lock, the delay only
+    serializes the schedulers: the losers observe RUNNING and back off.
+    """
+    real_can_execute = pm.state_manager.can_execute
+
+    def slow_can_execute(plugin_id):
+        result = real_can_execute(plugin_id)
+        time.sleep(delay)
+        return result
+
+    pm.state_manager.can_execute = slow_can_execute
+
+
+def _hammer(target, threads=8, rounds=1):
+    """Run `target` on N threads released simultaneously by a barrier."""
+    errors = []
+    barrier = threading.Barrier(threads)
+
+    def runner():
+        try:
+            barrier.wait(timeout=5)
+            for _ in range(rounds):
+                target()
+        except Exception as exc:  # noqa: BLE001 - surfaced by the assertion
+            errors.append(exc)
+
+    workers = [threading.Thread(target=runner) for _ in range(threads)]
+    for worker in workers:
+        worker.start()
+    for worker in workers:
+        worker.join(timeout=30)
+    assert not errors, f"worker raised: {errors[0]!r}"
+
+
+class TestReservationAtomicity:
+    """The check-and-claim itself, independent of any dispatch path."""
+
+    def test_only_one_caller_wins_the_reservation(self, pm):
+        plugin_id = _install(pm, OverlapDetectingPlugin())
+        _widen_check_then_act_window(pm)
+        wins = []
+        lock = threading.Lock()
+
+        def claim():
+            if pm._reserve_for_update(plugin_id):
+                with lock:
+                    wins.append(threading.current_thread().name)
+
+        _hammer(claim, threads=16)
+        assert len(wins) == 1, (
+            f"{len(wins)} threads reserved the same plugin concurrently; "
+            "the eligibility check and the RUNNING transition are not atomic")
+        assert pm.state_manager.get_state(plugin_id) == PluginState.RUNNING
+
+    def test_reservation_refused_while_running(self, pm):
+        plugin_id = _install(pm, OverlapDetectingPlugin())
+        assert pm._reserve_for_update(plugin_id) is True
+        assert pm._reserve_for_update(plugin_id) is False
+
+    def test_reservation_can_be_handed_back(self, pm):
+        plugin_id = _install(pm, OverlapDetectingPlugin())
+        assert pm._reserve_for_update(plugin_id) is True
+        pm._release_reservation(plugin_id)
+        assert pm.state_manager.get_state(plugin_id) == PluginState.ENABLED
+        assert pm._reserve_for_update(plugin_id) is True, \
+            "a released reservation must be claimable again"
+
+    def test_due_check_is_inside_the_reservation(self, pm):
+        """Two threads that both decided 'due' must not both get a turn.
+
+        With the due check outside the lock, the loser of the race could claim
+        the plugin the moment the winner finished, running update() twice
+        inside one interval.
+        """
+        plugin_id = _install(pm, OverlapDetectingPlugin(), interval=60.0)
+        now = time.time()
+        assert pm._reserve_for_update(plugin_id, now, 60.0) is True
+        pm.plugin_last_update[plugin_id] = now
+        pm._release_reservation(plugin_id)
+        assert pm._reserve_for_update(plugin_id, now, 60.0) is False, \
+            "plugin updated just now must not be due again"
+
+
+class TestNoConcurrentUpdate:
+    """The end-to-end invariant the issue is actually about."""
+
+    def test_synchronous_path_never_overlaps(self, pm):
+        """The kill-switch path ran update() inline with no dedup at all."""
+        pm._synchronous_updates = True
+        plugin = OverlapDetectingPlugin(update_seconds=0.25)
+        _install(pm, plugin)
+        _widen_check_then_act_window(pm)
+
+        _hammer(pm.run_scheduled_updates, threads=8)
+
+        assert plugin.max_concurrent == 1, (
+            f"update() ran {plugin.max_concurrent}x concurrently on the "
+            "synchronous path")
+
+    def test_update_all_plugins_never_overlaps(self, pm):
+        plugin = OverlapDetectingPlugin(update_seconds=0.25)
+        _install(pm, plugin)
+        _widen_check_then_act_window(pm)
+
+        _hammer(pm.update_all_plugins, threads=8)
+
+        assert plugin.max_concurrent == 1, (
+            f"update() ran {plugin.max_concurrent}x concurrently via "
+            "update_all_plugins()")
+
+    def test_async_path_never_overlaps(self, pm):
+        plugin = OverlapDetectingPlugin(update_seconds=0.2)
+        plugin_id = _install(pm, plugin)
+
+        _hammer(pm.run_scheduled_updates, threads=8, rounds=3)
+
+        # Wait for an update to have both started and finished. Polling only
+        # `_active` races the worker: before it picks the item up nothing is
+        # active yet, so the loop would fall straight through and assert on a
+        # plugin that never ran.
+        deadline = time.monotonic() + 15
+        while time.monotonic() < deadline:
+            if plugin.update_calls >= 1 and plugin._active == 0:
+                break
+            time.sleep(0.05)
+
+        assert plugin.update_calls >= 1, "no update ran on the async path"
+        assert plugin.max_concurrent == 1, (
+            f"update() ran {plugin.max_concurrent}x concurrently on the "
+            "async path")
+        assert plugin_id in pm.plugins
+
+
+class TestNoStrandedState:
+    """A reservation that is never dispatched must not wedge the plugin."""
+
+    def test_plugin_returns_to_enabled_after_async_updates(self, pm):
+        plugin = OverlapDetectingPlugin(update_seconds=0.1)
+        plugin_id = _install(pm, plugin)
+
+        _hammer(pm.run_scheduled_updates, threads=6, rounds=2)
+
+        deadline = time.monotonic() + 15
+        while time.monotonic() < deadline:
+            if (plugin.update_calls >= 1
+                    and pm.state_manager.get_state(plugin_id) == PluginState.ENABLED
+                    and not pm._pending_updates):
+                break
+            time.sleep(0.05)
+
+        # ENABLED is also the starting state, so without this the assertion
+        # below would pass on a plugin that never got scheduled at all.
+        assert plugin.update_calls >= 1, "no update ran; the state assertion would be vacuous"
+        assert pm.state_manager.get_state(plugin_id) == PluginState.ENABLED, \
+            "plugin stranded in RUNNING; can_execute() would refuse it forever"
+        assert not pm._pending_updates, "pending set not drained"
+
+    def test_pending_cleared_before_state_reset(self, pm):
+        """Invariant: never-RUNNING implies never-pending.
+
+        _finish() used to clear the pending entry *after* flipping the state
+        back to ENABLED. In that window a scheduler could reserve the plugin
+        and then have its enqueue dropped by the pending-dedup.
+        """
+        plugin = OverlapDetectingPlugin(update_seconds=0.05)
+        plugin_id = _install(pm, plugin)
+
+        violations = []
+        stop = threading.Event()
+
+        def watcher():
+            while not stop.is_set():
+                state = pm.state_manager.get_state(plugin_id)
+                if state != PluginState.RUNNING and plugin_id in pm._pending_updates:
+                    violations.append(state)
+                time.sleep(0.001)
+
+        thread = threading.Thread(target=watcher, daemon=True)
+        thread.start()
+        try:
+            for _ in range(15):
+                pm.run_scheduled_updates()
+                time.sleep(0.05)
+        finally:
+            stop.set()
+            thread.join(timeout=5)
+
+        assert not violations, (
+            f"{len(violations)} sample(s) saw a non-RUNNING plugin still in "
+            "_pending_updates")
+
+
+class TestDispatchFailure:
+    """A reservation must survive the dispatch itself failing.
+
+    _enqueue_update() claims the plugin, adds it to the pending set, and only
+    then starts the worker and queues the item. threading.Thread.start() raises
+    RuntimeError when the OS refuses a new thread — not hypothetical on a Pi
+    under memory or thread pressure. Nothing is queued to release the plugin at
+    that point, so without an explicit rollback it stays RUNNING with a stale
+    pending entry and can_execute() refuses it for the rest of the process.
+    """
+
+    def test_reservation_released_when_the_worker_cannot_start(self, pm):
+        plugin_id = _install(pm, OverlapDetectingPlugin())
+
+        def refuse_to_start():
+            raise RuntimeError("can't start new thread")
+
+        pm._ensure_update_worker = refuse_to_start
+
+        assert pm._reserve_for_update(plugin_id) is True
+        pm._enqueue_update(plugin_id, time.time())
+
+        assert pm.state_manager.get_state(plugin_id) == PluginState.ENABLED, \
+            "plugin left in RUNNING after a failed dispatch; can_execute() " \
+            "would refuse it forever"
+        assert plugin_id not in pm._pending_updates, \
+            "stale pending entry would make the next enqueue hit the dedup"
+        assert pm._reserve_for_update(plugin_id) is True, \
+            "plugin must be claimable again on the next tick"
+
+    def test_dispatch_failure_does_not_abort_the_rest_of_the_tick(self, pm):
+        """One plugin failing to queue must not skip the others in that tick."""
+        first = OverlapDetectingPlugin()
+        second = OverlapDetectingPlugin()
+        _install(pm, first, plugin_id="plugin-a")
+        _install(pm, second, plugin_id="plugin-b")
+
+        calls = []
+        real_ensure = pm._ensure_update_worker
+
+        def fail_first_only():
+            calls.append(1)
+            if len(calls) == 1:
+                raise RuntimeError("can't start new thread")
+            return real_ensure()
+
+        pm._ensure_update_worker = fail_first_only
+
+        pm.run_scheduled_updates()  # must not propagate the RuntimeError
+
+        assert len(calls) == 2, (
+            "run_scheduled_updates() stopped after the failing plugin; the "
+            "exception escaped the enqueue")
+        for plugin_id in ("plugin-a", "plugin-b"):
+            assert pm.state_manager.get_state(plugin_id) in (
+                PluginState.ENABLED, PluginState.RUNNING), \
+                f"{plugin_id} left in an unexpected state"