fix(store): serialize concurrent updates per plugin, check cleanup results

CodeRabbit review on #405 flagged two things in
_reinstall_with_rollback, both verified against current code:

- Real race: the web UI runs Flask with threaded=True and there's a
  single update route, so two overlapping requests for the same
  plugin_id (double-click, two tabs) can interleave. The loser could
  rename the winner's in-progress install aside mid-download, deleting
  its own rollback safety net — worse than the bug this function
  exists to fix. Added a lazy per-plugin_id lock dict (mirrors the
  plugin_manager per-plugin lock pattern) held for the whole function.
- _safe_remove_directory's return value was ignored at both call
  sites. Stale-aside cleanup failure now aborts cleanly instead of
  falling through to a rename that would fail anyway with a less
  useful error; post-success backup-removal failure now logs instead
  of failing silently (still returns True — the update itself
  succeeded, and the next update self-heals the leftover aside).

Left the third nitpick (test_stale_aside_from_previous_crash_is_cleared)
addressed by asserting the stale dir is actually gone and that
install_plugin was reached, rather than just the end-to-end result.

Added a concurrency regression test asserting install_plugin never
runs for the same plugin_id while another call is in flight.

src/plugin_system/plugin_manager.py

@@ -11,6 +11,7 @@ import json
 import sys
 import time
 import threading
+import types
 from pathlib import Path
 from typing import Dict, List, Optional, Any
 import logging
@@ -743,8 +744,18 @@ class PluginManager:
                         # If resource monitor exists, wrap the call
                         def monitored_update():
                             self.resource_monitor.monitor_call(plugin_id, plugin_instance.update)
+                        # SimpleNamespace stores `update` as an *instance*
+                        # attribute, so attribute lookup returns the plain
+                        # function object as-is. A dynamically-built class
+                        # (`type(..., {'update': monitored_update})`) instead
+                        # stores it as a *class* attribute, which the
+                        # descriptor protocol turns into a bound method on
+                        # access -- silently prepending the instance as an
+                        # implicit first argument to a function that takes
+                        # none, raising "monitored_update() takes 0
+                        # positional arguments but 1 was given" on every call.
                         success = self.plugin_executor.execute_update(
-                            type('obj', (object,), {'update': monitored_update})(),
+                            types.SimpleNamespace(update=monitored_update),
                             plugin_id
                         )
                     else:

src/plugin_system/store_manager.py

@@ -142,9 +142,28 @@ class PluginStoreManager:
         # then get the result from the warm cache (double-checked locking).
         self._registry_fetch_lock = threading.Lock()
 
+        # Per-plugin locks for _reinstall_with_rollback: the web UI runs
+        # Flask with threaded=True, so two overlapping requests for the
+        # same plugin_id (double-click, two browser tabs) would otherwise
+        # both rename the same directory aside — one succeeds, and the
+        # loser can end up renaming the winner's in-progress install aside
+        # mid-download, stealing its own rollback safety net. Keyed by
+        # plugin_id so unrelated plugins still update concurrently.
+        self._reinstall_locks: Dict[str, threading.Lock] = {}
+        self._reinstall_locks_guard = threading.Lock()
+
         # Ensure plugins directory exists
         self.plugins_dir.mkdir(exist_ok=True)
 
+    def _get_reinstall_lock(self, plugin_id: str) -> threading.Lock:
+        """Lazily create (or fetch) the per-plugin reinstall lock."""
+        with self._reinstall_locks_guard:
+            lock = self._reinstall_locks.get(plugin_id)
+            if lock is None:
+                lock = threading.Lock()
+                self._reinstall_locks[plugin_id] = lock
+            return lock
+
     def _record_cache_backoff(self, cache_dict: Dict, cache_key: str,
                               cache_timeout: int, payload: Any) -> None:
         """Bump a cache entry's timestamp so subsequent lookups hit the
@@ -2263,6 +2282,74 @@ class PluginStoreManager:
             self.logger.error(f"Error uninstalling plugin {plugin_id}: {e}")
             return False
     
+    def _reinstall_with_rollback(self, plugin_id: str, plugin_path: Path) -> bool:
+        """Replace an installed plugin with a fresh install, atomically.
+
+        The old install is renamed aside (not deleted) until the new install
+        succeeds, then removed; on ANY install failure the old directory is
+        restored. This is the difference between a failed update and a
+        destroyed plugin: the previous delete-then-install flow permanently
+        removed plugins whenever the download failed mid-update (seen in the
+        field during the monorepo migration on a Pi with broken DNS — every
+        old-remote plugin was deleted and none could be re-downloaded).
+
+        The aside name embeds '.standalone-backup-' so plugin discovery
+        (plugin_manager._scan_directory_for_plugins) ignores it even though
+        it still contains a manifest.json.
+
+        Held for the whole operation under a per-plugin_id lock: two
+        overlapping requests for the same plugin (double-click, two
+        browser tabs — the web UI runs Flask with threaded=True) must not
+        interleave their renames, or the second could steal the first's
+        rollback safety net mid-install. Other plugin_ids are unaffected.
+        """
+        with self._get_reinstall_lock(plugin_id):
+            backup_path = plugin_path.with_name(
+                f"{plugin_path.name}.standalone-backup-migrating")
+            # A stale aside from a previous crash would block the rename
+            if backup_path.exists():
+                if not self._safe_remove_directory(backup_path):
+                    self.logger.error(
+                        f"Could not clear stale backup for {plugin_id} at "
+                        f"{backup_path}; leaving old install in place")
+                    return False
+            try:
+                plugin_path.rename(backup_path)
+            except OSError as e:
+                self.logger.error(
+                    f"Could not set aside old plugin directory for {plugin_id}: {e}")
+                return False
+
+            try:
+                installed = self.install_plugin(plugin_id)
+            except Exception as e:
+                self.logger.error(f"Reinstall of {plugin_id} raised: {e}")
+                installed = False
+
+            if installed:
+                if not self._safe_remove_directory(backup_path):
+                    self.logger.warning(
+                        f"Update of {plugin_id} succeeded but the old backup "
+                        f"at {backup_path} could not be removed; it will be "
+                        f"cleared on the next update")
+                return True
+
+            # Install failed (bad network, registry error...) — put the old
+            # version back so the user still has a working plugin.
+            self.logger.error(
+                f"Reinstall of {plugin_id} failed; restoring previous version")
+            try:
+                if plugin_path.exists():
+                    # partial download debris from the failed install
+                    self._safe_remove_directory(plugin_path)
+                backup_path.rename(plugin_path)
+                self.logger.info(f"Restored previous install of {plugin_id}")
+            except OSError as e:
+                self.logger.error(
+                    f"CRITICAL: could not restore {plugin_id} from {backup_path}: {e}. "
+                    f"The previous install is preserved there — rename it back manually.")
+            return False
+
     def update_plugin(self, plugin_id: str) -> bool:
         """
         Update a plugin to the latest commit on its upstream branch.
@@ -2325,10 +2412,7 @@ class PluginStoreManager:
                             f"Plugin {resolved_id} git remote ({local_remote}) differs from registry ({registry_repo}). "
                             f"Reinstalling from registry to migrate to new source."
                         )
-                        if not self._safe_remove_directory(plugin_path):
-                            self.logger.error(f"Failed to remove old plugin directory for {resolved_id}")
-                            return False
-                        return self.install_plugin(resolved_id)
+                        return self._reinstall_with_rollback(resolved_id, plugin_path)
 
                     # Check if already up to date
                     if remote_sha and local_sha and remote_sha.startswith(local_sha):
@@ -2632,11 +2716,11 @@ class PluginStoreManager:
             # Plugin is not a git repo but is in registry and has a newer version - reinstall
             self.logger.info(f"Plugin {plugin_id} not installed via git; re-installing latest archive (registry id: {registry_id})")
 
-            # Remove directory and reinstall fresh
-            if not self._safe_remove_directory(plugin_path):
-                self.logger.error(f"Failed to remove old plugin directory for {plugin_id}")
-                return False
-            return self.install_plugin(registry_id)
+            # Reinstall with the old version kept aside until the new
+            # download succeeds — this is the path every routine store
+            # update takes, and a mid-update network failure must not
+            # destroy the user's plugin.
+            return self._reinstall_with_rollback(registry_id, plugin_path)
 
         except Exception as e:
             import traceback

src/plugin_system/testing/visual_display_manager.py

@@ -454,6 +454,18 @@ class VisualTestDisplayManager:
         """Check if display is currently scrolling."""
         return self._scrolling_state['is_scrolling']
 
+    def process_deferred_updates(self):
+        """Process any deferred updates (no-op for testing).
+
+        Several ticker-style plugins (news, odds-ticker, leaderboard,
+        stock-news, stocks) call this unconditionally between
+        set_scrolling_state() and their scroll-position update, mirroring the
+        real display_manager's deferred-update queue. This double has no such
+        queue, so there is nothing to process — the no-op just lets those
+        plugins render under the harness instead of raising AttributeError.
+        """
+        pass
+
     # ------------------------------------------------------------------
     # Utility methods
     # ------------------------------------------------------------------

test/plugins/test_visual_rendering.py

@@ -172,6 +172,16 @@ class TestVisualDisplayManager:
         vdm.set_scrolling_state(False)
         assert vdm.is_currently_scrolling() is False
 
+    def test_process_deferred_updates_is_noop(self):
+        # Ticker-style plugins (news, odds-ticker, leaderboard, stock-news,
+        # stocks) call this unconditionally alongside set_scrolling_state();
+        # it must exist and be harmless so those plugins render under the
+        # harness instead of raising AttributeError.
+        vdm = VisualTestDisplayManager(width=128, height=32)
+        vdm.set_scrolling_state(True)
+        vdm.process_deferred_updates()  # should not raise
+        assert vdm.is_currently_scrolling() is True
+
     def test_format_date_with_ordinal(self):
         from datetime import datetime
         vdm = VisualTestDisplayManager(width=128, height=32)

test/test_plugin_system.py

@@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch
 from pathlib import Path
 from src.plugin_system.plugin_manager import PluginManager
 from src.plugin_system.plugin_state import PluginState
+from src.plugin_system.resource_monitor import PluginResourceMonitor
 
 class TestPluginManager:
     """Test PluginManager functionality."""
@@ -74,10 +75,58 @@ class TestPluginManager:
             
             # No manifest in pm.plugin_manifests
             result = pm.load_plugin("non_existent_plugin")
-            
+
             assert result is False
             assert pm.state_manager.get_state("non_existent_plugin") == PluginState.ERROR
 
+    def test_run_scheduled_updates_calls_update_with_resource_monitor(
+        self, mock_config_manager, mock_display_manager, mock_cache_manager
+    ):
+        """Regression test: run_scheduled_updates() must actually call a
+        plugin's update() when self.resource_monitor is set (as it is in
+        every real deployment -- display_controller.py and web_interface/
+        app.py both assign a real PluginResourceMonitor after construction).
+
+        Previously, the resource_monitor branch wrapped the call in a
+        function stored as a *class* attribute on a dynamically-built type
+        (`type('obj', (object,), {'update': monitored_update})()`), which
+        the descriptor protocol turns into a bound method on access --
+        silently passing the synthetic instance as an implicit first
+        argument to monitored_update(), which takes none. Every plugin's
+        scheduled update failed with "monitored_update() takes 0 positional
+        arguments but 1 was given" and was silently swallowed into a
+        circuit-breaker retry loop that never succeeded, so plugin data
+        (scores, odds, etc.) never refreshed.
+        """
+        with patch('src.plugin_system.plugin_manager.ensure_directory_permissions'):
+            pm = PluginManager(
+                plugins_dir="plugins",
+                config_manager=mock_config_manager,
+                display_manager=mock_display_manager,
+                cache_manager=mock_cache_manager
+            )
+
+            plugin_instance = MagicMock()
+            plugin_instance.enabled = True
+            plugin_instance.update = MagicMock()
+
+            pm.plugins["test_plugin"] = plugin_instance
+            pm.plugin_manifests["test_plugin"] = {"update_interval": 10}
+            pm.state_manager.set_state("test_plugin", PluginState.ENABLED)
+            # Plain MagicMock, not the mock_cache_manager fixture: this test
+            # is about run_scheduled_updates() actually invoking update()
+            # through the resource-monitor wrapper, not about
+            # PluginResourceMonitor's own cache-backed metrics persistence
+            # (which calls cache_manager.get(..., memory_ttl=...) --
+            # a kwarg the fixture's mock_get() doesn't accept).
+            pm.resource_monitor = PluginResourceMonitor(MagicMock())
+
+            pm.run_scheduled_updates(current_time=time.time())
+
+            plugin_instance.update.assert_called_once()
+            assert "test_plugin" in pm.plugin_last_update
+            assert pm.state_manager.get_state("test_plugin") == PluginState.ENABLED
+
 
 class TestPluginLoader:
     """Test PluginLoader functionality."""

test/test_store_update_rollback.py

@@ -0,0 +1,171 @@
+"""Tests for atomic plugin updates (store_manager._reinstall_with_rollback).
+
+Regression for a field data-loss incident: update_plugin's reinstall paths
+(monorepo migration AND routine archive updates) deleted the installed
+plugin BEFORE downloading its replacement — a mid-update network failure
+permanently destroyed the plugin. Seen live: a Pi with broken DNS lost 12
+plugins from one update pass.
+"""
+
+import json
+import os
+import sys
+import threading
+import time
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from src.plugin_system.store_manager import PluginStoreManager  # noqa: E402
+
+PLUGIN_ID = "rollback-test-plugin"
+
+
+@pytest.fixture
+def store(tmp_path):
+    mgr = PluginStoreManager(plugins_dir=str(tmp_path))
+    plugin_dir = tmp_path / PLUGIN_ID
+    plugin_dir.mkdir()
+    (plugin_dir / "manifest.json").write_text(json.dumps(
+        {"id": PLUGIN_ID, "name": "Rollback Test", "version": "1.0.0"}))
+    (plugin_dir / "manager.py").write_text("# old version marker\n")
+    return mgr, plugin_dir
+
+
+class TestReinstallWithRollback:
+    def test_failed_install_restores_old_version(self, store):
+        """The whole point: a failed download must leave the old install."""
+        mgr, plugin_dir = store
+        with patch.object(mgr, "install_plugin", return_value=False):
+            ok = mgr._reinstall_with_rollback(PLUGIN_ID, plugin_dir)
+        assert ok is False
+        assert plugin_dir.exists()
+        assert "old version marker" in (plugin_dir / "manager.py").read_text()
+        # no aside debris left behind
+        leftovers = [p for p in plugin_dir.parent.iterdir()
+                     if "standalone-backup" in p.name]
+        assert leftovers == []
+
+    def test_install_exception_restores_old_version(self, store):
+        mgr, plugin_dir = store
+        with patch.object(mgr, "install_plugin",
+                          side_effect=RuntimeError("network down")):
+            ok = mgr._reinstall_with_rollback(PLUGIN_ID, plugin_dir)
+        assert ok is False
+        assert plugin_dir.exists()
+        assert "old version marker" in (plugin_dir / "manager.py").read_text()
+
+    def test_successful_install_removes_aside(self, store):
+        mgr, plugin_dir = store
+
+        def fake_install(plugin_id):
+            new_dir = plugin_dir  # same path, new content
+            new_dir.mkdir(exist_ok=True)
+            (new_dir / "manager.py").write_text("# new version\n")
+            (new_dir / "manifest.json").write_text(json.dumps(
+                {"id": PLUGIN_ID, "name": "Rollback Test", "version": "2.0.0"}))
+            return True
+
+        with patch.object(mgr, "install_plugin", side_effect=fake_install):
+            ok = mgr._reinstall_with_rollback(PLUGIN_ID, plugin_dir)
+        assert ok is True
+        assert "new version" in (plugin_dir / "manager.py").read_text()
+        leftovers = [p for p in plugin_dir.parent.iterdir()
+                     if "standalone-backup" in p.name]
+        assert leftovers == []
+
+    def test_partial_download_debris_is_replaced_by_old_version(self, store):
+        """A failed install that left a partial directory must still roll back."""
+        mgr, plugin_dir = store
+
+        def fake_partial_install(plugin_id):
+            plugin_dir.mkdir(exist_ok=True)
+            (plugin_dir / "half-downloaded.tmp").write_text("junk")
+            return False
+
+        with patch.object(mgr, "install_plugin", side_effect=fake_partial_install):
+            ok = mgr._reinstall_with_rollback(PLUGIN_ID, plugin_dir)
+        assert ok is False
+        assert "old version marker" in (plugin_dir / "manager.py").read_text()
+        assert not (plugin_dir / "half-downloaded.tmp").exists()
+
+    def test_stale_aside_from_previous_crash_is_cleared(self, store):
+        mgr, plugin_dir = store
+        stale = plugin_dir.parent / f"{PLUGIN_ID}.standalone-backup-migrating"
+        stale.mkdir()
+        (stale / "old.txt").write_text("stale")
+        with patch.object(mgr, "install_plugin", return_value=False) as mock_install:
+            ok = mgr._reinstall_with_rollback(PLUGIN_ID, plugin_dir)
+        # The reinstall itself still fails (mocked) and the old install is
+        # restored, but the stale aside must not have survived — otherwise
+        # it would have blocked this run's own rename (or a future one).
+        assert not stale.exists()
+        mock_install.assert_called_once_with(PLUGIN_ID)
+        assert ok is False
+        assert plugin_dir.exists()
+        assert "old version marker" in (plugin_dir / "manager.py").read_text()
+
+    def test_concurrent_updates_for_same_plugin_are_serialized(self, store):
+        """Two overlapping requests for the same plugin_id (double-click,
+        two browser tabs — the web UI runs Flask with threaded=True) must
+        not interleave: the loser must wait for the winner to finish
+        rather than renaming the winner's in-progress install aside and
+        stealing its rollback safety net."""
+        mgr, plugin_dir = store
+
+        active = 0
+        max_active = 0
+        guard = threading.Lock()
+
+        def fake_install(plugin_id):
+            nonlocal active, max_active
+            with guard:
+                active += 1
+                max_active = max(max_active, active)
+            time.sleep(0.05)
+            plugin_dir.mkdir(exist_ok=True)
+            (plugin_dir / "manager.py").write_text("# new version\n")
+            (plugin_dir / "manifest.json").write_text(json.dumps(
+                {"id": PLUGIN_ID, "name": "Rollback Test", "version": "2.0.0"}))
+            with guard:
+                active -= 1
+            return True
+
+        results = []
+
+        def worker():
+            results.append(mgr._reinstall_with_rollback(PLUGIN_ID, plugin_dir))
+
+        with patch.object(mgr, "install_plugin", side_effect=fake_install):
+            threads = [threading.Thread(target=worker) for _ in range(2)]
+            for t in threads:
+                t.start()
+            for t in threads:
+                t.join(timeout=5)
+
+        assert max_active == 1, "install_plugin ran concurrently for the same plugin_id"
+        assert results == [True, True]
+        assert plugin_dir.exists()
+        assert "new version" in (plugin_dir / "manager.py").read_text()
+        leftovers = [p for p in plugin_dir.parent.iterdir()
+                     if "standalone-backup" in p.name]
+        assert leftovers == []
+
+    def test_aside_name_is_invisible_to_discovery(self, store, tmp_path):
+        """The aside still contains a manifest.json — discovery must skip it
+        (relies on the existing '.standalone-backup-' exclusion)."""
+        mgr, plugin_dir = store
+        from src.plugin_system.plugin_manager import PluginManager
+        aside = plugin_dir.parent / f"{PLUGIN_ID}.standalone-backup-migrating"
+        plugin_dir.rename(aside)
+        pm = PluginManager(plugins_dir=str(tmp_path), config_manager=None,
+                           display_manager=None, cache_manager=None)
+        found = pm._scan_directory_for_plugins(Path(tmp_path))
+        assert PLUGIN_ID not in found
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-v"]))