fix(plugin-system): load/enable failures, atomic state files, pip lock, test-double parity (#645)

- load_plugin: an on_enable() that raises unregisters the instance, so the
  next load retries instead of returning True "already loaded".
- get_plugin_info: guard plugin.get_info(); one plugin raising no longer
  breaks /api/v3/plugins/installed.
- plugin_state.json and the operation history are written with
  atomic_write_text under their lock.
- plugin_loader: module-level lock serialises pip installs across the
  parallel startup loaders.
- store_manager._install_via_download: extract dir cleanup moved to finally.
- Test doubles: draw_image() warns (DeprecationWarning; the real
  DisplayManager has none), MockDisplayManager.draw_text accepts the real
  signature's optional params, VisualTestDisplayManager logs draw errors at
  WARNING.
- Docs/comments: compatibility.py method name, PluginState.LOADED meaning,
  brittle schema count, why _report_skip_once uses setdefault.
- Remove unused PluginOperationQueue.get_active_operations().

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

test/test_install_via_download_cleanup.py

@@ -0,0 +1,66 @@
+"""_install_via_download removes its extraction directory on failure too.
+
+The temp extract dir was only removed on the success path (and on a zip-slip
+abort), so an extract or move that raised left a full copy of the plugin in
+the system temp dir on every failed attempt. Cleanup now lives in ``finally``.
+"""
+
+import io
+import tempfile
+import zipfile
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+from src.plugin_system.store_manager import PluginStoreManager
+
+
+def _zip_bytes():
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, 'w') as zf:
+        zf.writestr('demo-main/manifest.json', '{"id": "demo"}')
+        zf.writestr('demo-main/manager.py', '')
+    return buf.getvalue()
+
+
+def _response(payload):
+    response = MagicMock()
+    response.raise_for_status.return_value = None
+    response.iter_content.return_value = [payload]
+    return response
+
+
+def _run(tmp_path, move_side_effect=None):
+    plugins_dir = tmp_path / "plugin-repos"
+    plugins_dir.mkdir()
+    sm = PluginStoreManager(plugins_dir=str(plugins_dir))
+    created = []
+    real_mkdtemp = tempfile.mkdtemp
+
+    def tracking_mkdtemp(*args, **kwargs):
+        path = real_mkdtemp(*args, **kwargs)
+        created.append(Path(path))
+        return path
+
+    with patch.object(sm, '_http_get_with_retries', return_value=_response(_zip_bytes())), \
+            patch('src.plugin_system.store_manager.tempfile.mkdtemp', side_effect=tracking_mkdtemp), \
+            patch('src.plugin_system.store_manager.shutil.move', side_effect=move_side_effect):
+        ok = sm._install_via_download('https://example.invalid/demo.zip', plugins_dir / 'demo')
+    return ok, created
+
+
+def test_extract_dir_removed_when_the_move_fails(tmp_path):
+    ok, created = _run(tmp_path, move_side_effect=OSError("disk full"))
+
+    assert ok is False
+    assert len(created) == 1
+    assert not created[0].exists()
+
+
+def test_extract_dir_removed_on_success(tmp_path):
+    # shutil.move patched to a no-op: the extracted tree stays in temp and
+    # must still be cleaned up.
+    ok, created = _run(tmp_path, move_side_effect=lambda *a, **k: None)
+
+    assert ok is True
+    assert len(created) == 1
+    assert not created[0].exists()

test/test_plugin_loader.py

@@ -342,3 +342,49 @@ class TestPluginLoader:
 
         assert result is False
         mock_subprocess.assert_not_called()
+
+    @patch('src.plugin_system.plugin_loader.requirements_are_satisfied', return_value=False)
+    def test_install_dependencies_never_runs_pip_concurrently(
+        self, mock_satisfied, tmp_plugins_dir
+    ):
+        """Startup loads plugins on a thread pool; two pip processes writing
+        the same site-packages at once can corrupt it, so installs for
+        different plugins must run one at a time."""
+        import threading
+        import time
+
+        state = {'running': 0, 'peak': 0}
+        guard = threading.Lock()
+
+        def fake_pip(*args, **kwargs):
+            with guard:
+                state['running'] += 1
+                state['peak'] = max(state['peak'], state['running'])
+            time.sleep(0.05)
+            with guard:
+                state['running'] -= 1
+            return MagicMock(returncode=0, stderr="")
+
+        plugin_dirs = []
+        for n in range(4):
+            d = tmp_plugins_dir / f"plugin_{n}"
+            d.mkdir()
+            (d / "requirements.txt").write_text("package1==1.0.0\n")
+            plugin_dirs.append(d)
+
+        results = []
+
+        def install(d):
+            results.append(PluginLoader().install_dependencies(
+                d, d.name, plugins_dir=tmp_plugins_dir))
+
+        with patch('subprocess.run', side_effect=fake_pip) as mock_subprocess:
+            threads = [threading.Thread(target=install, args=(d,)) for d in plugin_dirs]
+            for t in threads:
+                t.start()
+            for t in threads:
+                t.join()
+
+        assert results == [True] * 4
+        assert mock_subprocess.call_count == 4
+        assert state['peak'] == 1

test/test_plugin_manager_load_failures.py

@@ -0,0 +1,84 @@
+"""PluginManager failure paths that used to leave the manager inconsistent.
+
+- load_plugin registered the instance in ``self.plugins`` before calling
+  ``on_enable()``. When on_enable raised, the plugin stayed registered in
+  ERROR state, so the next load_plugin returned True ("already loaded") for
+  a plugin that never ran.
+- get_plugin_info called ``plugin.get_info()`` unguarded, so one plugin
+  raising broke /api/v3/plugins/installed for every plugin.
+"""
+
+import logging
+from unittest.mock import MagicMock
+
+import pytest
+
+from src.plugin_system.plugin_manager import PluginManager
+from src.plugin_system.plugin_state import PluginState
+
+
+class _Plugin:
+    def __init__(self, fail_enable=False, fail_info=False):
+        self.fail_enable = fail_enable
+        self.fail_info = fail_info
+        self.enabled_calls = 0
+
+    def on_enable(self):
+        self.enabled_calls += 1
+        if self.fail_enable:
+            raise RuntimeError("on_enable blew up")
+
+    def get_info(self):
+        if self.fail_info:
+            raise RuntimeError("get_info blew up")
+        return {"ok": True}
+
+
+@pytest.fixture
+def pm(tmp_path):
+    plugins_dir = tmp_path / "plugins"
+    (plugins_dir / "demo").mkdir(parents=True)
+    manager = PluginManager(plugins_dir=str(plugins_dir))
+    manager.plugin_manifests["demo"] = {"id": "demo", "name": "Demo"}
+    manager.schema_manager = MagicMock()
+    manager.schema_manager.get_schema_path.return_value = None
+    manager.plugin_loader = MagicMock()
+    manager.plugin_loader.find_plugin_directory.return_value = plugins_dir / "demo"
+    return manager
+
+
+def test_on_enable_failure_unregisters_the_plugin(pm):
+    plugin = _Plugin(fail_enable=True)
+    pm.plugin_loader.load_plugin.return_value = (plugin, None)
+
+    assert pm.load_plugin("demo") is False
+    assert "demo" not in pm.plugins
+    assert "demo" not in pm.plugin_last_update
+    assert pm.state_manager.get_state("demo") == PluginState.ERROR
+
+
+def test_load_after_on_enable_failure_retries_instead_of_already_loaded(pm):
+    pm.plugin_loader.load_plugin.return_value = (_Plugin(fail_enable=True), None)
+    assert pm.load_plugin("demo") is False
+
+    fixed = _Plugin()
+    pm.plugin_loader.load_plugin.return_value = (fixed, None)
+    assert pm.load_plugin("demo") is True
+    assert pm.plugins["demo"] is fixed
+    assert fixed.enabled_calls == 1
+    assert pm.state_manager.get_state("demo") == PluginState.ENABLED
+
+
+def test_get_info_failure_does_not_break_the_listing(pm, caplog):
+    pm.plugin_manifests["good"] = {"id": "good", "name": "Good"}
+    pm.plugins["demo"] = _Plugin(fail_info=True)
+    pm.plugins["good"] = _Plugin()
+
+    with caplog.at_level(logging.WARNING):
+        infos = {i["id"]: i for i in pm.get_all_plugin_info()}
+
+    assert infos["demo"]["loaded"] is True
+    assert infos["demo"]["runtime_info"] == {}
+    assert infos["good"]["runtime_info"] == {"ok": True}
+    assert any("demo" in r.getMessage() and r.levelno == logging.WARNING
+               for r in caplog.records)

test/test_plugin_state_files_atomic.py

@@ -0,0 +1,57 @@
+"""plugin_state.json and the operation history file are replaced atomically.
+
+Both were written with a plain ``open(path, 'w')`` + ``json.dump`` outside
+their lock. The open truncates first, so a value json can't encode (or a
+crash, or a second Flask thread saving at the same moment) left a partial
+file, and the next load dropped every saved state. They now serialise first
+and go through a temp file + rename while holding the lock.
+"""
+
+import json
+import threading
+
+from src.plugin_system.operation_history import OperationHistory
+from src.plugin_system.state_manager import PluginStateManager
+
+
+def test_state_file_survives_a_failed_save(tmp_path):
+    state_file = tmp_path / "plugin_state.json"
+    mgr = PluginStateManager(state_file=str(state_file))
+    mgr.set_plugin_enabled("clock", True)
+    before = json.loads(state_file.read_text())
+
+    # Not JSON-serialisable: the save fails (and is logged, not raised).
+    mgr.update_plugin_state("clock", {"metadata": {"bad": object()}})
+
+    assert json.loads(state_file.read_text()) == before
+    assert [p.name for p in tmp_path.iterdir()] == ["plugin_state.json"]
+
+
+def test_state_file_is_valid_after_concurrent_saves(tmp_path):
+    state_file = tmp_path / "plugin_state.json"
+    mgr = PluginStateManager(state_file=str(state_file))
+
+    def worker(n):
+        for i in range(15):
+            mgr.set_plugin_enabled(f"plugin-{n}", i % 2 == 0)
+
+    threads = [threading.Thread(target=worker, args=(n,)) for n in range(6)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+
+    data = json.loads(state_file.read_text())
+    assert set(data["states"]) == {f"plugin-{n}" for n in range(6)}
+
+
+def test_history_file_survives_a_failed_save(tmp_path):
+    history_file = tmp_path / "operation_history.json"
+    history = OperationHistory(history_file=str(history_file))
+    history.record_operation("install", plugin_id="clock")
+    before = json.loads(history_file.read_text())
+
+    history.record_operation("update", plugin_id="clock", details={"bad": object()})
+
+    assert json.loads(history_file.read_text()) == before
+    assert [p.name for p in tmp_path.iterdir()] == ["operation_history.json"]

test/test_testing_mocks.py

@@ -9,6 +9,8 @@ get_cached_data_with_strategy() and previously hit an AttributeError that
 its own broad except swallowed, producing an empty-but-green render).
 """
 
+import pytest
+
 from src.plugin_system.testing.mocks import MockCacheManager
 
 
@@ -43,3 +45,64 @@ class TestMockCacheManagerStrategyMethod:
         cm.get_cached_data_with_strategy("k", "sports_live")
         cm.reset()
         assert cm.get_cached_data_with_strategy_calls == []
+
+
+class TestDisplayDoublesMatchTheRealDisplayManager:
+    """The real DisplayManager has no draw_image(); the doubles keep it so
+    existing plugin test suites still pass, but warn, because a plugin that
+    calls it passes its tests and then raises AttributeError on the Pi."""
+
+    @staticmethod
+    def _real_display_manager(monkeypatch):
+        # Without EMULATOR the import needs the hardware rgbmatrix module.
+        monkeypatch.setenv("EMULATOR", "true")
+        from src.display_manager import DisplayManager
+        return DisplayManager
+
+    def test_real_display_manager_has_no_draw_image(self, monkeypatch):
+        DisplayManager = self._real_display_manager(monkeypatch)
+        assert not hasattr(DisplayManager, 'draw_image')
+
+    def test_mock_draw_image_warns_and_still_records(self):
+        from PIL import Image
+        from src.plugin_system.testing.mocks import MockDisplayManager
+
+        dm = MockDisplayManager()
+        with pytest.warns(DeprecationWarning, match=r"image\.paste\(img, \(x, y\)\)"):
+            dm.draw_image(Image.new('RGB', (4, 4)), 1, 2)
+        assert dm.draw_calls[-1]['type'] == 'image'
+
+    @pytest.mark.parametrize("cls_name", ["VisualTestDisplayManager", "BoundsCheckingDisplayManager"])
+    def test_visual_draw_image_warns_and_still_pastes(self, cls_name):
+        from PIL import Image
+        import src.plugin_system.testing as testing
+
+        dm = getattr(testing, cls_name)(width=16, height=8)
+        with pytest.warns(DeprecationWarning, match="DisplayManager has no such method"):
+            dm.draw_image(Image.new('RGB', (2, 2), (0, 0, 255)), 3, 3)
+        assert dm.image.getpixel((3, 3)) == (0, 0, 255)
+
+    def test_mock_draw_text_accepts_the_real_signature(self, monkeypatch):
+        import inspect
+        DisplayManager = self._real_display_manager(monkeypatch)
+        from src.plugin_system.testing.mocks import MockDisplayManager
+
+        real = inspect.signature(DisplayManager.draw_text).parameters
+        mock = inspect.signature(MockDisplayManager.draw_text).parameters
+        for name, param in real.items():
+            assert name in mock, name
+            assert mock[name].default == param.default, name
+
+        dm = MockDisplayManager()
+        dm.draw_text("hi", small_font=True, centered=True)
+        assert dm.draw_calls[-1]['text'] == "hi"
+
+    def test_visual_draw_failure_is_logged_at_warning(self, caplog):
+        import logging
+        from src.plugin_system.testing import VisualTestDisplayManager
+
+        dm = VisualTestDisplayManager(width=16, height=8)
+        with caplog.at_level(logging.WARNING):
+            dm.draw_image(object(), 0, 0)  # not an image: paste raises
+        assert any(r.levelno == logging.WARNING and "Error drawing image" in r.getMessage()
+                   for r in caplog.records)