LEDMatrix/test/test_plugin_runtime_snapshot.py

"""The display publishes its plugin runtime state; the web reads it back.

Only the display runs plugins, so only it knows which ones are loaded, where
each is in its lifecycle, why one failed and which version it runs. It
publishes that as one snapshot in the shared cache (plugin_runtime), and the
web interface fills /api/v3/plugins/installed's ``loaded`` / ``state`` /
``error_info`` from it -- but only while the snapshot is live. A stale,
stopped or missing snapshot is reported as such, with those fields null.

The cross-process tests use two CacheManagers over one temporary directory,
the arrangement of the real services (which share /var/cache/ledmatrix).
"""
import json
import sys
from pathlib import Path
from unittest.mock import MagicMock

import pytest

sys.path.insert(0, str(Path(__file__).parent.parent))

from src.cache_manager import CacheManager  # noqa: E402
from src.plugin_system import plugin_runtime as rt  # noqa: E402
from src.plugin_system.plugin_manager import PluginManager  # noqa: E402
from src.plugin_system.plugin_runtime import (  # noqa: E402
    PLUGIN_RUNTIME_KEY, PluginRuntimePublisher, read_plugin_runtime,
    view_from_snapshot,
)
from src.plugin_system.plugin_state import PluginState, PluginStateManager  # noqa: E402
from test._api_v3_test_helpers import api_v3_client, api_v3_module  # noqa: F401,E402


class FakeClock:
    def __init__(self, now=1000.0):
        self.now = now

    def __call__(self):
        return self.now


@pytest.fixture
def shared_cache(tmp_path, monkeypatch):
    """Two cache managers over one directory: the display's and the web's."""
    monkeypatch.setattr(CacheManager, "_get_writable_cache_dir", lambda self: str(tmp_path))
    display_cache, web_cache = CacheManager(), CacheManager()
    yield display_cache, web_cache
    display_cache.stop_cleanup_thread()
    web_cache.stop_cleanup_thread()


def _publisher(cache, states, wall=None, mono=None):
    return PluginRuntimePublisher(cache, states, clock=mono or FakeClock(),
                                  wall_clock=wall or FakeClock(1_800_000_000.0))


# --- The state machine: what counts as a change -----------------------------

class TestChangeCount:
    def test_an_ordinary_update_is_not_a_change(self):
        """ENABLED -> RUNNING -> ENABLED is every update() call. Publishing it
        would be an SD-card write per plugin update."""
        states = PluginStateManager()
        states.set_state("clock", PluginState.ENABLED)
        before = states.change_count
        for _ in range(50):
            states.set_state("clock", PluginState.RUNNING)
            states.set_state("clock", PluginState.ENABLED)
        assert states.change_count == before
        assert states.runtime_records()["clock"]["state"] == "enabled"

    def test_running_is_published_as_enabled(self):
        states = PluginStateManager()
        states.set_state("clock", PluginState.RUNNING)
        assert states.runtime_records()["clock"]["state"] == "enabled"

    @pytest.mark.parametrize("change", [
        lambda s: s.set_state("clock", PluginState.ERROR, error=ValueError("x")),
        lambda s: s.set_state_with_error("clock", PluginState.ENABLED, {"error": "x"}),
        lambda s: s.record_loaded("clock", "1.0.0"),
        lambda s: s.set_state("clock", PluginState.DISABLED),
        lambda s: s.clear_state("clock"),
        lambda s: s.set_state("weather", PluginState.LOADED),
    ])
    def test_reader_visible_changes_move_it(self, change):
        states = PluginStateManager()
        states.set_state("clock", PluginState.ENABLED)
        before = states.change_count
        change(states)
        assert states.change_count > before

    def test_recovering_from_an_error_is_a_change(self):
        states = PluginStateManager()
        states.set_state_with_error("clock", PluginState.ENABLED, {"error": "timeout"})
        before = states.change_count
        states.set_state("clock", PluginState.ENABLED)  # next update succeeded
        assert states.change_count > before
        assert states.runtime_records()["clock"]["error_info"] is None

    def test_clear_state_forgets_the_loaded_record(self):
        states = PluginStateManager()
        states.set_state("clock", PluginState.ENABLED)
        states.record_loaded("clock", "1.0.0", loaded_at=5.0)
        assert states.runtime_records()["clock"]["loaded"] is True
        states.clear_state("clock")
        assert states.runtime_records() == {}


class TestPluginManagerRecordsWhatItLoaded:
    @pytest.fixture
    def pm(self, tmp_path):
        plugins_dir = tmp_path / "plugins"
        (plugins_dir / "demo").mkdir(parents=True)
        manager = PluginManager(plugins_dir=str(plugins_dir))
        manager.plugin_manifests["demo"] = {"id": "demo", "name": "Demo", "version": "2.1.0"}
        manager.schema_manager = MagicMock()
        manager.schema_manager.get_schema_path.return_value = None
        manager.plugin_loader = MagicMock()
        manager.plugin_loader.find_plugin_directory.return_value = plugins_dir / "demo"
        manager.plugin_loader.load_plugin.return_value = (MagicMock(spec=["on_enable"]), None)
        return manager

    def test_load_records_the_manifest_version(self, pm):
        assert pm.load_plugin("demo") is True
        record = pm.state_manager.runtime_records()["demo"]
        assert record["loaded"] is True
        assert record["state"] == "enabled"
        assert record["version"] == "2.1.0"
        assert isinstance(record["loaded_at"], float)

    def test_unload_forgets_it(self, pm):
        pm.load_plugin("demo")
        pm.unload_plugin("demo")
        assert "demo" not in pm.state_manager.runtime_records()

    def test_a_failed_load_is_an_error_and_not_loaded(self, pm):
        pm.plugin_loader.load_plugin.side_effect = ImportError("No module named 'requests'")
        assert pm.load_plugin("demo") is False
        record = pm.state_manager.runtime_records()["demo"]
        assert record["loaded"] is False
        assert record["state"] == "error"
        assert "requests" in record["error_info"]["error"]


# --- Publishing -------------------------------------------------------------

class TestPublisher:
    def test_first_tick_publishes(self):
        cache = MagicMock()
        states = PluginStateManager()
        states.set_state("clock", PluginState.ENABLED)
        states.record_loaded("clock", "1.0.0", loaded_at=10.0)

        assert _publisher(cache, states).tick() is True

        key, snapshot = cache.set.call_args.args
        assert key == PLUGIN_RUNTIME_KEY
        assert snapshot["running"] is True
        assert snapshot["published_at"] == 1_800_000_000.0
        assert snapshot["stale_after"] == rt.STALE_AFTER
        assert snapshot["plugins"] == {"clock": {
            "loaded": True, "state": "enabled", "error": None,
            "version": "1.0.0", "loaded_at": 10.0}}

    def test_changes_are_throttled_and_quiet_displays_refresh(self):
        cache = MagicMock()
        states = PluginStateManager()
        mono = FakeClock()
        publisher = _publisher(cache, states, mono=mono)
        publisher.tick()
        assert cache.set.call_count == 1

        # Nothing changed: no write until the refresh is due.
        mono.now += rt.REFRESH_INTERVAL - 1
        assert publisher.tick() is False
        mono.now += 1
        assert publisher.tick() is True
        assert cache.set.call_count == 2

        # A burst of changes is written at most once per MIN_INTERVAL.
        states.set_state("a", PluginState.LOADED)
        mono.now += 1
        assert publisher.tick() is False
        mono.now += rt.MIN_INTERVAL
        assert publisher.tick() is True
        assert cache.set.call_count == 3

    def test_updates_alone_cause_no_writes(self):
        """A display running plugins but changing nothing writes once a
        minute, however many update() calls it makes."""
        cache = MagicMock()
        states = PluginStateManager()
        states.set_state("clock", PluginState.ENABLED)
        mono = FakeClock()
        publisher = _publisher(cache, states, mono=mono)
        publisher.tick()
        for _ in range(11):  # 55 s of 5 s ticks
            states.set_state("clock", PluginState.RUNNING)
            states.set_state("clock", PluginState.ENABLED)
            mono.now += rt.TICK_INTERVAL
            publisher.tick()
        assert cache.set.call_count == 1

    def test_errors_are_redacted_and_short(self):
        cache = MagicMock()
        states = PluginStateManager()
        secret = "https://api.example.com/v1?api_key=SUPERSECRET123&q=" + "x" * 500
        states.set_state("weather", PluginState.ERROR, error=ConnectionError(secret))

        _publisher(cache, states).tick()

        error = cache.set.call_args.args[1]["plugins"]["weather"]["error"]
        assert "SUPERSECRET123" not in json.dumps(error)
        assert len(error["message"]) <= 200
        assert error["type"] == "ConnectionError"
        assert isinstance(error["at"], float)
        assert error["recoverable"] is False

    def test_stop_publishes_stopped(self):
        cache = MagicMock()
        states = PluginStateManager()
        states.record_loaded("clock", "1.0.0")
        publisher = _publisher(cache, states)
        publisher.tick()

        publisher.stop()

        snapshot = cache.set.call_args.args[1]
        assert snapshot["running"] is False
        assert snapshot["plugins"] == {}

    def test_a_failing_cache_never_raises(self):
        cache = MagicMock()
        cache.set.side_effect = OSError("read-only file system")
        publisher = _publisher(cache, PluginStateManager())
        assert publisher.tick() is False
        publisher.stop()  # also swallowed

    def test_the_thread_starts_and_stops(self):
        cache = MagicMock()
        publisher = rt.start_plugin_runtime_publisher(cache, PluginStateManager())
        try:
            assert publisher is not None
        finally:
            publisher.stop()
        assert cache.set.call_args.args[1]["running"] is False


# --- Reading: live, stale, stopped, unknown ---------------------------------

def _snapshot(published_at, running=True, plugins=None, **extra):
    base = {"schema": rt.SNAPSHOT_SCHEMA, "running": running,
            "published_at": published_at, "stale_after": 180.0,
            "plugins": plugins or {}}
    base.update(extra)
    return base


class TestReader:
    NOW = 1_800_000_000.0
    CLOCK = {"loaded": True, "state": "enabled", "error": None,
             "version": "1.0.0", "loaded_at": 5.0}

    def test_live(self):
        view = view_from_snapshot(_snapshot(self.NOW - 30, plugins={"clock": self.CLOCK}),
                                  now=self.NOW)
        assert view.status == "live"
        assert view.plugin("clock") == {"loaded": True, "state": "enabled",
                                        "error_info": None, "loaded_version": "1.0.0",
                                        "loaded_at": 5.0}
        # Listed nowhere in a live snapshot: not loaded.
        assert view.plugin("weather")["loaded"] is False
        assert view.plugin("weather")["state"] == "unloaded"
        assert view.describe()["age_seconds"] == 30.0

    def test_stale_reports_nothing(self):
        view = view_from_snapshot(_snapshot(self.NOW - 181, plugins={"clock": self.CLOCK}),
                                  now=self.NOW)
        assert view.status == "stale"
        assert view.plugin("clock") == {"loaded": None, "state": None, "error_info": None,
                                        "loaded_version": None, "loaded_at": None}
        assert view.describe()["status"] == "stale"

    def test_stopped_reports_nothing(self):
        view = view_from_snapshot(_snapshot(self.NOW - 1, running=False), now=self.NOW)
        assert view.status == "stopped"
        assert view.plugin("clock")["loaded"] is None

    @pytest.mark.parametrize("snapshot", [
        None, "junk", [], {}, {"schema": 99, "running": True, "published_at": NOW},
        _snapshot(None), _snapshot("yesterday"), _snapshot(float("nan")),
    ])
    def test_unknown(self, snapshot):
        view = view_from_snapshot(snapshot, now=self.NOW)
        assert view.status == "unknown"
        assert view.plugin("clock")["state"] is None

    def test_a_little_in_the_future_is_live_far_is_stale(self):
        """The Pi has no RTC: its clock steps at NTP sync."""
        assert view_from_snapshot(_snapshot(self.NOW + 60), now=self.NOW).status == "live"
        assert view_from_snapshot(_snapshot(self.NOW + 3600), now=self.NOW).status == "stale"

    def test_a_corrupt_stale_after_is_bounded(self):
        forever = _snapshot(self.NOW - 7200, stale_after=1e12)
        assert view_from_snapshot(forever, now=self.NOW).status == "stale"
        eager = _snapshot(self.NOW - 20, stale_after=0)
        assert view_from_snapshot(eager, now=self.NOW).status == "live"

    def test_no_cache_manager_or_a_failing_one_is_unknown(self):
        assert read_plugin_runtime(None).status == "unknown"
        cache = MagicMock()
        cache.get.side_effect = OSError("gone")
        assert read_plugin_runtime(cache).status == "unknown"


# --- Across processes, through the real cache and the real routes -----------

class TestAcrossProcesses:
    def test_what_the_display_publishes_the_web_reads(self, shared_cache):
        display_cache, web_cache = shared_cache
        states = PluginStateManager()
        states.set_state("clock", PluginState.ENABLED)
        states.record_loaded("clock", "1.0.0")
        publisher = PluginRuntimePublisher(display_cache, states)

        assert read_plugin_runtime(web_cache).status == "unknown"
        publisher.tick()
        view = read_plugin_runtime(web_cache)
        assert view.status == "live"
        assert view.plugin("clock")["loaded"] is True

        # memory_ttl=0 on the reading side: a later write is seen at once.
        publisher.stop()
        assert read_plugin_runtime(web_cache).status == "stopped"

    def test_a_snapshot_left_by_a_dead_display_goes_stale(self, shared_cache):
        display_cache, web_cache = shared_cache
        states = PluginStateManager()
        states.record_loaded("clock", "1.0.0")
        old = FakeClock(1_000_000.0)  # long ago
        PluginRuntimePublisher(display_cache, states, wall_clock=old).tick()
        assert read_plugin_runtime(web_cache).status == "stale"


@pytest.fixture
def web_listing(api_v3_module, api_v3_client, shared_cache, tmp_path):  # noqa: F811
    """GET /api/v3/plugins/installed over one real plugin and the shared cache."""
    display_cache, web_cache = shared_cache
    api = api_v3_module.api_v3
    api.cache_manager = web_cache
    api.plugin_catalog.plugins_dir = str(tmp_path / "plugins")
    api.plugin_catalog.get_all_plugin_info = MagicMock(return_value=[
        {"id": "clock", "name": "Clock", "version": "1.1.0"},
        {"id": "weather", "name": "Weather", "version": "3.0.0"},
    ])
    api.plugin_store_manager.get_registry_info = MagicMock(return_value=None)
    api.plugin_store_manager._get_local_git_info = MagicMock(return_value=None)
    api.config_manager.load_config = MagicMock(return_value={
        "clock": {"enabled": True}, "weather": {"enabled": True}})

    def get():
        response = api_v3_client.get("/api/v3/plugins/installed")
        assert response.status_code == 200, response.get_data(as_text=True)
        data = response.get_json()["data"]
        return {p["id"]: p for p in data["plugins"]}, data["runtime"]

    return display_cache, get


class TestInstalledPluginsRoute:
    def _display(self, display_cache):
        states = PluginStateManager()
        states.set_state("clock", PluginState.ENABLED)
        states.record_loaded("clock", "1.0.0", loaded_at=123.0)
        states.set_state("weather", PluginState.ERROR,
                         error=RuntimeError("token=abc123 rejected"))
        return PluginRuntimePublisher(display_cache, states)

    def test_restores_loaded_state_and_error_info(self, web_listing):
        display_cache, get = web_listing
        self._display(display_cache).tick()

        plugins, runtime = get()

        assert runtime["status"] == "live"
        clock, weather = plugins["clock"], plugins["weather"]
        assert (clock["loaded"], clock["state"], clock["error_info"]) == (True, "enabled", None)
        # The display runs 1.0.0; 1.1.0 is on disk (an update awaiting restart).
        assert clock["version"] == "1.1.0" and clock["loaded_version"] == "1.0.0"
        assert clock["loaded_at"] == 123.0
        assert weather["loaded"] is False and weather["state"] == "error"
        assert weather["error_info"]["type"] == "RuntimeError"
        assert "abc123" not in weather["error_info"]["message"]

    def test_unknown_before_the_display_has_published(self, web_listing):
        _, get = web_listing
        plugins, runtime = get()
        assert runtime["status"] == "unknown"
        assert all(p["loaded"] is None and p["state"] is None and p["error_info"] is None
                   for p in plugins.values())
        # enabled still comes from config.json.
        assert plugins["clock"]["enabled"] is True

    def test_stopped_display_is_not_reported_as_running(self, web_listing):
        display_cache, get = web_listing
        publisher = self._display(display_cache)
        publisher.tick()
        publisher.stop()

        plugins, runtime = get()

        assert runtime["status"] == "stopped"
        assert plugins["clock"]["loaded"] is None

    def test_stale_snapshot_is_not_reported_as_truth(self, web_listing):
        display_cache, get = web_listing
        states = PluginStateManager()
        states.record_loaded("clock", "1.0.0")
        PluginRuntimePublisher(display_cache, states,
                               wall_clock=FakeClock(1_000_000.0)).tick()

        plugins, runtime = get()

        assert runtime["status"] == "stale"
        assert runtime["age_seconds"] > rt.STALE_AFTER
        assert plugins["clock"]["loaded"] is None


class TestDisplayControllerStopsThePublisher:
    def test_cleanup_publishes_stopped(self, monkeypatch):
        # display_manager binds the hardware rgbmatrix module unless
        # EMULATOR=true is set before import (test_initial_update_budget.py).
        monkeypatch.setenv("EMULATOR", "true")
        from src.display_controller import DisplayController
        controller = DisplayController.__new__(DisplayController)
        publisher = MagicMock()
        controller._plugin_runtime_publisher = publisher
        controller.plugin_manager = None
        controller.vegas_coordinator = None
        controller.sync_manager = None
        controller.display_manager = MagicMock()
        controller.cleanup()
        publisher.stop.assert_called_once_with()