From 5b81cca6846112814124f82a6f85645c555429fa Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:55:45 -0400 Subject: [PATCH 01/53] fix(espn): send an identifying User-Agent so ESPN stops returning 403 (#436) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Around 11:00 EDT on 2026-08-04 ESPN's site.api began rejecting the agents this repo sends. Every scoreboard that goes through the shared data sources returned `403 Client Error: Forbidden` — standings, game summaries and scoreboards alike. A device that had been running fine logged 287 ESPN errors in a day. The filter is not the familiar one. Probing site.api across agents and libraries, using requests as the plugins do: bare 'LEDMatrix/1.0' 403 (with or without Accept) browser string 403 (no header rescues it) 'LEDMatrix/1.0 (+https://github.com/...)' 200 requests / urllib / curl defaults 200 So it rejects browser-style strings outright and bare custom tokens, and accepts honest client tokens or an agent that identifies the client and links to it. The instinct to "just send a browser User-Agent" is now exactly backwards — that is the one thing guaranteed to stay blocked. Both call sites here sent a bare token: `LEDMatrix/1.0` in the sports data sources and `LEDMatrix-Common/1.0` in the API helper. The Accept header the data sources already sent does not save it. Both now send an agent carrying the project URL, which also gives ESPN someone to contact rather than an anonymous token to rate-limit. Verified on a live device: ESPN errors went from a steady stream to zero across a restart, with live MLB games fetching again. Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 Co-authored-by: Claude Opus 5 (1M context) --- src/base_classes/data_sources.py | 11 +++++++++-- src/common/api_helper.py | 4 +++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/base_classes/data_sources.py b/src/base_classes/data_sources.py index 777320ea..065ea577 100644 --- a/src/base_classes/data_sources.py +++ b/src/base_classes/data_sources.py @@ -44,9 +44,16 @@ class DataSource(ABC): """Fetch standings for a sport/league.""" def get_headers(self) -> Dict[str, str]: - """Get headers for API requests.""" + """Get headers for API requests. + + The agent carries the project URL deliberately. Around 2026-08-04 ESPN + began returning 403 for bare custom tokens like 'LEDMatrix/1.0' — and + for browser-style strings — while accepting an agent that identifies + the client and links to it. An Accept header alone does not rescue the + bare form when the request goes out through requests. + """ return { - 'User-Agent': 'LEDMatrix/1.0', + 'User-Agent': 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)', 'Accept': 'application/json' } diff --git a/src/common/api_helper.py b/src/common/api_helper.py index 9d9b076f..fa9b7694 100644 --- a/src/common/api_helper.py +++ b/src/common/api_helper.py @@ -56,7 +56,9 @@ class APIHelper: # Default headers self.session.headers.update({ - 'User-Agent': 'LEDMatrix-Common/1.0', + # Identifies the client and links to it: ESPN began 403ing bare + # custom tokens (and browser strings) around 2026-08-04. + 'User-Agent': 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)', 'Accept': 'application/json', 'Accept-Language': 'en-US,en;q=0.9', 'Accept-Encoding': 'gzip, deflate, br', From 2af41c561bfb5c296704bcfbf488f222fd3705df Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:38:11 -0400 Subject: [PATCH 02/53] fix(plugins): make update scheduling atomic so update() cannot run twice at once (#437) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(plugins): make update scheduling atomic so update() cannot run twice at once Closes #401. `run_scheduled_updates()` decided whether to update a plugin with a check-then-act sequence: `can_execute()` and `set_state(RUNNING)` were separate calls with nothing between them, so two scheduler threads could both observe ENABLED and both go on to call the same plugin's `update()`. `update_all_plugins()` had the identical pattern. Two schedulers really do run at once. The render loop calls `_tick_plugin_updates()`, and Vegas mode fires its own `vegas-plugin-tick` daemon thread that is never joined when `VegasModeCoordinator.play()` returns — a slow `update()` still in flight overlaps the next tick from the main loop. A plugin running `update()` twice concurrently is unsafe unless it happens to be reentrant; shared mutable state, a non-thread-safe HTTP session or cache all break. The async path was already covered by the `_pending_lock` dedup in `_enqueue_update`, so the live exposure was the synchronous kill-switch path and `update_all_plugins()`. Both now claim the plugin through `_reserve_for_update()`, which holds one lock across the eligibility check, the due-time check and the RUNNING transition — and nothing more. Holding it across `execute_update()` would serialize slow plugins behind each other and reintroduce the render stall the async worker exists to avoid. The due-time check moved inside the lock deliberately. Left outside, a thread that had already decided "due" could claim the plugin the instant the winner finished, running `update()` twice within one interval. Two supporting changes fall out of it: - `_enqueue_update()` no longer sets RUNNING (the reservation did), and hands the reservation back if the pending-dedup ever fires. Otherwise a reserved-but-unqueued plugin would sit in RUNNING with nothing left to release it, and `can_execute()` would refuse it forever. - `_finish()` now clears the pending entry *before* flipping the state back to ENABLED. The old order left a window where a scheduler saw ENABLED, reserved the plugin, then had its enqueue silently dropped by the dedup — harmless as a missed tick before, a stuck plugin once a reservation is involved. Regression suite added and enrolled in CI, along with test_async_plugin_updates.py which was not previously run there. The overlap tests delay `can_execute()` to hold every thread inside the check-then-act gap: the real window is a couple of bytecodes wide, so a plain hammering test passes against the unfixed scheduler and proves nothing. With that delay the suite reports `update() ran 8x concurrently` on both affected paths before the fix, and passes after. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 * test: fix a race in the reservation suite's own wait loops `test_async_path_never_overlaps` failed in CI with "update() ran 0x concurrently" — the test's bug, not the scheduler's. It polled `plugin._active` to wait for the update to finish, but before the worker picks the item up nothing is active yet, so the loop fell straight through and asserted on a plugin that had never run. Both async waits now key on `update_calls >= 1` as well, so they wait for an update to have started *and* finished. The stranded-state test gets the same guard for a second reason: ENABLED is also the starting state, so without it that assertion passes vacuously on a plugin that was never scheduled. Verified over 12 consecutive local runs, 12 passed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 * fix(plugins): roll back the claim when dispatch fails _enqueue_update() reserved the plugin and added it to the pending set, then started the worker and queued the item. Thread.start() raises RuntimeError when the OS refuses a new thread — not hypothetical on a Pi under memory or thread pressure — and nothing is queued at that point to release the plugin. It stayed RUNNING with a stale pending entry, so can_execute() refused it for the rest of the process, and the exception escaped run_scheduled_updates() and skipped every remaining plugin in that tick. That is the same stranded-RUNNING failure the reservation was introduced to prevent, just reached through the dispatch rather than the dedup, so it is handled the same way: discard the pending entry, hand the reservation back, log the cause. Swallowed rather than raised so one plugin failing to queue cannot abort the others' turn. Both new tests fail against the un-rolled-back version — the second on the escaping RuntimeError itself — and pass with it. 74 tests across the reservation, async-update, plugin-system, health, Vegas-adapter and controller-toggle suites still pass. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WexvwNDtWLVymGVqKD7BGk --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/test.yml | 4 +- src/plugin_system/plugin_manager.py | 138 +++++++--- test/test_plugin_update_reservation.py | 335 +++++++++++++++++++++++++ 3 files changed, 448 insertions(+), 29 deletions(-) create mode 100644 test/test_plugin_update_reservation.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0000c47f..2c29fd44 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,4 +80,6 @@ jobs: test/test_version_consistency.py \ test/test_plugin_compatibility_gate.py \ test/test_install_preserves_existing.py \ - test/test_core_owned_config_keys.py + test/test_core_owned_config_keys.py \ + test/test_async_plugin_updates.py \ + test/test_plugin_update_reservation.py diff --git a/src/plugin_system/plugin_manager.py b/src/plugin_system/plugin_manager.py index 50d6419e..8bfc13ab 100644 --- a/src/plugin_system/plugin_manager.py +++ b/src/plugin_system/plugin_manager.py @@ -113,6 +113,15 @@ class PluginManager: self._update_queue: "queue.Queue[Optional[Tuple[str, float]]]" = queue.Queue() self._pending_updates: set = set() self._pending_lock = threading.Lock() + # Serializes the "is this plugin eligible?" -> "claim it (RUNNING)" + # transition. Two schedulers run concurrently in practice — the render + # loop's _tick_plugin_updates() and Vegas mode's vegas-plugin-tick + # daemon thread, which is never joined — so without this both can + # observe ENABLED and both call update() on the same plugin. Held only + # across the check and the state transition, never across update() + # itself: that would serialize slow plugins behind each other and + # reintroduce the stall the async worker exists to avoid. + self._reservation_lock = threading.Lock() self._plugin_locks: Dict[str, threading.Lock] = {} self._plugin_locks_guard = threading.Lock() self._update_worker: Optional[threading.Thread] = None @@ -808,25 +817,69 @@ class PluginManager: if self.health_tracker and self.health_tracker.should_skip_plugin(plugin_id): continue - # Check if plugin can execute - if not self.state_manager.can_execute(plugin_id): - continue - interval = self._get_plugin_update_interval(plugin_id, plugin_instance) if interval is None: continue - with self._plugin_last_update_lock: - last_update = self.plugin_last_update.get(plugin_id, 0.0) + # Eligibility check, due check and the RUNNING transition happen + # together, so a concurrent scheduler cannot claim the same plugin. + if not self._reserve_for_update(plugin_id, current_time, interval): + continue - if last_update == 0.0 or (current_time - last_update) >= interval: - if self._synchronous_updates: - # Kill-switch path: the original inline execution - # (blocks the caller until update() completes/times out) - self.state_manager.set_state(plugin_id, PluginState.RUNNING) - self._execute_update_now(plugin_id, plugin_instance, current_time) - else: - self._enqueue_update(plugin_id, current_time) + if self._synchronous_updates: + # Kill-switch path: the original inline execution + # (blocks the caller until update() completes/times out) + self._execute_update_now(plugin_id, plugin_instance, current_time) + else: + self._enqueue_update(plugin_id, current_time) + + def _reserve_for_update( + self, + plugin_id: str, + current_time: Optional[float] = None, + interval: Optional[float] = None, + ) -> bool: + """Atomically claim a plugin for update, returning True if we won it. + + can_execute() and the RUNNING transition have to happen under one lock. + As two separate calls, two scheduler threads can both see ENABLED and + both go on to run the same plugin's update() concurrently — unsafe for + any plugin that isn't reentrant (shared mutable state, a non-thread-safe + HTTP session or cache). + + The due-time check is inside the lock too. Leaving it outside would let + a second thread that had already decided "due" claim the plugin the + instant the first finished, running update() twice in one interval. + + Args: + plugin_id: Plugin to claim. + current_time: Now, for the due check. Omit to skip that check. + interval: Seconds between updates. Omit to skip the due check. + + Returns: + True if this caller reserved the plugin and must dispatch it, + False if it is ineligible, not yet due, or already claimed. + """ + with self._reservation_lock: + if not self.state_manager.can_execute(plugin_id): + return False + + if current_time is not None and interval is not None: + with self._plugin_last_update_lock: + last_update = self.plugin_last_update.get(plugin_id, 0.0) + if last_update != 0.0 and (current_time - last_update) < interval: + return False + + self.state_manager.set_state(plugin_id, PluginState.RUNNING) + return True + + def _release_reservation(self, plugin_id: str) -> None: + """Hand a claimed plugin back when it never got dispatched. + + Without this a plugin reserved but not queued would sit in RUNNING + forever, and can_execute() would refuse it on every later tick. + """ + self.state_manager.set_state(plugin_id, PluginState.ENABLED) def get_plugin_lock(self, plugin_id: str) -> threading.Lock: """Per-plugin lock keeping update() and display() mutually exclusive. @@ -843,16 +896,40 @@ class PluginManager: return lock def _enqueue_update(self, plugin_id: str, scheduled_time: float) -> None: - """Queue a due update for the background worker (dedup while pending).""" + """Queue an already-reserved update for the background worker. + + The caller has reserved the plugin (RUNNING), which is what blocks + re-entry and shows the truthful state in the web UI while the item + waits its turn. The pending set stays as a second line of defence; if + it ever fires the reservation has to be handed back, or the plugin + would sit in RUNNING with nothing queued to release it. + """ with self._pending_lock: if plugin_id in self._pending_updates: + self.logger.warning( + "Plugin %s reserved for update but already queued; " + "releasing the reservation", plugin_id) + self._release_reservation(plugin_id) return self._pending_updates.add(plugin_id) - # RUNNING is set at enqueue time so can_execute() blocks re-entry and - # the web UI shows the truthful state while the item waits its turn. - self.state_manager.set_state(plugin_id, PluginState.RUNNING) - self._ensure_update_worker() - self._update_queue.put((plugin_id, scheduled_time)) + try: + self._ensure_update_worker() + self._update_queue.put((plugin_id, scheduled_time)) + except Exception as exc: # pylint: disable=broad-except + # Thread.start() raises RuntimeError when the OS refuses a new + # thread — a real condition on a Pi under memory pressure. Nothing + # is queued to release the plugin at that point, so the claim has to + # be undone here, or it sits in RUNNING with nothing to clear it and + # can_execute() refuses it for the rest of the process. Swallowed + # rather than raised so the remaining plugins in this tick still get + # their turn. + self.logger.error( + "Could not queue update for plugin %s (%s: %s); releasing the " + "reservation so the next tick can retry", + plugin_id, type(exc).__name__, exc, exc_info=True) + with self._pending_lock: + self._pending_updates.discard(plugin_id) + self._release_reservation(plugin_id) def _ensure_update_worker(self) -> None: if self._update_worker is not None and self._update_worker.is_alive(): @@ -937,6 +1014,14 @@ class PluginManager: return finished['done'] = True try: + # Drop the queue reservation *before* the state goes back to + # ENABLED. The other order leaves a window where a scheduler + # sees ENABLED, reserves the plugin, then finds it still in + # _pending_updates -- the enqueue is dropped and the plugin + # would sit in RUNNING with nothing left to release it. + if lock is not None: + with self._pending_lock: + self._pending_updates.discard(plugin_id) if success: with self._plugin_last_update_lock: self.plugin_last_update[plugin_id] = scheduled_time @@ -949,8 +1034,6 @@ class PluginManager: finally: if lock is not None: lock.release() - with self._pending_lock: - self._pending_updates.discard(plugin_id) if lock is None: # Synchronous / no-lock path: unchanged behavior. @@ -1041,13 +1124,12 @@ class PluginManager: if not hasattr(plugin_instance, "update"): continue - # Check if plugin can execute - if not self.state_manager.can_execute(plugin_id): + # Eligibility check and the RUNNING transition together, so a + # concurrent scheduler cannot claim the same plugin (see + # _reserve_for_update). + if not self._reserve_for_update(plugin_id): continue - - # Update state to RUNNING - self.state_manager.set_state(plugin_id, PluginState.RUNNING) - + try: success = self.plugin_executor.execute_update(plugin_instance, plugin_id) if success: diff --git a/test/test_plugin_update_reservation.py b/test/test_plugin_update_reservation.py new file mode 100644 index 00000000..90ea62ae --- /dev/null +++ b/test/test_plugin_update_reservation.py @@ -0,0 +1,335 @@ +"""Plugin update scheduling must be atomic (issue #401). + +`run_scheduled_updates()` decided whether to update a plugin with a +check-then-act sequence: `can_execute()` and `set_state(RUNNING)` were separate +calls with nothing between them, so two scheduler threads could both observe +ENABLED and both go on to call the same plugin's `update()`. + +Two schedulers really do run at once. The render loop calls +`_tick_plugin_updates()`, and Vegas mode fires its own `vegas-plugin-tick` +daemon thread every few seconds; that thread is never joined when +`VegasModeCoordinator.play()` returns, so a slow `update()` still in flight can +overlap the next tick from the main loop. + +A plugin whose `update()` runs twice at once is unsafe unless it happens to be +reentrant — shared mutable state, a non-thread-safe HTTP session or cache all +break. These tests pin the reservation that closes it, and the state +bookkeeping that has to survive it: a plugin reserved but never dispatched must +not be stranded in RUNNING, because `can_execute()` would then refuse it +forever. +""" + +import os +import sys +import threading +import time + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from src.plugin_system.plugin_manager import PluginManager # noqa: E402 +from src.plugin_system.plugin_state import PluginState # noqa: E402 + + +class OverlapDetectingPlugin: + """Records the high-water mark of concurrent update() calls.""" + + def __init__(self, update_seconds=0.25): + self.enabled = True + self.update_seconds = update_seconds + self.update_calls = 0 + self.max_concurrent = 0 + self._active = 0 + self._guard = threading.Lock() + + def update(self): + with self._guard: + self._active += 1 + self.update_calls += 1 + self.max_concurrent = max(self.max_concurrent, self._active) + try: + time.sleep(self.update_seconds) + finally: + with self._guard: + self._active -= 1 + return True + + def display(self, force_clear=False): + return True + + +@pytest.fixture +def pm(tmp_path): + manager = PluginManager(plugins_dir=str(tmp_path), config_manager=None, + display_manager=None, cache_manager=None) + yield manager + manager.stop_update_worker() + + +def _install(pm, plugin, plugin_id="racy-plugin", interval=0.01): + pm.plugins[plugin_id] = plugin + pm._update_interval_cache[plugin_id] = interval + pm.state_manager.set_state(plugin_id, PluginState.ENABLED) + return plugin_id + + +def _widen_check_then_act_window(pm, delay=0.02): + """Hold every scheduler thread inside the eligibility check at once. + + The real gap between can_execute() and the RUNNING transition is a couple + of bytecodes wide, so a plain thread race under the GIL almost never lands + in it — an unfixed scheduler looks correct in a test that just hammers it. + Delaying the check reproduces the interleaving that Vegas's tick thread and + the render loop actually produce when a slow update() overlaps the next + tick, and it is what makes these tests fail without the reservation. + + Once the check and the transition are under one lock, the delay only + serializes the schedulers: the losers observe RUNNING and back off. + """ + real_can_execute = pm.state_manager.can_execute + + def slow_can_execute(plugin_id): + result = real_can_execute(plugin_id) + time.sleep(delay) + return result + + pm.state_manager.can_execute = slow_can_execute + + +def _hammer(target, threads=8, rounds=1): + """Run `target` on N threads released simultaneously by a barrier.""" + errors = [] + barrier = threading.Barrier(threads) + + def runner(): + try: + barrier.wait(timeout=5) + for _ in range(rounds): + target() + except Exception as exc: # noqa: BLE001 - surfaced by the assertion + errors.append(exc) + + workers = [threading.Thread(target=runner) for _ in range(threads)] + for worker in workers: + worker.start() + for worker in workers: + worker.join(timeout=30) + assert not errors, f"worker raised: {errors[0]!r}" + + +class TestReservationAtomicity: + """The check-and-claim itself, independent of any dispatch path.""" + + def test_only_one_caller_wins_the_reservation(self, pm): + plugin_id = _install(pm, OverlapDetectingPlugin()) + _widen_check_then_act_window(pm) + wins = [] + lock = threading.Lock() + + def claim(): + if pm._reserve_for_update(plugin_id): + with lock: + wins.append(threading.current_thread().name) + + _hammer(claim, threads=16) + assert len(wins) == 1, ( + f"{len(wins)} threads reserved the same plugin concurrently; " + "the eligibility check and the RUNNING transition are not atomic") + assert pm.state_manager.get_state(plugin_id) == PluginState.RUNNING + + def test_reservation_refused_while_running(self, pm): + plugin_id = _install(pm, OverlapDetectingPlugin()) + assert pm._reserve_for_update(plugin_id) is True + assert pm._reserve_for_update(plugin_id) is False + + def test_reservation_can_be_handed_back(self, pm): + plugin_id = _install(pm, OverlapDetectingPlugin()) + assert pm._reserve_for_update(plugin_id) is True + pm._release_reservation(plugin_id) + assert pm.state_manager.get_state(plugin_id) == PluginState.ENABLED + assert pm._reserve_for_update(plugin_id) is True, \ + "a released reservation must be claimable again" + + def test_due_check_is_inside_the_reservation(self, pm): + """Two threads that both decided 'due' must not both get a turn. + + With the due check outside the lock, the loser of the race could claim + the plugin the moment the winner finished, running update() twice + inside one interval. + """ + plugin_id = _install(pm, OverlapDetectingPlugin(), interval=60.0) + now = time.time() + assert pm._reserve_for_update(plugin_id, now, 60.0) is True + pm.plugin_last_update[plugin_id] = now + pm._release_reservation(plugin_id) + assert pm._reserve_for_update(plugin_id, now, 60.0) is False, \ + "plugin updated just now must not be due again" + + +class TestNoConcurrentUpdate: + """The end-to-end invariant the issue is actually about.""" + + def test_synchronous_path_never_overlaps(self, pm): + """The kill-switch path ran update() inline with no dedup at all.""" + pm._synchronous_updates = True + plugin = OverlapDetectingPlugin(update_seconds=0.25) + _install(pm, plugin) + _widen_check_then_act_window(pm) + + _hammer(pm.run_scheduled_updates, threads=8) + + assert plugin.max_concurrent == 1, ( + f"update() ran {plugin.max_concurrent}x concurrently on the " + "synchronous path") + + def test_update_all_plugins_never_overlaps(self, pm): + plugin = OverlapDetectingPlugin(update_seconds=0.25) + _install(pm, plugin) + _widen_check_then_act_window(pm) + + _hammer(pm.update_all_plugins, threads=8) + + assert plugin.max_concurrent == 1, ( + f"update() ran {plugin.max_concurrent}x concurrently via " + "update_all_plugins()") + + def test_async_path_never_overlaps(self, pm): + plugin = OverlapDetectingPlugin(update_seconds=0.2) + plugin_id = _install(pm, plugin) + + _hammer(pm.run_scheduled_updates, threads=8, rounds=3) + + # Wait for an update to have both started and finished. Polling only + # `_active` races the worker: before it picks the item up nothing is + # active yet, so the loop would fall straight through and assert on a + # plugin that never ran. + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if plugin.update_calls >= 1 and plugin._active == 0: + break + time.sleep(0.05) + + assert plugin.update_calls >= 1, "no update ran on the async path" + assert plugin.max_concurrent == 1, ( + f"update() ran {plugin.max_concurrent}x concurrently on the " + "async path") + assert plugin_id in pm.plugins + + +class TestNoStrandedState: + """A reservation that is never dispatched must not wedge the plugin.""" + + def test_plugin_returns_to_enabled_after_async_updates(self, pm): + plugin = OverlapDetectingPlugin(update_seconds=0.1) + plugin_id = _install(pm, plugin) + + _hammer(pm.run_scheduled_updates, threads=6, rounds=2) + + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if (plugin.update_calls >= 1 + and pm.state_manager.get_state(plugin_id) == PluginState.ENABLED + and not pm._pending_updates): + break + time.sleep(0.05) + + # ENABLED is also the starting state, so without this the assertion + # below would pass on a plugin that never got scheduled at all. + assert plugin.update_calls >= 1, "no update ran; the state assertion would be vacuous" + assert pm.state_manager.get_state(plugin_id) == PluginState.ENABLED, \ + "plugin stranded in RUNNING; can_execute() would refuse it forever" + assert not pm._pending_updates, "pending set not drained" + + def test_pending_cleared_before_state_reset(self, pm): + """Invariant: never-RUNNING implies never-pending. + + _finish() used to clear the pending entry *after* flipping the state + back to ENABLED. In that window a scheduler could reserve the plugin + and then have its enqueue dropped by the pending-dedup. + """ + plugin = OverlapDetectingPlugin(update_seconds=0.05) + plugin_id = _install(pm, plugin) + + violations = [] + stop = threading.Event() + + def watcher(): + while not stop.is_set(): + state = pm.state_manager.get_state(plugin_id) + if state != PluginState.RUNNING and plugin_id in pm._pending_updates: + violations.append(state) + time.sleep(0.001) + + thread = threading.Thread(target=watcher, daemon=True) + thread.start() + try: + for _ in range(15): + pm.run_scheduled_updates() + time.sleep(0.05) + finally: + stop.set() + thread.join(timeout=5) + + assert not violations, ( + f"{len(violations)} sample(s) saw a non-RUNNING plugin still in " + "_pending_updates") + + +class TestDispatchFailure: + """A reservation must survive the dispatch itself failing. + + _enqueue_update() claims the plugin, adds it to the pending set, and only + then starts the worker and queues the item. threading.Thread.start() raises + RuntimeError when the OS refuses a new thread — not hypothetical on a Pi + under memory or thread pressure. Nothing is queued to release the plugin at + that point, so without an explicit rollback it stays RUNNING with a stale + pending entry and can_execute() refuses it for the rest of the process. + """ + + def test_reservation_released_when_the_worker_cannot_start(self, pm): + plugin_id = _install(pm, OverlapDetectingPlugin()) + + def refuse_to_start(): + raise RuntimeError("can't start new thread") + + pm._ensure_update_worker = refuse_to_start + + assert pm._reserve_for_update(plugin_id) is True + pm._enqueue_update(plugin_id, time.time()) + + assert pm.state_manager.get_state(plugin_id) == PluginState.ENABLED, \ + "plugin left in RUNNING after a failed dispatch; can_execute() " \ + "would refuse it forever" + assert plugin_id not in pm._pending_updates, \ + "stale pending entry would make the next enqueue hit the dedup" + assert pm._reserve_for_update(plugin_id) is True, \ + "plugin must be claimable again on the next tick" + + def test_dispatch_failure_does_not_abort_the_rest_of_the_tick(self, pm): + """One plugin failing to queue must not skip the others in that tick.""" + first = OverlapDetectingPlugin() + second = OverlapDetectingPlugin() + _install(pm, first, plugin_id="plugin-a") + _install(pm, second, plugin_id="plugin-b") + + calls = [] + real_ensure = pm._ensure_update_worker + + def fail_first_only(): + calls.append(1) + if len(calls) == 1: + raise RuntimeError("can't start new thread") + return real_ensure() + + pm._ensure_update_worker = fail_first_only + + pm.run_scheduled_updates() # must not propagate the RuntimeError + + assert len(calls) == 2, ( + "run_scheduled_updates() stopped after the failing plugin; the " + "exception escaped the enqueue") + for plugin_id in ("plugin-a", "plugin-b"): + assert pm.state_manager.get_state(plugin_id) in ( + PluginState.ENABLED, PluginState.RUNNING), \ + f"{plugin_id} left in an unexpected state" From d9683e28be16d509f4ce851336fe81dce7dffb7e Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:04:23 -0400 Subject: [PATCH 03/53] Codebase audit: fix shipping bugs, remove verified-dead code, repair doc drift, add regression guards (#438) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(web): implement delete_cached so the font catalog cache actually invalidates api_v3.py's font upload/delete handlers import delete_cached from web_interface.cache, but the function was never defined. The surrounding except ImportError silently swallowed the failure, so the fonts_catalog cache entry survived uploads/deletes and newly uploaded fonts did not appear until the TTL expired or the service restarted. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * fix(web): remove dead weather/stocks partial routes that returned 500 The partial dispatcher still routed 'weather' and 'stocks' to loaders rendering v3/partials/weather.html and stocks.html — templates that no longer exist since weather and stocks became store plugins. Requesting either partial raised TemplateNotFound, which the catch-all turned into a 500. No template or JS references these partials (the only 'weather' hit in the front end is a plugin-store category filter option), so the branches and both loader functions are removed; unknown partials now fall through to the existing 404 handler. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * fix(deps): align contradictory psutil/Flask-Limiter/freetype-py pins requirements.txt's optional-install comment recommended psutil>=5.9,<6.0 while web_interface/requirements.txt hard-requires >=6.0,<7.0 — anyone following the comment ends up with an unsatisfiable pair. The comment now recommends the same range the web interface requires (all psutil APIs used — Process, boot_time, cpu_percent, disk_usage, virtual_memory — are stable in 6.x). Flask-Limiter gains the same <4.0 cap in both files and freetype-py the same >=2.5.1 floor. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * fix(config): add template keys the code already reads display.hardware gains pixel_mapper_config, row_address_type, multiplexing and panel_type (read at display_manager.py with these exact fallbacks — users on non-standard panels previously had no way to discover them from the template). vegas_scroll gains frame_based_scrolling and scroll_delay, the only two of its 27 keys the template omitted (read in src/vegas_mode/config.py). plugin_system gains development_mode, which the web UI reads and writes but the template never declared. Every added value is byte-identical to the code-side .get() fallback, so ConfigManager._migrate_config() merging these keys into existing user configs cannot change behavior on any installed device. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * fix(scripts): repair broken sys.path setup in utility scripts clear_cache.py and download_nba_logos.py pointed sys.path at a 'src' directory relative to the script's own folder (scripts/utils/src and scripts/src — neither exists), so both crashed on import; they now insert the project root and import via the src package like the other scripts. debug_web_manual.py resolved 'project root' to scripts/debug/ instead of two levels up. fix_nhl_cache.sh is removed: it used Python docstring syntax in a bash script and invoked clear_nhl_cache.py, which does not exist anywhere in the repo — it cannot ever have worked in its current location. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * docs: correct stale file:line references and the loader-fallback contradiction CLAUDE.md and .cursorrules disagreed about plugin-directory fallback behavior; the code (SchemaManager.get_schema_path) probes plugins/ BEFORE plugin-repos/, and the main discovery path has no fallback at all — both files now describe the real behavior, preferring symbol names over line numbers so the references rot slower. REST_API_REFERENCE.md pointed at app.py:144/:607 for mounts that live at :199/:799 and counted 92 routes where there are 94. PLUGIN_ARCHITECTURE_SPEC.md's historical banner gains a note that its example imports (src/plugin_system/base_classes/*_plugin.py) never shipped — the real base classes are src.base_classes.sports.SportsCore and src.base_classes.hockey.Hockey. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * docs: fix broken links, phantom script references, and stale CI description Repairs every broken relative link in active docs (targets renamed or archived long ago: PLUGIN_DEVELOPMENT.md -> PLUGIN_DEVELOPMENT_GUIDE.md, API_REFERENCE.md -> REST_API_REFERENCE.md, PLUGIN_STORE_USER_GUIDE.md -> PLUGIN_STORE_GUIDE.md, plugin_docs/ dir, TROUBLESHOOTING_QUICK_START.md, and MIGRATION_GUIDE's README link that silently resolved to the docs index instead of the project README). Replaces commands invoking scripts that do not exist (scripts/update_stats.py, validate_registry.py, check_updates.py, fix_permissions.sh) with the real tooling, and rewrites HOW_TO_RUN_TESTS.md's CI section, which described a security-audit workflow that was never committed and a pytest workflow 'queued to land' that landed long ago as test.yml. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * docs: complete the docs index and refresh the web interface file tree docs/README.md's own policy says every page must be linked from the index, yet five weren't — including the entire skin system (SKIN_SYSTEM.md, CREATING_SKINS.md), ADAPTIVE_LAYOUT.md, plugin-safety-harness.md and SPORTS_UNIFICATION.md. Each is now listed in the section it belongs to, and PLUGIN_ARCHITECTURE_SPEC.md is marked historical in the index (the doc itself already carries the banner). web_interface/README.md's static/v3 tree showed only app.css/app.js; it now reflects the actual contents. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * chore: remove dead modules confirmed unused in-repo and across all store plugins - src/common/cli.py: imports a 'ledmatrix_common' package that exists nowhere (not in this repo, any requirements file, or the plugin monorepo), so it cannot ever have run; its README section claimed scripts/dev/* used it, which was also untrue. - src/web_interface/logging_config.py: zero callers — the web app uses web_interface/logging_config.py (a different module), and nothing imports the src copy. - handle_errors decorator in src/web_interface/error_handler.py: zero call sites (the module's response helpers stay — they are used). - ConfigManager.get_clock_config(): reads a 'clock' config key that no longer exists anywhere; only caller was its own unit test. Deliberately kept despite zero in-repo callers: DisplayError, src/common/config_helper.py and display_helper.py — all documented as plugin-facing API (docs/PLUGIN_ERROR_HANDLING.md, src/common/README.md), and third-party plugins outside the official monorepo cannot be enumerated. Verified against a fresh clone of ledmatrix-plugins (43 plugins): zero references to any removed symbol. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * chore: remove manager-era NBA test files and one-off debug scripts The four test_nba_*.py files imported nba_managers, leaderboard_manager and odds_manager — top-level modules deleted when sports displays became plugins — inside try/except blocks that swallowed the ImportError, so they passed while exercising nothing. test_nba_data_structure.py and debug_nba_api.py (a diagnostic script living in test/) made live ESPN API calls rather than testing repo code. None were enrolled in CI. scripts/debug/direct_fix_imports.py and check_imports.py were one-shot artifacts that edited/inspected a hardcoded ~/LEDMatrix/web_interface/ app.py to fix an import problem solved long ago; nothing references them. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * chore: remove generate_report.py, which aggregates artifacts of CI jobs that do not exist The script's only function is to merge JSON artifacts (bandit/semgrep/pip-audit/safety/gitleaks results) produced by a security-audit workflow that was never committed — .github/workflows/ has no such jobs, so there is nothing for it to aggregate and no way to run it usefully. Its siblings stay: prove_security.py and audit_plugins.py both run standalone (verified), and .codacy.yml stays because the Codacy service (README badge) reads it server-side without a workflow file. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * fix(web): load the three widget scripts store plugins already declare time-picker.js, file-upload-single.js and plugin-file-manager.js register widgets that installed store plugins reference in their config schemas (countdown uses x-widget: time-picker and file-upload-single; of-the-day uses plugin-file-manager), but base.html never included the scripts. plugin_config.html renders such fields as an empty container that polls LEDMatrixWidgets.get(...) on a 50ms loop forever, so those plugin config fields appeared permanently blank. The audit initially flagged these files as dead code; the monorepo cross-check proved the opposite — they were unreachable, not unused. example-color-picker.js (the documented custom-widget example) gains an explicit warning that including it in base.html would shadow the built-in color-picker widget. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * chore: drop the legacy youtube block from the secrets template No code reads a top-level youtube secrets key: the youtube-stats plugin receives its API key namespaced under its own plugin id (declared via x-secret in its config schema), like every other store plugin. The key survives only in state_reconciliation.py's non-plugin-key exclusion set, which stays — existing installs still carry the key in their generated config_secrets.json, and the exclusion prevents it from being misclassified as a plugin config. New installs simply stop being asked for a YouTube API key they have nowhere to use. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * chore(deps): remove packages nothing imports, declare direct imports, move mypy to test deps Removed from requirements.txt: python-socketio, python-engineio, websockets, websocket-client — zero imports anywhere in this repo, and the one store plugin that needs Socket.IO (ledmatrix-music) declares it in its own requirements.txt, which the plugin store installs. Removed the same quartet plus timezonefinder, geopy, google-auth-oauthlib, google-auth-httplib2, google-api-python-client, unidecode, icalevents, python-dateutil, flask-wtf and the werkzeug pin from web_interface/requirements.txt — all leftovers from the deleted built-in weather/calendar/music displays (flask-wtf was doubly dead: app.py explicitly disables CSRF and sets csrf=None). scripts/ install_dependencies_apt.py, which mirrors these lists for the first-time installer, drops the same packages. Added: urllib3 (imported directly in four core modules), jinja2 and markupsafe (imported directly in pages_v3.py) — previously reachable only as transitives. mypy moves from runtime requirements to requirements-test.txt. Verified in a fresh venv: all four requirements files co-install, pip check is clean, the full CI-enrolled suite (907 tests) and a Flask boot smoke pass with the trimmed dependency set. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * refactor: single canonical DateTimeEncoder src/cache_manager.py and src/cache/disk_cache.py each defined an identical DateTimeEncoder (datetime -> ISO-8601). The disk_cache copy is the only one actually used for serialization; cache_manager now re-exports it instead of defining a twin, so the two can never silently diverge. Import compatibility is preserved — from src.cache_manager import DateTimeEncoder still works and is the same class object. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * docs(code): document deliberate duplicates instead of merging them The audit surfaced several near-duplicate implementations that turned out to be either deliberate forks or behaviorally different — merging any of them would risk changing behavior on installed devices, so each now carries an explicit comment stating the relationship: - VisualDisplayManager: headless fork of DisplayManager; header now lists the ~15 mirrored methods and warns that DisplayManager changes must be mirrored. - normalize_abbreviation: LogoDownloader's version (called directly by nine scoreboard plugins) replaces filesystem-unsafe characters; LogoHelper's strips spaces. Logo filenames on existing installs depend on both behaviors staying put. - The two PluginTestBase classes: the shipped one is plugin-author API, the repo's own richer harness lives in test/plugins/ — now cross-referenced. Also verified (no change needed): ConfigManager's backup/rollback methods genuinely delegate to AtomicConfigManager, and SportsCore already delegates _read_bdf_native_size to FontManager. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * docs: add a unified configuration reference There was no single place documenting what lives in config.json — display.* keys were scattered across README sections, vegas_scroll lived in ADVANCED_FEATURES.md, and dim_schedule, display.double_sided, sync.follower_position, plugin_system.development_mode and the four newly-templated hardware keys were documented nowhere. CONFIG_REFERENCE.md now lists every template key plus the code-read-only keys, each with type, default, and the code location that reads it, and explains the secrets file's plugin-id namespacing. Linked from the docs index. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * docs: bring the README's feature tour into the plugin era The Core Features section still presented clock/weather/sports/stocks/ music displays as built into the project, when all of them are store plugins installed from the ledmatrix-plugins monorepo — only starlark-apps and web-ui-info ship in this repo. The intro now says so (the showcase itself is unchanged; those are real displays available in the store). The display_durations reference drops its built-in-calendar example in favor of plugin-id keys, and the Configuration section links the new CONFIG_REFERENCE.md. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * docs: archive the custom-icons status report, cross-link config docs, document assets/ PLUGIN_CUSTOM_ICONS_FEATURE.md was a 'What Was Implemented' status report duplicating the actual guide (PLUGIN_CUSTOM_ICONS.md) — moved to docs/archive/ per the docs index's own policy. The overlapping plugin-config docs keep their content but PLUGIN_CONFIG_ARCHITECTURE.md now states up front which doc is canonical for which purpose. assets/README.md is new and load-bearing: assets/stocks, weather, news_logos and broadcast_logos have zero references in this repo's code, which makes them look deletable — but store plugins (ledmatrix-stocks, ledmatrix-weather, news, odds-ticker) resolve those exact paths at runtime against the install directory. The README records that evidence so a future cleanup doesn't break installed plugins. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * test: add regression guards for the bug classes fixed in this PR Three lightweight static checks, all enrolled in CI's unit-test allowlist along with the new web-cache test: - test_template_targets.py: every literal render_template() target must exist (would have caught the weather/stocks partial 500s at commit time). - test_widget_scripts.py: every widget JS file must be script-included in base.html or explicitly allowlisted with a reason (would have caught the unloaded time-picker/file-upload-single/plugin-file-manager widgets), and allowlisted files must NOT be included (prevents the example widget from shadowing the real color-picker). - test_doc_links.py: relative markdown links in active docs must resolve (docs/archive/ exempt). Each guard was verified to fail against the pre-PR tree and pass now. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * fix(deps): restore the werkzeug version floor Commit 1ec22db removed the werkzeug>=3.1.6,<4.0.0 pin along with the genuinely-unused packages, but this one was a version floor on Flask's transitive dependency, not a phantom: Flask 3.1.3 itself only requires werkzeug>=3.1.0, so dropping the pin let fresh installs resolve 3.1.0-3.1.5. Restored with a comment explaining why it exists. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * fix(web): route pixel_mapper_config into display.hardware; guard time-picker registration pixel_mapper_config was the only display.hardware key absent from both the display_fields detection allowlist and the hardware write loop in the settings save path. No form posts it today, but if one ever did the key would fall through to the generic handler and land at the TOP level of config.json — where state_reconciliation would mistake it for a missing plugin id and loop auto-repair attempts (the failure class the 'github'/'youtube' exclusion comment documents). It now round-trips into display.hardware like its siblings. time-picker.js gains the same LEDMatrixWidgets-undefined guard its two sibling widgets already have; correct today only via defer ordering. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * chore: align installer leftovers with the dependency cleanup first_time_install.sh's fallback secrets heredoc (used only when the template is missing) still wrote the legacy youtube block — now matches the template (github only). install_dependencies_apt.py drops the IMPORT_NAME_MAP entries for packages no longer in its install lists and a stale google-api reference in a docstring. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SXb4mKcAkVaxkeTb3YnAdr * fix: address CodeRabbit review findings Verified each finding against the code; fixes for the valid ones: - install_dependencies_apt.py: the installer listed 'freetype', but the declared dependency is freetype-py — an apt miss would pip-install the wrong PyPI package. Now installs freetype-py with an import-name mapping (pre-existing bug, surfaced by the review). - api_v3.py: pixel_mapper_config is validated as a string before being saved to display.hardware (JSON callers could previously store an object/list the matrix library can't use). - .cursorrules: the Plugin Loading Process and File Organization sections still said discovery scans plugins/ — now consistent with the corrected overview (configured directory, default plugin-repos/). - README.md: removed the stale '(except the core calendar)' claim — no core calendar exists in src/ — and qualified the plugin inventory (official plugins in the monorepo; third-party from their own repos). - CONFIG_REFERENCE.md: hardware_mapping now shows the code fallback (adafruit-hat-pwm) alongside the template value. - PLUGIN_REGISTRY_SETUP_GUIDE.md: check_plugin.py takes --plugin, not a positional id. - scripts/fix_perms/fix_*.sh: exec bits set so the documented 'sudo ./...' invocations work. - Guard tests hardened: template guard now catches multi-line render_template() calls; widget guard parses actual + + + @@ -1010,7 +1013,11 @@ - + +
+
+

Branch

+

Choose which branch this pi follows. Switching attaches tracking, so Pull Latest works afterwards.

+
+
+ + +
+
+ +
@@ -467,6 +485,14 @@
`; } + if (d.upstream) { + html += `

tracking ${escHtml(d.upstream)}

`; + } else if (d.can_pull) { + html += `

No upstream set; Pull Latest will use origin/${escHtml(d.branch || '')} and set it.

`; + } else { + html += `

No upstream and no matching branch on origin — Pull Latest cannot run. Switch to a branch that exists on the remote.

`; + } + if (d.remote_url) { html += `

${escHtml(d.remote_url)}

`; } @@ -479,6 +505,84 @@ }); } + // ── branch picker ───────────────────────────────────────────────────── + // A pi can end up on a branch with no tracking information (checked out by + // name, restored from a backup), where `git pull` refuses to run. Being + // able to see and change the branch from here beats needing SSH. + + function loadBranches() { + const sel = document.getElementById('branch-select'); + if (!sel) return; + + fetch('/api/v3/system/git-branches') + .then(r => r.ok ? r.json() : r.json().then(d => Promise.reject(d.message || `HTTP ${r.status}`))) + .then(d => { + if (d.status === 'error') { + sel.innerHTML = ``; + sel.disabled = true; + return; + } + sel.innerHTML = ''; + const add = (name, suffix) => { + const o = document.createElement('option'); + o.value = name; + o.textContent = name + (suffix || ''); + if (name === d.current) o.selected = true; + sel.appendChild(o); + }; + (d.local || []).forEach(b => add(b, b === d.current ? ' (current)' : '')); + // Remote-only branches are checked out on demand. + (d.remote_only || []).forEach(b => add(b, ' (remote)')); + if (!sel.options.length) add('', 'no branches found'); + }) + .catch(err => { + sel.innerHTML = ``; + sel.disabled = true; + }); + } + + window.checkoutBranch = function(stash) { + const sel = document.getElementById('branch-select'); + const branch = sel && sel.value; + if (!branch) return; + + setBusy('btn-checkout-branch', true); + const el = document.getElementById('result-checkout-branch'); + if (el) el.classList.add('hidden'); + + fetch('/api/v3/system/action', { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({action: 'checkout_branch', branch: branch, stash: !!stash}) + }) + .then(r => r.json().catch(() => ({status: 'error', message: `HTTP ${r.status}`}))) + .then(d => { + const ok = d.status === 'success'; + // Show git's own list of blocking files, then offer the single + // action that clears it. Stashing is never done unasked. + showResult('result-checkout-branch', ok, d.message || '', d.detail || ''); + if (!ok && d.can_retry_with_stash && el) { + const retry = document.createElement('div'); + retry.className = 'mt-2 flex items-center gap-2'; + const label = document.createElement('span'); + label.className = 'text-xs text-gray-700'; + label.textContent = 'Stash these changes and switch anyway?'; + const btn = document.createElement('button'); + btn.className = 'inline-flex items-center px-2 py-1 border border-gray-300 text-xs font-medium rounded text-gray-700 bg-white hover:bg-gray-50'; + btn.textContent = 'Stash and switch'; + btn.onclick = function() { window.checkoutBranch(true); }; + retry.appendChild(label); + retry.appendChild(btn); + el.appendChild(retry); + } + // Both panels describe the checkout, so refresh them together. + loadGitInfo(); + loadBranches(); + }) + .catch(err => showResult('result-checkout-branch', false, String(err))) + .finally(() => setBusy('btn-checkout-branch', false)); + }; + // ── power supply diagnostics panel ──────────────────────────────────────── // Reuses the same SSE stream (window.statsSource, set up in base.html) // that already drives the header badge/banner and Overview card, instead @@ -810,6 +914,7 @@ // Load on first render; HTMX will have already swapped us in by this point. loadGitInfo(); + loadBranches(); // Plugin health: initial load + periodic refresh. Guard against duplicate // timers if this partial is re-swapped in by HTMX; the handler re-resolves From ee59caa57738fb7c66ec7409b56c7ff1ab6f8735 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:17:11 -0400 Subject: [PATCH 08/53] Follow-ups from #441: secret-helper migration, ten more bug fixes, and coverage for every remaining untested module (#444) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(web): use canonical secret helpers in api_v3; make ConfigManager secret strip/merge array-aware api_v3.py carried three inline nested copies of find_secret_fields/ separate_secrets (main-config save, plugin-config save, plugin-config reset). They drifted from each other (one lacked isinstance guards) and none supported the canonical module's array-item secrets (accounts[].token). All three endpoints now import from src/web_interface/secret_helpers. Adopting the canonical behavior makes array-item secrets reachable, and their parallel-placeholder shape ([{'token': ...}, {}] alongside the regular list) was not survivable by ConfigManager's round-trip: _strip_secrets_recursive dropped the whole key (losing the regular fields from config.json) and _deep_merge replaced the regular list wholesale on load. Both are now array-aware: - strip removes the secret fields from each item and ALWAYS keeps the list so indices survive for merge-on-load; whole-key secrets (scalar lists, shape mismatches) still drop the key entirely — never leak. - merge folds each secrets item into the config item at the same index, skipping {} placeholders. The regular list's length is authoritative in both directions: a user deleting an array item never has it resurrected from a stale secrets entry (extras warn and are ignored). api_v3's own deep_merge intentionally still replaces lists wholesale — form posts carry complete arrays and index-merging would resurrect deleted items; a comment now documents that. Tests: the parity guard flips from 'exactly 3 inline copies' to 'zero, and the canonical import must exist'; TestArraySecretStripAndMerge covers the new strip/merge semantics incl. length-mismatch contracts; new test_api_v3_secret_roundtrip.py drives all three endpoints through a Flask client with a REAL ConfigManager+SchemaManager over tmp_path, proving secrets land in config_secrets.json, config.json stays clean, and a fresh load merges them back into the right array items. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NohXi78cwsAKtN1sCfxjUh * fix: repair broken helper paths across display, cache, odds, logging, resolver, repos, config, validator Nine fixes for bugs surfaced while writing coverage for previously untested modules (plus the bool-duration quirk pinned in PR #441): - base_plugin.get_display_duration: exclude bools from both numeric branches — display_duration=True no longer reads as a 1-second slot; it falls through to config, then the 15.0 default. - display_helper: draw_error_message/draw_no_data_message called _draw_centered_text with the wrong arguments and crashed with AttributeError — both now delegate to draw_centered_text. draw_scorebug_layout drew status and clock at the same y, overprinting each other — they now share one combined top line. draw_ticker_layout drew its text starting at x=display_width (fully off-canvas), returning a blank frame every time — now draws at x=0; scroll_speed stays accepted-but-unused and is documented as such. - api_helper.clear_cache guarded on a nonexistent CacheManager.clear() method, silently never clearing anything; it now uses the real surface (clear_cache/delete/list_cache_files) and no-ops safely otherwise. - base_odds_manager._extract_espn_data raised AttributeError when ESPN sent explicit JSON nulls ("homeTeamOdds": null) — every level now null-safes with 'or {}'. format_odds_summary gated on is_odds_available, which deliberately ignores money lines, so ML-only odds formatted as "No odds available" — it now gates only on empty/no_odds data and formats money lines. - logging_config.ContextualFormatter mutated record.msg in place, so a second handler prepended the context prefix twice; it now formats a copy. log_error hardcoded exc_info=True and raised TypeError when the caller passed exc_info — now kwargs.setdefault. - dynamic_team_resolver wrote its "shared" class cache through self, creating instance shadows — the cache was per-instance and every scoreboard refetched rankings. Writes now go through the class. - saved_repositories cleaned URLs with an unanchored .replace('.git','') that mangled URLs merely containing '.git' (my.github.io -> myhub.io); now strips only a trailing suffix. add/remove also roll back the in-memory list when the save fails, so memory always matches disk. - config_helper.merge_configs shallow-copied the base, aliasing every un-overridden nested dict into the result — now deep-copies. - startup_validator.validate_all accumulated errors/warnings across calls — now resets both lists per run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NohXi78cwsAKtN1sCfxjUh * test: cover the previously untested modules Nine new suites plus an extension, asserting the Phase-1b fixed behavior and pinning the quirks deliberately left alone: - test_logging_config.py: formatters (JSON shape, no record mutation, single prefix through two handlers), PluginLoggerAdapter precedence, setup_logging handler hygiene and LEDMATRIX_DEBUG, log_error exc_info. - test_startup_validator.py: exact messages, error-vs-warning split, accessor split (load_config vs get_config), cache-dir branches with os.access monkeypatched (root can write anything in CI), idempotence, raise_on_errors classification precedence. - test_config_helper.py (full): load/save round trips, dot-notation get/set incl. silent-failure contract, post-fix no-aliasing merge, schema validation branches, the '{id}_config' key pin, default-enabled pin. - test_saved_repositories.py: three load shapes, bare-list rewrite pin, trailing-only .git strip (my.github.io regression), save-failure rollback, type-classification case-sensitivity pin. - test_api_helper.py: rate-limit math, cache-hit short circuit, ESPN URL/key formats, exact User-Agent guard, retry adapter, post-fix clear_cache against the real CacheManager surface, ttl-dropped pin. - test_base_odds_manager.py: cache-key/URL construction, no_odds sentinel round trip, stale-cache fallback, null-safe extraction, ML-only formatting, is_odds_available truth table (ML-blind by contract), config key/attr mismatch pin. - test_dynamic_team_resolver.py: expansion/dedup/slicing, dropped unknown-dynamic names (TOP_ substring hazard pinned), genuinely shared class cache (second instance: zero HTTP), TTL expiry, failure degradation without raising. - test_display_helper.py (full): the fixed error/no-data renders, combined scorebug top line, non-blank ticker with scroll_speed no-op pin, composite upconversion, logo bleed positions, square orientation pin. - test_skin_runtime_cache.py: discovery-cache hit/invalidation semantics (manifest mtime, .py edits pinned as non-invalidating), sys.modules namespacing contract incl. bare-name restore and stdlib shadowing, entry-module execute-once, API minor-version tolerance, skin_matches_target table. - test_sports_capabilities.py (extended): _draw_celebration_layout executed for real (flash window, matrix-dims fallback, highlight alternation, logo-failure isolation), _should_celebrate_for direct, strict duration boundary, score_to_int edges, both-teams-score precedence, expired-coalesce refire, disabled-win baseline preservation, id-less prune. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NohXi78cwsAKtN1sCfxjUh * test: real schedule/dim coverage for DisplayController; fix two vacuous schedule tests New test_display_controller_schedule.py drives _check_schedule and _check_dim_schedule on a bare controller stub: same-day and midnight-crossing windows with inclusive boundaries, global vs per-day vs legacy-inferred modes (and dim's global-only default — no legacy inference), per-day disabled days, invalid %H:%M fallbacks, unknown timezone -> UTC, dim_brightness default 30, inactive-display short circuit, and the _was_display_active/_was_dimmed transition flags. test_display_controller.py's test_schedule_disabled and test_active_hours patched config_service.get_config — which _check_schedule never reads — so both asserted the init-default value and could not fail. Rewritten on the test_inactive_hours pattern (inject controller.config['schedule'], reset the minute gate, flip the flag to the opposite state first so the assertion has teeth). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NohXi78cwsAKtN1sCfxjUh * ci: raise coverage floor to 48% Measured 50% with the new suites in place (was 47% baseline when the gate was introduced at 45); floor stays two points under measured. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NohXi78cwsAKtN1sCfxjUh * fix: address CodeQL alert and review findings - config_manager: the "secrets list longer than config list" warning now interpolates only config-side data (no key name or secrets-derived values), resolving the CodeQL clear-text-logging alert. - base_plugin: validate_config rejects bool display_duration, matching get_display_duration (bool is an int subclass and would otherwise pass as a positive number). - config_helper: merge_configs deep-copies override values in the non-recursive branch so mutating the merged result cannot reach back into override_config. - saved_repositories: saves are atomic (temp file + fsync + os.replace), so a failed write can no longer truncate saved_repositories.json. - tests: regression cases for each fix, plus a pin that whole-item array secrets (key[] + key[].field both marked) strip to empty {} skeletons — no secret values can reach config.json. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NohXi78cwsAKtN1sCfxjUh --------- Co-authored-by: Claude --- .github/workflows/test.yml | 2 +- src/base_odds_manager.py | 26 +- src/common/api_helper.py | 39 +- src/common/config_helper.py | 14 +- src/common/display_helper.py | 53 +-- src/config_manager.py | 73 +++- src/dynamic_team_resolver.py | 20 +- src/logging_config.py | 27 +- src/plugin_system/base_plugin.py | 17 +- src/plugin_system/saved_repositories.py | 74 +++- src/startup_validator.py | 7 +- test/test_api_helper.py | 275 +++++++++++++ test/test_base_odds_manager.py | 359 +++++++++++++++++ test/test_base_plugin_duration.py | 44 +- test/test_config_helper.py | 253 ++++++++++++ test/test_config_manager_secrets.py | 156 ++++++++ test/test_display_controller.py | 45 ++- test/test_display_controller_schedule.py | 277 +++++++++++++ test/test_display_helper.py | 307 ++++++++++++++ test/test_dynamic_team_resolver.py | 259 ++++++++++++ test/test_logging_config.py | 274 +++++++++++++ test/test_saved_repositories.py | 240 +++++++++++ test/test_skin_runtime_cache.py | 377 ++++++++++++++++++ test/test_sports_capabilities.py | 197 +++++++++ test/test_startup_validator.py | 293 ++++++++++++++ .../test_api_v3_secret_roundtrip.py | 259 ++++++++++++ .../test_secret_separation_parity.py | 92 ++--- web_interface/blueprints/api_v3.py | 108 +---- 28 files changed, 3875 insertions(+), 292 deletions(-) create mode 100644 test/test_api_helper.py create mode 100644 test/test_base_odds_manager.py create mode 100644 test/test_config_helper.py create mode 100644 test/test_display_controller_schedule.py create mode 100644 test/test_display_helper.py create mode 100644 test/test_dynamic_team_resolver.py create mode 100644 test/test_logging_config.py create mode 100644 test/test_saved_repositories.py create mode 100644 test/test_skin_runtime_cache.py create mode 100644 test/test_startup_validator.py create mode 100644 test/web_interface/test_api_v3_secret_roundtrip.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ecc49eb0..4942a0ab 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,4 +72,4 @@ jobs: --ignore=test/plugins \ --cov=src --cov=web_interface \ --cov-report=term \ - --cov-fail-under=45 + --cov-fail-under=48 diff --git a/src/base_odds_manager.py b/src/base_odds_manager.py index 3520ce67..f35c4d54 100644 --- a/src/base_odds_manager.py +++ b/src/base_odds_manager.py @@ -163,19 +163,25 @@ class BaseOddsManager: item = data["items"][0] self.logger.debug(f"First item keys: {list(item.keys())}") - # The ESPN API returns odds data directly in the item, not in a providers array - # Extract the odds data directly from the item + # The ESPN API returns odds data directly in the item, not in a + # providers array. ESPN sends explicit JSON nulls for absent + # sides ("homeTeamOdds": null), so every level uses `or {}` — + # .get's default only applies when the key is missing entirely. + home = item.get("homeTeamOdds") or {} + away = item.get("awayTeamOdds") or {} extracted_data = { "details": item.get("details"), "over_under": item.get("overUnder"), "spread": item.get("spread"), "home_team_odds": { - "money_line": item.get("homeTeamOdds", {}).get("moneyLine"), - "spread_odds": item.get("homeTeamOdds", {}).get("current", {}).get("pointSpread", {}).get("value") + "money_line": home.get("moneyLine"), + "spread_odds": ((home.get("current") or {}) + .get("pointSpread") or {}).get("value") }, "away_team_odds": { - "money_line": item.get("awayTeamOdds", {}).get("moneyLine"), - "spread_odds": item.get("awayTeamOdds", {}).get("current", {}).get("pointSpread", {}).get("value") + "money_line": away.get("moneyLine"), + "spread_odds": ((away.get("current") or {}) + .get("pointSpread") or {}).get("value") } } self.logger.debug(f"Returning extracted odds data: {json.dumps(extracted_data, indent=2)}") @@ -260,9 +266,13 @@ class BaseOddsManager: Returns: Formatted odds summary string """ - if not self.is_odds_available(odds_data): + # Gate only on truly-empty / negative-cached data. is_odds_available + # deliberately ignores money lines (its callers decide whether to + # RENDER an odds widget), but a summary of money-line-only odds is + # still meaningful — the parts loop below handles them. + if not odds_data or odds_data.get('no_odds'): return "No odds available" - + parts = [] # Add spread information diff --git a/src/common/api_helper.py b/src/common/api_helper.py index fa9b7694..d6974b22 100644 --- a/src/common/api_helper.py +++ b/src/common/api_helper.py @@ -272,20 +272,35 @@ class APIHelper: def clear_cache(self, pattern: Optional[str] = None) -> None: """ Clear cache data. - + + Uses CacheManager's real surface (clear_cache / delete / + list_cache_files); safely no-ops on managers without it. The old + implementation guarded on a nonexistent ``clear`` method, so it + silently never cleared anything. + Args: - pattern: Optional pattern to match cache keys + pattern: Optional substring to match cache keys; only matching + entries are deleted. """ - if self.cache_manager: - if hasattr(self.cache_manager, 'clear'): - if pattern: - # Clear only keys matching pattern - keys = self.cache_manager.keys() - for key in keys: - if pattern in key: - self.cache_manager.delete(key) - else: - self.cache_manager.clear() + if not self.cache_manager: + return + if pattern: + if (hasattr(self.cache_manager, 'list_cache_files') + and hasattr(self.cache_manager, 'delete')): + for entry in self.cache_manager.list_cache_files(): + key = entry.get('key') if isinstance(entry, dict) else None + if key and pattern in key: + self.cache_manager.delete(key) + else: + self.logger.debug( + "Cache manager lacks list_cache_files/delete; " + "cannot clear by pattern") + elif hasattr(self.cache_manager, 'clear_cache'): + self.cache_manager.clear_cache() + elif hasattr(self.cache_manager, 'clear'): + self.cache_manager.clear() + else: + self.logger.debug("Cache manager exposes no clear method; no-op") def _get_from_cache(self, key: str) -> Optional[Any]: """Get data from cache.""" diff --git a/src/common/config_helper.py b/src/common/config_helper.py index 79840188..b3e5aeac 100644 --- a/src/common/config_helper.py +++ b/src/common/config_helper.py @@ -5,6 +5,7 @@ Handles configuration management and validation for LED matrix plugins. Extracted from LEDMatrix core to provide reusable functionality for plugins. """ +import copy import json import logging from pathlib import Path @@ -160,17 +161,20 @@ class ConfigHelper: override_config: Configuration to merge in (takes precedence) Returns: - Merged configuration dictionary + Merged configuration dictionary (fully independent of both + inputs — a shallow copy would alias un-overridden nested dicts, + so mutating the result would mutate the caller's base config). """ - merged = base_config.copy() - + merged = copy.deepcopy(base_config) + for key, value in override_config.items(): if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): # Recursively merge nested dictionaries merged[key] = self.merge_configs(merged[key], value) else: - # Override with new value - merged[key] = value + # Override with new value — deep-copied so mutating the + # merged result can't reach back into override_config. + merged[key] = copy.deepcopy(value) return merged diff --git a/src/common/display_helper.py b/src/common/display_helper.py index 13dc3030..c88c9e63 100644 --- a/src/common/display_helper.py +++ b/src/common/display_helper.py @@ -115,17 +115,13 @@ class DisplayHelper: if home_logo and away_logo: self._draw_logos(main_img, home_logo, away_logo) - # Draw status/period text (top center) - if status_text or period_text: - status_display = f"{period_text} {status_text}".strip() - if status_display: - self._draw_centered_text(draw, status_display, - fonts.get('time', fonts.get('status')), - y_position=1) - - # Draw clock if available - if clock: - self._draw_centered_text(draw, clock, fonts.get('time'), y_position=1) + # Draw one combined top line (period/status/clock all share y=1 — + # drawing them separately overprinted each other). + top_line = " ".join(p for p in [period_text, status_text, clock] if p) + if top_line: + self._draw_centered_text(draw, top_line, + fonts.get('time', fonts.get('status')), + y_position=1) # Draw scores (center) score_text = f"{away_score}-{home_score}" @@ -153,26 +149,28 @@ class DisplayHelper: """ Draw a ticker/scrolling text layout. + Renders a single static frame with the text at the left edge; the + caller advances the scroll by re-rendering or shifting. The + scroll_speed parameter is accepted for API compatibility but does + not affect this frame. (Previously the text was drawn starting at + x=display_width — entirely off-canvas — so every frame was blank.) + Args: text: Text to display font: Font to use background_color: Background color text_color: Text color - scroll_speed: Pixels to scroll per frame - + scroll_speed: Accepted for compatibility; unused per-frame + Returns: PIL Image with ticker layout """ img = self.create_base_image(background_color) draw = ImageDraw.Draw(img) - - # Start text off-screen to the right - x_position = self.display_width - - # Draw text - self._draw_text_with_outline(draw, text, (x_position, self.display_height // 2 - 6), + + self._draw_text_with_outline(draw, text, (0, self.display_height // 2 - 6), font, fill=text_color) - + return img def draw_centered_text(self, text: str, font: ImageFont.ImageFont, @@ -214,15 +212,9 @@ class DisplayHelper: Returns: PIL Image with error message """ - img = self.create_base_image((50, 0, 0)) # Dark red background - - # Use default font + # Dark red background, white text font = ImageFont.load_default() - - # Draw centered error message - self._draw_centered_text(message, font, (50, 0, 0), (255, 255, 255)) - - return img + return self.draw_centered_text(message, font, (50, 0, 0), (255, 255, 255)) def draw_no_data_message(self, message: str = "No Data") -> Image.Image: """ @@ -234,11 +226,8 @@ class DisplayHelper: Returns: PIL Image with no data message """ - img = self.create_base_image((0, 0, 0)) font = ImageFont.load_default() - self._draw_centered_text(message, font, (0, 0, 0), (150, 150, 150)) - - return img + return self.draw_centered_text(message, font, (0, 0, 0), (150, 150, 150)) def get_display_dimensions(self) -> Tuple[int, int]: """ diff --git a/src/config_manager.py b/src/config_manager.py index 569b295e..453e73b1 100644 --- a/src/config_manager.py +++ b/src/config_manager.py @@ -269,20 +269,47 @@ class ConfigManager: self.logger.error(error_msg, exc_info=True) raise ConfigError(error_msg, config_path=self.config_path) from e + @staticmethod + def _is_parallel_secrets_list(value: Any) -> bool: + """True for the parallel-placeholder list shape emitted by + ``secret_helpers.separate_secrets`` for array-item secrets: a + non-empty list whose elements are ALL dicts (``{}`` marks an item + with no secrets). Any other list-shaped secrets value is a + whole-key secret (e.g. a list of secret scalars).""" + return (isinstance(value, list) and bool(value) + and all(isinstance(item, dict) for item in value)) + def _strip_secrets_recursive(self, data_to_filter: Dict[str, Any], secrets: Dict[str, Any]) -> Dict[str, Any]: """Recursively remove secret keys from a dictionary.""" result = {} for key, value in data_to_filter.items(): - if key in secrets: - if isinstance(value, dict) and isinstance(secrets[key], dict): - # This key is a shared group, recurse - stripped_sub_dict = self._strip_secrets_recursive(value, secrets[key]) - if stripped_sub_dict: # Only add if there's non-secret data left - result[key] = stripped_sub_dict - # Else, it's a secret key at this level, so we skip it - else: + if key not in secrets: # This key is not in secrets, so we keep it result[key] = value + continue + sec = secrets[key] + if isinstance(value, dict) and isinstance(sec, dict): + # This key is a shared group, recurse + stripped_sub_dict = self._strip_secrets_recursive(value, sec) + if stripped_sub_dict: # Only add if there's non-secret data left + result[key] = stripped_sub_dict + elif isinstance(value, list) and self._is_parallel_secrets_list(sec): + # Parallel-list shape from separate_secrets: sec[i] holds the + # secret fields of value[i] ({} = item i has none). Strip each + # item and ALWAYS keep the list — indices must survive so the + # merge-on-load can realign secrets with their items. The + # regular list's length is authoritative: extra secrets + # entries are ignored. + stripped_items = [] + for i, item in enumerate(value): + s_item = sec[i] if i < len(sec) else {} + if isinstance(item, dict) and s_item: + stripped_items.append(self._strip_secrets_recursive(item, s_item)) + else: + stripped_items.append(item) + result[key] = stripped_items + # Else: whole-key secret (scalar, list of secret scalars, or a + # shape mismatch) -> drop the key entirely. Never leak. return result def _load_secrets_for_save(self) -> Dict[str, Any]: @@ -358,11 +385,39 @@ class ConfigManager: return None def _deep_merge(self, target: Dict[str, Any], source: Dict[str, Any]) -> None: - """Deep merge source dict into target dict.""" + """Deep merge source dict into target dict. + + Sole call site: merging config_secrets.json into the loaded config. + Understands the parallel-list shape separate_secrets emits for + array-item secrets (see _is_parallel_secrets_list): each secrets + list item is merged into the config list item at the same index + ({} placeholders skipped). The config list's length is + authoritative — a user deleting an array item from config.json + must not have it resurrected from a stale secrets entry.""" for key, value in source.items(): if key in target and isinstance(target[key], dict) and isinstance(value, dict): self._deep_merge(target[key], value) + elif (key in target and isinstance(target[key], list) + and self._is_parallel_secrets_list(value)): + tlist = target[key] + for i, s_item in enumerate(value): + if i >= len(tlist): + # Interpolate only config-side data here — nothing + # iterated out of the secrets dict (not even the key + # name) may reach the log. + self.logger.warning( + "A secrets list is longer than the config list it " + "parallels (config has %d item(s)); ignoring the " + "extra entries", len(tlist)) + break + if not s_item: + continue # {} placeholder: item i has no secrets + if isinstance(tlist[i], dict): + self._deep_merge(tlist[i], s_item) + else: + tlist[i] = s_item # shape drift; the secret wins else: + # Scalars AND whole-secret scalar arrays: replace (legacy). target[key] = value def _create_config_from_template(self) -> None: diff --git a/src/dynamic_team_resolver.py b/src/dynamic_team_resolver.py index ab969e17..7cbe09cd 100644 --- a/src/dynamic_team_resolver.py +++ b/src/dynamic_team_resolver.py @@ -167,10 +167,14 @@ class DynamicTeamResolver: # Sort by ranking (1, 2, 3, etc.) sorted_rankings = dict(sorted(rankings.items(), key=lambda x: x[1])) - - # Cache the results - self._rankings_cache = sorted_rankings - self._cache_timestamp = current_time + + # Cache the results ON THE CLASS. Assigning through self + # would create instance attributes that shadow the shared + # class-level cache, making it per-instance — and every + # scoreboard constructs its own resolver, so the cache + # would never actually be shared. + DynamicTeamResolver._rankings_cache = sorted_rankings + DynamicTeamResolver._cache_timestamp = current_time self.logger.info(f"Fetched rankings for {len(sorted_rankings)} teams") return sorted_rankings @@ -216,9 +220,11 @@ class DynamicTeamResolver: return any(pattern in team_name.upper() for pattern in dynamic_patterns) def clear_cache(self): - """Clear the rankings cache to force fresh data on next request.""" - self._rankings_cache = {} - self._cache_timestamp = 0 + """Clear the SHARED rankings cache to force fresh data on next + request. Writes through the class — assigning via self would only + shadow the shared cache for this instance.""" + DynamicTeamResolver._rankings_cache = {} + DynamicTeamResolver._cache_timestamp = 0 self.logger.info("Cleared dynamic team rankings cache") diff --git a/src/logging_config.py b/src/logging_config.py index 48eba67c..17d831bb 100644 --- a/src/logging_config.py +++ b/src/logging_config.py @@ -5,6 +5,7 @@ Provides consistent logging configuration across the LEDMatrix application. Supports structured logging with context information and appropriate log levels. """ +import copy import logging import sys import os @@ -65,24 +66,29 @@ class ContextualFormatter(logging.Formatter): self.include_context = include_context def format(self, record: logging.LogRecord) -> str: - """Format log record with context.""" - # Add context to message if present + """Format log record with context. + + Works on a shallow copy of the record: a record is formatted once + PER HANDLER, so mutating record.msg in place (the old behavior) + prepended the context prefix again for every additional handler. + """ if self.include_context: context_parts = [] - + if hasattr(record, 'plugin_id'): context_parts.append(f"[Plugin: {record.plugin_id}]") - + if hasattr(record, 'operation_id'): context_parts.append(f"[Op: {record.operation_id}]") - + if hasattr(record, 'context') and isinstance(record.context, dict): for key, value in record.context.items(): context_parts.append(f"[{key}: {value}]") - + if context_parts: + record = copy.copy(record) record.msg = ' '.join(context_parts) + ' ' + str(record.msg) - + return super().format(record) @@ -224,8 +230,11 @@ def log_warning(logger: logging.Logger, message: str, **kwargs) -> None: def log_error(logger: logging.Logger, message: str, **kwargs) -> None: - """Log error message with context.""" - log_with_context(logger, logging.ERROR, message, **kwargs, exc_info=True) + """Log error message with context. Defaults exc_info=True; a caller + passing exc_info explicitly wins (the old hardcoded keyword raised + TypeError on that duplicate).""" + kwargs.setdefault('exc_info', True) + log_with_context(logger, logging.ERROR, message, **kwargs) def log_debug(logger: logging.Logger, message: str, **kwargs) -> None: diff --git a/src/plugin_system/base_plugin.py b/src/plugin_system/base_plugin.py index 8ec4414d..ceee88bf 100644 --- a/src/plugin_system/base_plugin.py +++ b/src/plugin_system/base_plugin.py @@ -364,8 +364,10 @@ class BasePlugin(ABC): # Handle None case if duration is None: pass # Fall through to config - # Try to convert to float if it's a number or numeric string - elif isinstance(duration, (int, float)): + # Try to convert to float if it's a number or numeric string. + # bool is excluded: it's an int subclass, and True would + # otherwise read as a 1-second duration. + elif isinstance(duration, (int, float)) and not isinstance(duration, bool): if duration > 0: return float(duration) else: @@ -403,8 +405,9 @@ class BasePlugin(ABC): # Fall back to config config_duration = self.config.get("display_duration", 15.0) try: - # Ensure config value is also a valid float - if isinstance(config_duration, (int, float)): + # Ensure config value is also a valid float (bool excluded — an + # int subclass that would otherwise read True as 1 second) + if isinstance(config_duration, (int, float)) and not isinstance(config_duration, bool): if config_duration > 0: return float(config_duration) else: @@ -794,10 +797,12 @@ class BasePlugin(ABC): self.logger.error("'enabled' must be a boolean") return False - # Check display_duration if present + # Check display_duration if present. bool is excluded explicitly: + # it's an int subclass, and get_display_duration rejects it too. if "display_duration" in self.config: duration = self.config["display_duration"] - if not isinstance(duration, (int, float)) or duration <= 0: + if (not isinstance(duration, (int, float)) + or isinstance(duration, bool) or duration <= 0): self.logger.error("'display_duration' must be a positive number") return False diff --git a/src/plugin_system/saved_repositories.py b/src/plugin_system/saved_repositories.py index bd0e1872..c8da3c5b 100644 --- a/src/plugin_system/saved_repositories.py +++ b/src/plugin_system/saved_repositories.py @@ -6,6 +6,7 @@ Manages saved GitHub repository URLs for easy plugin discovery and installation. import json import logging +import os from pathlib import Path from typing import List, Dict, Optional @@ -43,20 +44,45 @@ class SavedRepositoriesManager: return [] def _save_repositories(self) -> bool: - """Save repositories to file.""" + """Save repositories to file atomically. + + Writes to a temp file in the same directory and os.replace()s it + over the target, so a failed write can never truncate or + half-overwrite an existing saved_repositories.json. + """ + tmp_path = self.config_path.with_suffix(self.config_path.suffix + '.tmp') try: # Ensure directory exists self.config_path.parent.mkdir(parents=True, exist_ok=True) - - with open(self.config_path, 'w') as f: + + with open(tmp_path, 'w') as f: json.dump(self.repositories, f, indent=2) - + f.flush() + os.fsync(f.fileno()) + os.replace(tmp_path, self.config_path) + self.logger.info(f"Saved {len(self.repositories)} repositories to {self.config_path}") return True except Exception as e: self.logger.error(f"Error saving repositories: {e}") + try: + tmp_path.unlink(missing_ok=True) + except OSError: + pass return False + @staticmethod + def _clean_url(repo_url: str) -> str: + """Normalize a repo URL: strip whitespace, trailing slashes, and a + trailing ``.git`` suffix ONLY. (The old ``.replace('.git', '')`` + was an unanchored substring replace that mangled URLs merely + containing ``.git``, e.g. ``https://github.com/user/my.github.io``.) + """ + repo_url = repo_url.strip().rstrip('/') + if repo_url.endswith('.git'): + repo_url = repo_url[:-4] + return repo_url + def get_all(self) -> List[Dict[str, str]]: """Get all saved repositories.""" return self.repositories.copy() @@ -72,15 +98,14 @@ class SavedRepositoriesManager: Returns: True if added successfully """ - # Clean URL - repo_url = repo_url.strip().rstrip('/').replace('.git', '') - + repo_url = self._clean_url(repo_url) + # Check if already exists for repo in self.repositories: if repo.get('url') == repo_url: self.logger.warning(f"Repository already exists: {repo_url}") return False - + # Extract name from URL if not provided if not name: parts = repo_url.split('/') @@ -88,15 +113,20 @@ class SavedRepositoriesManager: name = parts[-1] else: name = repo_url - + # Add repository self.repositories.append({ 'url': repo_url, 'name': name, 'type': 'registry' if 'plugins.json' in repo_url or 'ledmatrix-plugins' in repo_url.lower() else 'single' }) - - return self._save_repositories() + + if not self._save_repositories(): + # Keep memory consistent with disk: a failed save must not leave + # a phantom entry that only this process can see. + self.repositories.pop() + return False + return True def remove(self, repo_url: str) -> bool: """ @@ -108,21 +138,25 @@ class SavedRepositoriesManager: Returns: True if removed successfully """ - # Clean URL - repo_url = repo_url.strip().rstrip('/').replace('.git', '') - - original_count = len(self.repositories) - self.repositories = [r for r in self.repositories if r.get('url') != repo_url] - - if len(self.repositories) < original_count: - return self._save_repositories() + repo_url = self._clean_url(repo_url) + + previous = self.repositories + remaining = [r for r in previous if r.get('url') != repo_url] + + if len(remaining) < len(previous): + self.repositories = remaining + if not self._save_repositories(): + # Failed save: restore so memory matches disk. + self.repositories = previous + return False + return True else: self.logger.warning(f"Repository not found: {repo_url}") return False def has(self, repo_url: str) -> bool: """Check if a repository is already saved.""" - repo_url = repo_url.strip().rstrip('/').replace('.git', '') + repo_url = self._clean_url(repo_url) return any(r.get('url') == repo_url for r in self.repositories) def get_registry_repositories(self) -> List[Dict[str, str]]: diff --git a/src/startup_validator.py b/src/startup_validator.py index 86c703f6..8dd9fce2 100644 --- a/src/startup_validator.py +++ b/src/startup_validator.py @@ -37,7 +37,12 @@ class StartupValidator: Tuple of (is_valid, errors, warnings) """ self.logger.info("Starting startup validation...") - + + # Fresh lists each run — without this, calling validate_all() twice + # duplicated every message. + self.errors = [] + self.warnings = [] + # Validate configuration self._validate_config() diff --git a/test/test_api_helper.py b/test/test_api_helper.py new file mode 100644 index 00000000..b3c24e99 --- /dev/null +++ b/test/test_api_helper.py @@ -0,0 +1,275 @@ +""" +Tests for src/common/api_helper.py (APIHelper). + +Covers rate limiting, cached GETs, ESPN URL/cache-key construction, +session header defaults and per-call merging, the retry adapter, and the +fixed clear_cache() behavior (real CacheManager surface: clear_cache / +delete / list_cache_files, with safe no-ops elsewhere). + +No real network: helper.session.get/post are always replaced with mocks. +""" + +import types +from unittest.mock import MagicMock, Mock + +import pytest +import requests +from freezegun import freeze_time + +import src.common.api_helper as api_helper_module +from src.common.api_helper import APIHelper + + +def _make_response(payload): + response = MagicMock() + response.json.return_value = payload + response.raise_for_status.return_value = None + return response + + +@pytest.fixture +def cache(): + cache = MagicMock() + cache.get.return_value = None + return cache + + +@pytest.fixture +def helper(cache): + helper = APIHelper(cache_manager=cache) + # Default min interval is 1.0s and would really sleep between requests. + helper.set_rate_limit(0) + return helper + + +# --------------------------------------------------------------------------- +# Rate limiting +# --------------------------------------------------------------------------- + +class TestRateLimiting: + def test_sleeps_for_remaining_interval(self, helper, monkeypatch): + fake_time = MagicMock() + fake_time.time.side_effect = [102.0, 105.0] + monkeypatch.setattr(api_helper_module, 'time', fake_time) + + helper.set_rate_limit(5) + helper._last_request_time = 100.0 + helper._enforce_rate_limit() + + # 2s elapsed of a 5s interval -> sleep the remaining 3s. + fake_time.sleep.assert_called_once() + assert fake_time.sleep.call_args[0][0] == pytest.approx(3.0) + assert helper._last_request_time == 105.0 + + def test_no_sleep_when_interval_elapsed(self, helper, monkeypatch): + fake_time = MagicMock() + fake_time.time.side_effect = [200.0, 201.0] + monkeypatch.setattr(api_helper_module, 'time', fake_time) + + helper.set_rate_limit(5) + helper._last_request_time = 100.0 + helper._enforce_rate_limit() + + fake_time.sleep.assert_not_called() + assert helper._last_request_time == 201.0 + + +# --------------------------------------------------------------------------- +# get() +# --------------------------------------------------------------------------- + +class TestGet: + def test_cache_hit_skips_request_and_rate_limit(self, helper, cache): + cache.get.return_value = {'cached': True} + helper.session.get = Mock() + rate_spy = Mock() + helper._enforce_rate_limit = rate_spy + + result = helper.get('https://example.com/api', cache_key='k') + + assert result == {'cached': True} + helper.session.get.assert_not_called() + rate_spy.assert_not_called() + + def test_cache_miss_fetches_and_caches_without_ttl(self, helper, cache): + cache.get.return_value = None + helper.session.get = Mock(return_value=_make_response({'a': 1})) + + result = helper.get('https://example.com/api', cache_key='k', + cache_ttl=999) + + assert result == {'a': 1} + # Pin the ttl-dropped contract: CacheManager.set is called with + # (key, data) only — the cache_ttl argument is discarded. + cache.set.assert_called_once_with('k', {'a': 1}) + + def test_request_exception_returns_none_and_caches_nothing( + self, helper, cache): + helper.session.get = Mock( + side_effect=requests.exceptions.RequestException('boom')) + + result = helper.get('https://example.com/api', cache_key='k') + + assert result is None + cache.set.assert_not_called() + + def test_timeout_zero_falls_back_to_default(self, helper): + # Quirk pin: `timeout or self.default_timeout` treats an explicit + # timeout=0 as falsy, so the default (30) is used instead. + helper.session.get = Mock(return_value=_make_response({})) + + helper.get('https://example.com/api', timeout=0) + + assert helper.session.get.call_args.kwargs['timeout'] == 30 + + def test_per_call_headers_merge_over_session_headers(self, helper): + helper.session.get = Mock(return_value=_make_response({})) + + helper.get('https://example.com/api', headers={'X-Custom': 'yes'}) + + sent = helper.session.get.call_args.kwargs['headers'] + # Merged, not replaced: session defaults survive alongside the + # per-call header. + assert sent['X-Custom'] == 'yes' + assert sent['User-Agent'] == ( + 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)') + assert sent['Accept'] == 'application/json' + # The session's own headers are not polluted by the per-call ones. + assert 'X-Custom' not in helper.session.headers + + +# --------------------------------------------------------------------------- +# ESPN helpers +# --------------------------------------------------------------------------- + +class TestEspnHelpers: + @freeze_time('2026-08-07') + def test_fetch_espn_scoreboard_url_params_and_cache_key(self, helper): + helper.get = Mock(return_value={'ok': 1}) + + result = helper.fetch_espn_scoreboard('football', 'nfl') + + assert result == {'ok': 1} + helper.get.assert_called_once_with( + 'https://site.api.espn.com/apis/site/v2/sports/football/nfl/scoreboard', + params={'dates': '20260807', 'limit': 1000}, + cache_key='espn_football_nfl_20260807', + cache_ttl=300, + ) + + def test_fetch_espn_scoreboard_explicit_date(self, helper): + helper.get = Mock(return_value=None) + + helper.fetch_espn_scoreboard('basketball', 'nba', date='20250115') + + kwargs = helper.get.call_args.kwargs + assert kwargs['params'] == {'dates': '20250115', 'limit': 1000} + assert kwargs['cache_key'] == 'espn_basketball_nba_20250115' + + def test_fetch_espn_standings_url_and_cache_key(self, helper): + helper.get = Mock(return_value={'ok': 1}) + + helper.fetch_espn_standings('football', 'nfl') + + helper.get.assert_called_once_with( + 'https://site.api.espn.com/apis/site/v2/sports/football/nfl/standings', + cache_key='espn_standings_football_nfl', + cache_ttl=3600, + ) + + def test_fetch_espn_rankings_url_and_cache_key(self, helper): + helper.get = Mock(return_value={'ok': 1}) + + helper.fetch_espn_rankings('football', 'college-football') + + helper.get.assert_called_once_with( + 'https://site.api.espn.com/apis/site/v2/sports/football/college-football/rankings', + cache_key='espn_rankings_football_college-football', + cache_ttl=3600, + ) + + +# --------------------------------------------------------------------------- +# Session setup +# --------------------------------------------------------------------------- + +class TestSessionSetup: + def test_user_agent_exact(self, helper): + # Regression guard: ESPN began 403ing other user agents; this exact + # string must be sent on every request. + assert helper.session.headers['User-Agent'] == ( + 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)') + + def test_retry_adapter_configuration(self): + helper = APIHelper(cache_manager=None, max_retries=7) + + retries = helper.session.get_adapter('https://x').max_retries + assert retries.total == 7 + assert {429, 500, 502, 503, 504} <= set(retries.status_forcelist) + + +# --------------------------------------------------------------------------- +# clear_cache (fixed behavior: real CacheManager surface) +# --------------------------------------------------------------------------- + +class TestClearCache: + def test_no_pattern_uses_clear_cache_method(self): + manager = types.SimpleNamespace(clear_cache=Mock()) + helper = APIHelper(cache_manager=manager) + helper.set_rate_limit(0) + + helper.clear_cache() + + manager.clear_cache.assert_called_once_with() + + def test_no_pattern_falls_back_to_clear(self): + manager = types.SimpleNamespace(clear=Mock()) + helper = APIHelper(cache_manager=manager) + helper.set_rate_limit(0) + + helper.clear_cache() + + manager.clear.assert_called_once_with() + + def test_no_pattern_manager_without_any_clear_is_noop(self): + helper = APIHelper(cache_manager=object()) + helper.set_rate_limit(0) + + helper.clear_cache() # must not raise + + def test_pattern_deletes_only_matching_keys(self): + manager = types.SimpleNamespace( + list_cache_files=Mock(return_value=[ + {'key': 'espn_nfl_x'}, + {'key': 'other'}, + ]), + delete=Mock(), + ) + helper = APIHelper(cache_manager=manager) + helper.set_rate_limit(0) + + helper.clear_cache(pattern='espn') + + manager.delete.assert_called_once_with('espn_nfl_x') + + def test_pattern_manager_without_list_cache_files_is_noop(self): + helper = APIHelper(cache_manager=object()) + helper.set_rate_limit(0) + + helper.clear_cache(pattern='espn') # must not raise + + +# --------------------------------------------------------------------------- +# No cache manager +# --------------------------------------------------------------------------- + +class TestNoCacheManager: + def test_all_cache_operations_safe_without_manager(self): + helper = APIHelper(cache_manager=None) + helper.set_rate_limit(0) + + assert helper.get_cache('k') is None + assert helper._get_from_cache('k') is None + assert helper.set_cache('k', {'a': 1}) is None + assert helper.clear_cache() is None + assert helper.clear_cache(pattern='espn') is None diff --git a/test/test_base_odds_manager.py b/test/test_base_odds_manager.py new file mode 100644 index 00000000..b08183ad --- /dev/null +++ b/test/test_base_odds_manager.py @@ -0,0 +1,359 @@ +""" +Tests for src/base_odds_manager.py (BaseOddsManager). + +Covers get_odds validation/caching/URL construction, the null-safe +_extract_espn_data fix (ESPN sends explicit JSON nulls for absent sides), +the no_odds sentinel, stale-cache fallback on request failure, +is_odds_available's ML-blind truth table, the fixed format_odds_summary +gate (money-line-only odds now format), get_odds_for_games, and +configuration loading. + +No real network: src.base_odds_manager.requests.get is always patched. +""" + +from unittest.mock import MagicMock, patch + +import pytest +import requests + +from src.base_odds_manager import BaseOddsManager + + +FULL_ITEM = { + 'details': 'DAL -3.5', + 'overUnder': 47.5, + 'spread': -3.5, + 'homeTeamOdds': {'moneyLine': -150, 'current': {'pointSpread': {'value': -3.5}}}, + 'awayTeamOdds': {'moneyLine': 130, 'current': {'pointSpread': {'value': 3.5}}}, +} + +FULL_EXTRACTED = { + 'details': 'DAL -3.5', + 'over_under': 47.5, + 'spread': -3.5, + 'home_team_odds': {'money_line': -150, 'spread_odds': -3.5}, + 'away_team_odds': {'money_line': 130, 'spread_odds': 3.5}, +} + + +def _make_response(payload): + response = MagicMock() + response.json.return_value = payload + response.raise_for_status.return_value = None + return response + + +@pytest.fixture +def cache_manager(): + cm = MagicMock() + # A bare MagicMock returns truthy Mocks from every call, so every + # get_odds() would look like a cache hit. Explicitly wire a miss. + cm.get_with_auto_strategy.return_value = None + return cm + + +@pytest.fixture +def manager(cache_manager): + return BaseOddsManager(cache_manager) + + +@pytest.fixture +def mock_get(): + with patch('src.base_odds_manager.requests.get') as m: + m.return_value = _make_response({'items': [dict(FULL_ITEM)]}) + yield m + + +# --------------------------------------------------------------------------- +# get_odds +# --------------------------------------------------------------------------- + +class TestGetOdds: + def test_none_sport_raises(self, manager): + with pytest.raises(ValueError): + manager.get_odds(None, 'nfl', '1') + + def test_none_league_raises(self, manager): + with pytest.raises(ValueError): + manager.get_odds('football', None, '1') + + def test_cache_key_and_url(self, manager, cache_manager, mock_get): + manager.get_odds('football', 'nfl', '401') + + cache_manager.get_with_auto_strategy.assert_called_once_with( + 'odds_espn_football_nfl_401') + url = mock_get.call_args[0][0] + # Event id appears twice: /events//competitions//odds + assert '/events/401/competitions/401/odds' in url + assert url == ('https://sports.core.api.espn.com/v2/sports/football/' + 'leagues/nfl/events/401/competitions/401/odds') + assert mock_get.call_args.kwargs['timeout'] == 30 + + def test_ncaa_fb_maps_to_college_football(self, manager, mock_get): + manager.get_odds('football', 'ncaa_fb', '401') + + url = mock_get.call_args[0][0] + assert '/leagues/college-football/' in url + + def test_unknown_league_passes_through(self, manager, mock_get): + manager.get_odds('football', 'xfl', '401') + + assert '/leagues/xfl/' in mock_get.call_args[0][0] + + def test_cache_hit_skips_http(self, manager, cache_manager, mock_get): + cache_manager.get_with_auto_strategy.return_value = {'spread': -3.0} + + result = manager.get_odds('football', 'nfl', '401') + + assert result == {'spread': -3.0} + mock_get.assert_not_called() + + def test_cached_no_odds_sentinel_returned_verbatim( + self, manager, cache_manager, mock_get): + cache_manager.get_with_auto_strategy.return_value = {'no_odds': True} + + result = manager.get_odds('football', 'nfl', '401') + + assert result == {'no_odds': True} + mock_get.assert_not_called() + assert manager.is_odds_available(result) is False + + def test_success_caches_extracted_data_with_interval_ttl( + self, manager, cache_manager, mock_get): + result = manager.get_odds('football', 'nfl', '401', + update_interval_seconds=100) + + assert result == FULL_EXTRACTED + cache_manager.set.assert_called_once_with( + 'odds_espn_football_nfl_401', FULL_EXTRACTED, ttl=100) + + def test_no_odds_caches_sentinel(self, manager, cache_manager, mock_get): + mock_get.return_value = _make_response({'count': 0, 'items': []}) + + result = manager.get_odds('football', 'nfl', '401') + + assert result is None + cache_manager.set.assert_called_once_with( + 'odds_espn_football_nfl_401', {'no_odds': True}, ttl=3600) + + def test_zero_interval_falls_back_to_default( + self, manager, cache_manager, mock_get): + # Quirk pin: `update_interval_seconds or self.update_interval` + # treats an explicit 0 as falsy, so the 3600 default wins. + manager.get_odds('football', 'nfl', '401', update_interval_seconds=0) + + assert cache_manager.set.call_args.kwargs['ttl'] == 3600 + + def test_request_exception_falls_back_to_stale_cache( + self, manager, cache_manager, mock_get): + cache_manager.get_with_auto_strategy.side_effect = [ + None, {'stale': True}] + mock_get.side_effect = requests.exceptions.RequestException('boom') + + result = manager.get_odds('football', 'nfl', '401') + + assert result == {'stale': True} + assert cache_manager.get_with_auto_strategy.call_count == 2 + + +# --------------------------------------------------------------------------- +# _extract_espn_data +# --------------------------------------------------------------------------- + +class TestExtractEspnData: + def test_full_item_extracts_all_fields(self, manager): + result = manager._extract_espn_data({'items': [dict(FULL_ITEM)]}) + assert result == FULL_EXTRACTED + + def test_explicit_nulls_do_not_raise(self, manager): + # Post-fix: ESPN sends explicit JSON nulls for absent sides + # ("homeTeamOdds": null, "current": null); extraction must not + # raise and yields None fields. + payload = {'items': [{ + 'homeTeamOdds': None, + 'awayTeamOdds': {'moneyLine': 150, 'current': None}, + }]} + + result = manager._extract_espn_data(payload) + + assert result is not None + assert result['home_team_odds']['money_line'] is None + assert result['home_team_odds']['spread_odds'] is None + assert result['away_team_odds']['money_line'] == 150 + assert result['away_team_odds']['spread_odds'] is None + + def test_valid_empty_response_returns_none(self, manager): + assert manager._extract_espn_data({'count': 0, 'items': []}) is None + + def test_unexpected_structure_returns_none(self, manager): + assert manager._extract_espn_data({'unexpected': True}) is None + + def test_item_without_odds_fields_cached_as_data_not_sentinel( + self, manager, cache_manager, mock_get): + # Characterization pin: an item with no odds fields still extracts + # to a truthy dict of all-None values, so get_odds caches it as + # real data (NOT the no_odds sentinel) — but is_odds_available + # correctly reports False for it. + mock_get.return_value = _make_response({'items': [{}]}) + + result = manager.get_odds('football', 'nfl', '401') + + assert result == { + 'details': None, + 'over_under': None, + 'spread': None, + 'home_team_odds': {'money_line': None, 'spread_odds': None}, + 'away_team_odds': {'money_line': None, 'spread_odds': None}, + } + cache_manager.set.assert_called_once_with( + 'odds_espn_football_nfl_401', result, ttl=3600) + assert manager.is_odds_available(result) is False + + +# --------------------------------------------------------------------------- +# is_odds_available +# --------------------------------------------------------------------------- + +class TestIsOddsAvailable: + def test_none_is_false(self, manager): + assert manager.is_odds_available(None) is False + + def test_empty_dict_is_false(self, manager): + assert manager.is_odds_available({}) is False + + def test_no_odds_sentinel_is_false(self, manager): + assert manager.is_odds_available({'no_odds': True}) is False + + def test_spread_is_true(self, manager): + assert manager.is_odds_available({'spread': -3.5}) is True + + def test_over_under_is_true(self, manager): + assert manager.is_odds_available({'over_under': 47.5}) is True + + def test_nested_home_spread_odds_is_true(self, manager): + assert manager.is_odds_available( + {'home_team_odds': {'spread_odds': -3.5}}) is True + + def test_nested_away_spread_odds_is_true(self, manager): + assert manager.is_odds_available( + {'away_team_odds': {'spread_odds': 3.5}}) is True + + def test_moneyline_only_is_false(self, manager): + # Pinned ML-blind contract: is_odds_available ignores money lines + # (its callers decide whether to render an odds widget). Note that + # format_odds_summary deliberately uses a DIFFERENT gate — it will + # still format money-line-only odds (see TestFormatOddsSummary). + ml_only = { + 'home_team_odds': {'money_line': -120}, + 'away_team_odds': {'money_line': 100}, + } + assert manager.is_odds_available(ml_only) is False + + +# --------------------------------------------------------------------------- +# format_odds_summary (fixed gate: empty / no_odds only) +# --------------------------------------------------------------------------- + +class TestFormatOddsSummary: + def test_moneyline_only_formats(self, manager): + result = manager.format_odds_summary({ + 'home_team_odds': {'money_line': -120}, + 'away_team_odds': {'money_line': 100}, + }) + assert result == 'Home ML: -120 | Away ML: 100' + + def test_full_data_formats_all_parts(self, manager): + result = manager.format_odds_summary(FULL_EXTRACTED) + assert result == 'Spread: -3.5 | O/U: 47.5 | Home ML: -150 | Away ML: 130' + + def test_none_is_no_odds(self, manager): + assert manager.format_odds_summary(None) == 'No odds available' + + def test_empty_dict_is_no_odds(self, manager): + assert manager.format_odds_summary({}) == 'No odds available' + + def test_no_odds_sentinel_is_no_odds(self, manager): + assert manager.format_odds_summary( + {'no_odds': True}) == 'No odds available' + + +# --------------------------------------------------------------------------- +# get_odds_for_games +# --------------------------------------------------------------------------- + +class TestGetOddsForGames: + def test_missing_fields_get_none_odds_without_http(self, manager, mock_get): + games = [ + {'sport': 'football'}, + {'league': 'nfl'}, + {'id': '9'}, + {}, + ] + + result = manager.get_odds_for_games(games) + + assert all(g['odds'] is None for g in result) + mock_get.assert_not_called() + + def test_per_game_exception_continues_loop(self, manager, monkeypatch): + def fake_get_odds(sport, league, event_id, + update_interval_seconds=None): + if event_id == 'bad': + raise RuntimeError('boom') + return {'spread': -1.0} + + monkeypatch.setattr(manager, 'get_odds', fake_get_odds) + games = [ + {'sport': 'football', 'league': 'nfl', 'id': 'bad'}, + {'sport': 'football', 'league': 'nfl', 'id': 'ok'}, + ] + + result = manager.get_odds_for_games(games) + + assert len(result) == 2 + assert result[0]['odds'] is None + assert result[1]['odds'] == {'spread': -1.0} + + def test_input_dicts_mutated_in_place_and_returned(self, manager, mock_get): + # Pin: get_odds_for_games mutates the caller's game dicts in place + # and returns the same objects, not copies. + game = {'sport': 'football', 'league': 'nfl', 'id': '401'} + + result = manager.get_odds_for_games([game]) + + assert result[0] is game + assert game['odds'] == FULL_EXTRACTED + + +# --------------------------------------------------------------------------- +# _load_configuration +# --------------------------------------------------------------------------- + +class TestLoadConfiguration: + def test_loads_values_from_config(self, cache_manager): + config_manager = MagicMock() + config_manager.get_config.return_value = { + 'base_odds_manager': { + 'update_interval': 100, + 'timeout': 5, + 'cache_ttl': 42, + } + } + + manager = BaseOddsManager(cache_manager, config_manager=config_manager) + + assert manager.update_interval == 100 + # Key/attr mismatch pin: the config key is 'timeout' but the + # attribute is request_timeout. + assert manager.request_timeout == 5 + assert manager.cache_ttl == 42 + + def test_get_config_raising_keeps_defaults(self, cache_manager): + config_manager = MagicMock() + config_manager.get_config.side_effect = RuntimeError('boom') + + manager = BaseOddsManager(cache_manager, config_manager=config_manager) + + assert manager.update_interval == 3600 + assert manager.request_timeout == 30 + assert manager.cache_ttl == 1800 diff --git a/test/test_base_plugin_duration.py b/test/test_base_plugin_duration.py index 4039e1fc..af7fe976 100644 --- a/test/test_base_plugin_duration.py +++ b/test/test_base_plugin_duration.py @@ -78,10 +78,20 @@ class TestInstanceVariable: instance_duration=[30]) assert plugin.get_display_duration() == 20.0 - def test_bool_true_is_one_second(self): - # Characterized quirk: bool is an int subclass, so display_duration = - # True passes the isinstance((int, float)) branch and returns 1.0. - assert make_plugin(instance_duration=True).get_display_duration() == 1.0 + def test_bool_true_falls_through_like_any_non_number(self): + # bool is an int subclass, but a boolean is not a duration: True + # must NOT read as 1 second — it falls through to config/default. + assert make_plugin(instance_duration=True).get_display_duration() == 15.0 + + def test_bool_true_falls_through_to_config(self): + plugin = make_plugin(config={"display_duration": 20}, + instance_duration=True) + assert plugin.get_display_duration() == 20.0 + + def test_bool_false_still_falls_through(self): + plugin = make_plugin(config={"display_duration": 20}, + instance_duration=False) + assert plugin.get_display_duration() == 20.0 class TestConfigFallback: @@ -108,3 +118,29 @@ class TestConfigFallback: def test_config_none_uses_default(self): assert make_plugin({"display_duration": None}).get_display_duration() == 15.0 + + def test_config_bool_uses_default(self): + assert make_plugin({"display_duration": True}).get_display_duration() == 15.0 + assert make_plugin({"display_duration": False}).get_display_duration() == 15.0 + + +class TestValidateConfigDuration: + # validate_config must agree with get_display_duration about what a + # valid duration is — a config it accepts must not then be rejected + # (or silently defaulted) when the duration is actually read. + + def test_positive_number_valid(self): + assert make_plugin({"display_duration": 20}).validate_config() is True + + def test_zero_and_negative_invalid(self): + assert make_plugin({"display_duration": 0}).validate_config() is False + assert make_plugin({"display_duration": -5}).validate_config() is False + + def test_bool_invalid(self): + # bool is an int subclass; True would otherwise pass as "positive + # number" here while get_display_duration rejects it. + assert make_plugin({"display_duration": True}).validate_config() is False + assert make_plugin({"display_duration": False}).validate_config() is False + + def test_missing_duration_valid(self): + assert make_plugin({}).validate_config() is True diff --git a/test/test_config_helper.py b/test/test_config_helper.py new file mode 100644 index 00000000..4931fccc --- /dev/null +++ b/test/test_config_helper.py @@ -0,0 +1,253 @@ +""" +Tests for src/common/config_helper.py — pins the ConfigHelper contract. + +Covers: load/save round trips (missing/malformed files return {} rather +than raising, non-ASCII preserved via ensure_ascii=False, top-level JSON +lists returned as-is), dot-notation get/set including the silent-failure +contract when an intermediate key holds a non-dict, merge_configs deep +semantics with NO aliasing of the base config (the fixed bug — the old +shallow copy let mutations of the merged result leak into base's nested +dicts), simplified schema validation including the caught-TypeError path +when a schema 'type' is given as a string, plugin config key conventions +('{plugin_id}_config', enabled defaults True), and required-key checks +where a key present with value None counts as present. +""" + +import json + +import pytest + +from src.common.config_helper import ConfigHelper + + +@pytest.fixture +def helper(): + return ConfigHelper() + + +class TestLoadConfig: + def test_missing_file_returns_empty_dict(self, helper, tmp_path): + assert helper.load_config(tmp_path / "nope.json") == {} + + def test_malformed_json_returns_empty_dict(self, helper, tmp_path): + path = tmp_path / "bad.json" + path.write_text("{ this is not json", encoding="utf-8") + assert helper.load_config(path) == {} + + def test_top_level_list_returned_as_is(self, helper, tmp_path): + # load_config does not enforce a dict shape: a JSON list comes + # straight back. Pinned as a characterization of current behavior. + path = tmp_path / "list.json" + path.write_text("[1, 2, 3]", encoding="utf-8") + assert helper.load_config(path) == [1, 2, 3] + + +class TestSaveConfig: + def test_round_trip(self, helper, tmp_path): + path = tmp_path / "config.json" + config = {'display': {'hardware': {'rows': 32}}, 'timezone': 'UTC'} + assert helper.save_config(config, path) is True + assert helper.load_config(path) == config + + def test_creates_parent_directories(self, helper, tmp_path): + path = tmp_path / "deep" / "nested" / "config.json" + assert helper.save_config({'a': 1}, path) is True + assert path.exists() + assert helper.load_config(path) == {'a': 1} + + def test_non_ascii_survives_round_trip(self, helper, tmp_path): + path = tmp_path / "config.json" + config = {'city': 'Zürich', 'note': 'météo ☀'} + assert helper.save_config(config, path) is True + assert helper.load_config(path) == config + # ensure_ascii=False: characters are written raw, not \u-escaped + assert 'Zürich' in path.read_text(encoding='utf-8') + + def test_directory_path_returns_false_not_raise(self, helper, tmp_path): + assert helper.save_config({'a': 1}, tmp_path) is False + + +class TestGetConfigValue: + def test_dot_notation_hit(self, helper): + config = {'display': {'hardware': {'rows': 32}}} + assert helper.get_config_value(config, 'display.hardware.rows') == 32 + + def test_missing_returns_default(self, helper): + sentinel = object() + assert helper.get_config_value({}, 'display.rows', default=sentinel) is sentinel + + def test_intermediate_non_dict_returns_default(self, helper): + config = {'display': 'not-a-dict'} + assert helper.get_config_value(config, 'display.hardware.rows', default=64) == 64 + + def test_required_missing_raises_keyerror(self, helper): + with pytest.raises(KeyError): + helper.get_config_value({}, 'display.rows', required=True) + + +class TestSetConfigValue: + def test_sets_top_level(self, helper): + config = {} + helper.set_config_value(config, 'timezone', 'UTC') + assert config == {'timezone': 'UTC'} + + def test_auto_creates_intermediates(self, helper): + config = {} + helper.set_config_value(config, 'display.hardware.rows', 32) + assert config == {'display': {'hardware': {'rows': 32}}} + + def test_silent_failure_on_non_dict_intermediate(self, helper): + # 'a' exists but holds an int; the assignment attempt raises + # TypeError internally, which set_config_value swallows and logs. + # The config is left unchanged — pinned silent-failure contract. + config = {'a': 5} + helper.set_config_value(config, 'a.b', 1) + assert config == {'a': 5} + + +class TestMergeConfigs: + def test_nested_dicts_merge_recursively(self, helper): + base = {'display': {'rows': 32, 'cols': 64}, 'timezone': 'UTC'} + override = {'display': {'cols': 128, 'brightness': 90}} + merged = helper.merge_configs(base, override) + assert merged == { + 'display': {'rows': 32, 'cols': 128, 'brightness': 90}, + 'timezone': 'UTC', + } + + def test_scalar_override_wins_over_dict(self, helper): + merged = helper.merge_configs({'display': {'rows': 32}}, {'display': 7}) + assert merged['display'] == 7 + + def test_dict_override_wins_over_scalar(self, helper): + merged = helper.merge_configs({'display': 7}, {'display': {'rows': 32}}) + assert merged['display'] == {'rows': 32} + + def test_no_aliasing_of_base(self, helper): + # Post-fix: merge deep-copies base, so mutating the result never + # leaks back into the caller's base config. + base = {'display': {'x': 1}} + merged = helper.merge_configs(base, {}) + assert merged['display'] is not base['display'] + merged['display']['x'] = 99 + assert base['display']['x'] == 1 + + def test_inputs_unchanged(self, helper): + base = {'a': {'b': 1}} + override = {'a': {'c': 2}} + helper.merge_configs(base, override) + assert base == {'a': {'b': 1}} + assert override == {'a': {'c': 2}} + + def test_no_aliasing_of_override_values(self, helper): + # The non-recursive branch must deep-copy the override value too: + # mutating a merged-in list or dict must not reach back into + # override_config. + override = {'teams': ['A', 'B'], 'nested': {'x': [1]}} + merged = helper.merge_configs({}, override) + merged['teams'].append('C') + merged['nested']['x'].append(2) + assert override == {'teams': ['A', 'B'], 'nested': {'x': [1]}} + + +class TestValidateConfig: + def test_no_schema_dict_is_valid(self, helper): + assert helper.validate_config({'a': 1}) is True + + def test_no_schema_list_is_invalid(self, helper): + assert helper.validate_config([1, 2]) is False + + def test_required_key_missing_is_invalid(self, helper): + schema = {'rows': {'required': True, 'type': int}} + assert helper.validate_config({}, schema) is False + + def test_optional_key_missing_is_valid(self, helper): + schema = {'rows': {'required': False, 'type': int}} + assert helper.validate_config({}, schema) is True + + def test_wrong_type_is_invalid(self, helper): + schema = {'rows': {'type': int}} + assert helper.validate_config({'rows': 'thirty-two'}, schema) is False + assert helper.validate_config({'rows': 32}, schema) is True + + def test_allowed_values_violation_is_invalid(self, helper): + schema = {'mode': {'allowed_values': ['clock', 'weather']}} + assert helper.validate_config({'mode': 'stocks'}, schema) is False + assert helper.validate_config({'mode': 'clock'}, schema) is True + + def test_string_type_in_schema_is_invalid_via_typeerror(self, helper): + # 'type' given as the STRING "int" makes isinstance() raise + # TypeError; validate_config catches it and returns False rather + # than raising. Pinned characterization. + schema = {'rows': {'type': 'int'}} + assert helper.validate_config({'rows': 32}, schema) is False + + +class TestPluginConfigHelpers: + def test_get_plugin_config_uses_suffixed_key(self, helper): + plugin_cfg = {'enabled': True, 'display_duration': 30} + assert helper.get_plugin_config({'clock_config': plugin_cfg}, 'clock') == plugin_cfg + + def test_get_plugin_config_bare_id_key_not_found(self, helper): + # Only '{plugin_id}_config' is consulted — a bare 'clock' section + # is invisible to this helper. Pinned key contract. + assert helper.get_plugin_config({'clock': {'enabled': True}}, 'clock') == {} + + def test_create_default_config_wraps_in_suffixed_key(self, helper): + defaults = {'enabled': True} + assert helper.create_default_config('clock', defaults) == {'clock_config': defaults} + + def test_is_plugin_enabled_defaults_true_for_unknown(self, helper): + assert helper.is_plugin_enabled({}, 'clock') is True + + def test_is_plugin_enabled_false_when_disabled(self, helper): + config = {'clock_config': {'enabled': False}} + assert helper.is_plugin_enabled(config, 'clock') is False + + def test_is_plugin_enabled_ignores_bare_id_key(self, helper): + # Disabled under the wrong key -> still reported enabled (default). + config = {'clock': {'enabled': False}} + assert helper.is_plugin_enabled(config, 'clock') is True + + +class TestSportsAndDisplayHelpers: + def test_get_display_config(self, helper): + display = {'hardware': {'rows': 32}} + assert helper.get_display_config({'display': display}) == display + assert helper.get_display_config({}) == {} + + def test_get_sports_config_uses_scoreboard_suffix(self, helper): + sport_cfg = {'favorite_teams': ['TB']} + config = {'football_scoreboard': sport_cfg} + assert helper.get_sports_config(config, 'football') == sport_cfg + assert helper.get_sports_config(config, 'hockey') == {} + + def test_get_favorite_teams(self, helper): + config = {'football_scoreboard': {'favorite_teams': ['TB', 'DAL']}} + assert helper.get_favorite_teams(config, 'football') == ['TB', 'DAL'] + assert helper.get_favorite_teams({}, 'football') == [] + + def test_get_display_modes(self, helper): + modes = {'live': True, 'recent': False} + config = {'football_scoreboard': {'display_modes': modes}} + assert helper.get_display_modes(config, 'football') == modes + assert helper.get_display_modes({}, 'football') == {} + + +class TestValidateRequiredKeys: + def test_returns_missing_subset(self, helper): + config = {'a': 1, 'c': {'d': 2}} + missing = helper.validate_required_keys(config, ['a', 'b', 'c.d', 'c.e']) + assert missing == ['b', 'c.e'] + + def test_dot_notation_present(self, helper): + config = {'display': {'hardware': {'rows': 32}}} + assert helper.validate_required_keys(config, ['display.hardware.rows']) == [] + + def test_empty_requirements(self, helper): + assert helper.validate_required_keys({'a': 1}, []) == [] + + def test_present_with_none_counts_as_present(self, helper): + # _has_key checks key membership, not truthiness — a key set to + # None is NOT reported missing. Pinned semantics. + assert helper.validate_required_keys({'a': None}, ['a']) == [] diff --git a/test/test_config_manager_secrets.py b/test/test_config_manager_secrets.py index 94c7a6a2..f7070c9c 100644 --- a/test/test_config_manager_secrets.py +++ b/test/test_config_manager_secrets.py @@ -177,3 +177,159 @@ class TestLoadFastPath: os.utime(config_file, ns=(1_000_000_000, 1_000_000_000)) assert manager.load_config()["timezone"] == "AAA" # stale, by design + + +class TestArraySecretStripAndMerge: + """Array-item secrets round-trip (parallel-placeholder lists). + + secret_helpers.separate_secrets emits array secrets as a list parallel + to the regular list, with {} for items that carry no secrets. Strip + must remove the secret fields from config.json while preserving item + indices; load must merge them back into the right items. The regular + list's length is authoritative in both directions. + """ + + def test_strip_removes_array_item_secrets_keeps_indices(self, tmp_path): + manager = make_manager(tmp_path) + data = {"plugin": {"accounts": [ + {"name": "a", "token": "ta"}, + {"name": "b"}, + ]}} + secrets = {"plugin": {"accounts": [{"token": "ta"}, {}]}} + stripped = manager._strip_secrets_recursive(data, secrets) + assert stripped == {"plugin": {"accounts": [{"name": "a"}, {"name": "b"}]}} + + def test_strip_keeps_all_placeholder_items(self, tmp_path): + # Even when every item strips to nothing extra, the list survives + # with its indices — required for merge-on-load alignment. + manager = make_manager(tmp_path) + data = {"accounts": [{"token": "t1"}, {"token": "t2"}]} + secrets = {"accounts": [{"token": "t1"}, {"token": "t2"}]} + stripped = manager._strip_secrets_recursive(data, secrets) + assert stripped == {"accounts": [{}, {}]} + + def test_strip_whole_scalar_array_secret_drops_key(self, tmp_path): + # A list of secret scalars is a whole-key secret, not the parallel + # shape — the key must vanish from config.json entirely. + manager = make_manager(tmp_path) + data = {"recovery_codes": ["a", "b"], "city": "Austin"} + secrets = {"recovery_codes": ["a", "b"]} + stripped = manager._strip_secrets_recursive(data, secrets) + assert stripped == {"city": "Austin"} + + def test_strip_shape_mismatch_drops_key(self, tmp_path): + # Conservative contract: if the shapes disagree, never leak. + manager = make_manager(tmp_path) + data = {"accounts": {"name": "not-a-list"}} + secrets = {"accounts": [{"token": "t"}]} + stripped = manager._strip_secrets_recursive(data, secrets) + assert stripped == {} + + def test_strip_ignores_extra_secrets_entries(self, tmp_path): + # Regular list length is authoritative: a user deleted an item. + manager = make_manager(tmp_path) + data = {"accounts": [{"name": "a", "token": "ta"}]} + secrets = {"accounts": [{"token": "ta"}, {"token": "tb"}]} + stripped = manager._strip_secrets_recursive(data, secrets) + assert stripped == {"accounts": [{"name": "a"}]} + + def test_merge_restores_array_item_secrets(self, tmp_path): + manager = make_manager(tmp_path) + target = {"accounts": [{"name": "a"}, {"name": "b"}]} + manager._deep_merge(target, {"accounts": [{"token": "ta"}, {}]}) + assert target == {"accounts": [ + {"name": "a", "token": "ta"}, + {"name": "b"}, + ]} + + def test_merge_ignores_extra_secrets_entries_with_warning(self, tmp_path, caplog): + manager = make_manager(tmp_path) + target = {"accounts": [{"name": "a"}]} + with caplog.at_level("WARNING"): + manager._deep_merge( + target, {"accounts": [{"token": "ta"}, {"token": "ghost"}]}) + assert target == {"accounts": [{"name": "a", "token": "ta"}]} + assert any("longer than the config list" in r.message for r in caplog.records) + + def test_merge_non_dict_item_replaced_by_secret(self, tmp_path): + # Shape drift inside the list: the secret wins for that index. + manager = make_manager(tmp_path) + target = {"accounts": ["oddball", {"name": "b"}]} + manager._deep_merge(target, {"accounts": [{"token": "ta"}, {}]}) + assert target == {"accounts": [{"token": "ta"}, {"name": "b"}]} + + def test_merge_whole_scalar_array_still_replaces(self, tmp_path): + # Legacy behavior preserved: a non-parallel list replaces wholesale. + manager = make_manager(tmp_path) + target = {"recovery_codes": ["old"]} + manager._deep_merge(target, {"recovery_codes": ["new1", "new2"]}) + assert target == {"recovery_codes": ["new1", "new2"]} + + def test_full_save_load_round_trip(self, tmp_path): + # End to end on real files: save strips array secrets out of + # config.json; load merges them back into the right items. + manager = make_manager( + tmp_path, + config={"plugin": {"accounts": [ + {"name": "a", "token": "s3cret-a"}, + {"name": "b", "token": "s3cret-b"}, + ]}}, + secrets={"plugin": {"accounts": [ + {"token": "s3cret-a"}, {"token": "s3cret-b"}, + ]}}, + ) + loaded = manager.load_config() + assert loaded["plugin"]["accounts"][0]["token"] == "s3cret-a" + + manager.save_config(loaded) + + raw = (tmp_path / "config.json").read_text() + assert "s3cret" not in raw + on_disk = json.loads(raw) + assert on_disk["plugin"]["accounts"] == [{"name": "a"}, {"name": "b"}] + + # A fresh manager (constructed directly — make_manager would + # overwrite the just-saved config.json) re-merges from the secrets + # file on load. + fresh = ConfigManager(config_path=str(tmp_path / "config.json"), + secrets_path=str(tmp_path / "config_secrets.json")) + fresh.template_path = str(tmp_path / "no-template.json") + reloaded = fresh.load_config() + assert reloaded["plugin"]["accounts"] == [ + {"name": "a", "token": "s3cret-a"}, + {"name": "b", "token": "s3cret-b"}, + ] + + def test_whole_item_secret_list_never_leaks_values(self, tmp_path): + # When the ENTIRE array item is secret (schema marks both key[] + # and key[].field), separate_secrets stores the full item dicts in + # the secrets file. That shape also matches the parallel-list + # discriminator — which is safe: strip drops every leaf key that + # appears in the secret item, so only empty {} skeletons (item + # count, no values) can reach config.json, and merge-on-load + # restores the full items from those skeletons. + from src.web_interface.secret_helpers import ( + find_secret_fields, separate_secrets) + schema_props = {"accounts": { + "type": "array", + "items": {"type": "object", "x-secret": True, "properties": { + "id": {"type": "string"}, + "token": {"type": "string", "x-secret": True}, + }}, + }} + paths = find_secret_fields(schema_props) + assert paths == {"accounts[]", "accounts[].token"} + full = {"accounts": [{"id": "i1", "token": "s3cret-a"}, + {"id": "i2", "token": "s3cret-b"}]} + _, secrets = separate_secrets(full, paths) + assert secrets == full # whole items are secret + + manager = make_manager(tmp_path) + stripped = manager._strip_secrets_recursive(full, secrets) + assert stripped == {"accounts": [{}, {}]} + + raw = json.dumps(stripped) + assert "s3cret" not in raw and "i1" not in raw + + manager._deep_merge(stripped, secrets) + assert stripped == full # round trip restores the items diff --git a/test/test_display_controller.py b/test/test_display_controller.py index 8f89c31f..da551b93 100644 --- a/test/test_display_controller.py +++ b/test/test_display_controller.py @@ -338,31 +338,48 @@ class TestDisplayControllerSchedule: """Test schedule management.""" def test_schedule_disabled(self, test_display_controller): - """Test when schedule is disabled.""" + """schedule.enabled=False keeps the display active even outside the + configured window. (This test used to patch config_service, which + _check_schedule never reads — it asserted the init default.)""" controller = test_display_controller - schedule_config = {"schedule": {"enabled": False}} - with patch.object(controller.config_service, 'get_config', return_value=schedule_config): + controller.config['schedule'] = { + "enabled": False, + "start_time": "09:00", + "end_time": "17:00", + } + controller._schedule_checked_minute = None + controller._tz = None + controller.is_display_active = False # prove the method flips it back + + with patch('src.display_controller.datetime') as mock_datetime: + mock_datetime.now.return_value.strftime.return_value.lower.return_value = "monday" + mock_datetime.now.return_value.time.return_value = datetime.strptime("20:00", "%H:%M").time() + mock_datetime.strptime = datetime.strptime + controller._check_schedule() assert controller.is_display_active is True def test_active_hours(self, test_display_controller): - """Test active hours check.""" + """A time inside the window activates the display. (This test used + to patch config_service, which _check_schedule never reads — it + asserted the init default.)""" controller = test_display_controller + controller.config['schedule'] = { + "enabled": True, + "start_time": "09:00", + "end_time": "17:00", + } + controller._schedule_checked_minute = None + controller._tz = None + controller.is_display_active = False # prove the method flips it on + with patch('src.display_controller.datetime') as mock_datetime: mock_datetime.now.return_value.strftime.return_value.lower.return_value = "monday" mock_datetime.now.return_value.time.return_value = datetime.strptime("12:00", "%H:%M").time() mock_datetime.strptime = datetime.strptime - schedule_config = { - "schedule": { - "enabled": True, - "start_time": "09:00", - "end_time": "17:00" - } - } - with patch.object(controller.config_service, 'get_config', return_value=schedule_config): - controller._check_schedule() - assert controller.is_display_active is True + controller._check_schedule() + assert controller.is_display_active is True def test_inactive_hours(self, test_display_controller): """Test inactive hours check.""" diff --git a/test/test_display_controller_schedule.py b/test/test_display_controller_schedule.py new file mode 100644 index 00000000..a7237d4b --- /dev/null +++ b/test/test_display_controller_schedule.py @@ -0,0 +1,277 @@ +""" +Behavioral tests for DisplayController._check_schedule and +_check_dim_schedule — the on/off window and night-dimming logic. + +test_display_controller_optimizations.py::TestScheduleMinuteGate already +covers the once-per-minute gating; this file covers what it doesn't: +midnight-crossing windows, mode selection (global / per-day / legacy +inference), per-day disabled days, invalid time strings, unknown +timezones, boundary equality, and the transition-tracking flags. + +Both methods read only self.config and a handful of instance attributes, +so a bare stub via object.__new__ (the test_display_controller_vegas_tick +pattern) is enough — no managers needed. +""" + +import os +from datetime import datetime + +from unittest.mock import patch + +import pytest + +os.environ.setdefault("EMULATOR", "true") + +from src.display_controller import DisplayController # noqa: E402 + + +def make_controller(config=None, *, normal_brightness=90): + dc = object.__new__(DisplayController) + dc.config = config or {} + dc._tz = None + dc._schedule_checked_minute = None + dc.is_display_active = True + dc._was_display_active = True + dc._normal_brightness = normal_brightness + dc._dim_checked_minute = None + dc._cached_target_brightness = None + dc.is_dimmed = False + dc._was_dimmed = False + return dc + + +def at(time_str, day="monday"): + """Context manager patching the controller module's clock.""" + patcher = patch("src.display_controller.datetime") + mock_dt = patcher.start() + mock_dt.strptime = datetime.strptime + mock_dt.now.return_value.time.return_value = ( + datetime.strptime(time_str, "%H:%M").time()) + mock_dt.now.return_value.strftime.return_value.lower.return_value = day + mock_dt.now.return_value.hour = int(time_str.split(":")[0]) + mock_dt.now.return_value.minute = int(time_str.split(":")[1]) + return patcher + + +@pytest.fixture +def clock(): + patchers = [] + + def _at(time_str, day="monday"): + patchers.append(p := at(time_str, day)) + return p + + yield _at + for p in patchers: + p.stop() + + +def check_at(dc, time_str, day="monday", clock=None): + """Run _check_schedule at a mocked wall time, resetting the minute gate.""" + dc._schedule_checked_minute = None + p = at(time_str, day) + try: + dc._check_schedule() + finally: + p.stop() + return dc.is_display_active + + +def dim_at(dc, time_str, day="monday"): + dc._dim_checked_minute = None + p = at(time_str, day) + try: + return dc._check_dim_schedule() + finally: + p.stop() + + +class TestScheduleWindows: + def _config(self, start, end, **extra): + return {"schedule": {"enabled": True, "start_time": start, + "end_time": end, **extra}, + "timezone": "UTC"} + + def test_same_day_window(self): + dc = make_controller(self._config("09:00", "17:00")) + assert check_at(dc, "12:00") is True + assert check_at(dc, "20:00") is False + assert check_at(dc, "08:59") is False + + def test_boundaries_are_inclusive(self): + dc = make_controller(self._config("09:00", "17:00")) + assert check_at(dc, "09:00") is True # now == start + assert check_at(dc, "17:00") is True # now == end + + def test_midnight_crossing_window(self): + # 21:00 -> 07:00: active late evening AND early morning, inactive + # mid-day. + dc = make_controller(self._config("21:00", "07:00")) + assert check_at(dc, "23:00") is True + assert check_at(dc, "03:00") is True + assert check_at(dc, "12:00") is False + assert check_at(dc, "21:00") is True # boundary + assert check_at(dc, "07:00") is True # boundary + + def test_no_schedule_config_is_always_active(self): + dc = make_controller({"timezone": "UTC"}) + dc.is_display_active = False + dc._check_schedule() + assert dc.is_display_active is True + + def test_invalid_time_string_falls_back_to_active(self): + dc = make_controller(self._config("9 o'clock", "17:00")) + dc.is_display_active = False + assert check_at(dc, "03:00") is True # ValueError -> stay on + + def test_unknown_timezone_falls_back_to_utc(self): + dc = make_controller({"schedule": {"enabled": True, + "start_time": "09:00", + "end_time": "17:00"}, + "timezone": "Mars/Olympus_Mons"}) + assert check_at(dc, "12:00") is True + import pytz + assert dc._tz is pytz.UTC + + +class TestScheduleModes: + DAYS = { + "monday": {"enabled": True, "start_time": "10:00", + "end_time": "18:00"}, + "tuesday": {"enabled": False}, + } + + def test_global_mode_ignores_days(self): + dc = make_controller({"schedule": { + "enabled": True, "mode": "global", + "start_time": "09:00", "end_time": "17:00", + "days": self.DAYS}, "timezone": "UTC"}) + # 09:30 is inside the global window but outside monday's per-day one. + assert check_at(dc, "09:30", day="monday") is True + + def test_per_day_mode_uses_day_window(self): + dc = make_controller({"schedule": { + "enabled": True, "mode": "per-day", + "start_time": "09:00", "end_time": "17:00", + "days": self.DAYS}, "timezone": "UTC"}) + assert check_at(dc, "09:30", day="monday") is False # before 10:00 + assert check_at(dc, "12:00", day="monday") is True + + def test_per_day_underscore_spelling_accepted(self): + dc = make_controller({"schedule": { + "enabled": True, "mode": "per_day", + "days": self.DAYS}, "timezone": "UTC"}) + assert check_at(dc, "12:00", day="monday") is True + + def test_legacy_no_mode_infers_per_day_from_days_config(self): + dc = make_controller({"schedule": { + "enabled": True, + "start_time": "09:00", "end_time": "17:00", + "days": self.DAYS}, "timezone": "UTC"}) + assert check_at(dc, "09:30", day="monday") is False # per-day won + + def test_per_day_disabled_day_turns_display_off(self): + dc = make_controller({"schedule": { + "enabled": True, "mode": "per-day", + "days": self.DAYS}, "timezone": "UTC"}) + assert check_at(dc, "12:00", day="tuesday") is False + + def test_per_day_missing_day_falls_back_to_global(self): + dc = make_controller({"schedule": { + "enabled": True, "mode": "per-day", + "start_time": "09:00", "end_time": "17:00", + "days": self.DAYS}, "timezone": "UTC"}) + # Wednesday has no per-day entry -> global window applies. + assert check_at(dc, "09:30", day="wednesday") is True + + def test_missing_enabled_key_means_enabled(self): + # Backward compat: schedules written before the enabled flag. + dc = make_controller({"schedule": { + "start_time": "09:00", "end_time": "17:00"}, "timezone": "UTC"}) + assert check_at(dc, "20:00") is False + + +class TestScheduleTransitions: + def test_was_display_active_tracks_state(self): + dc = make_controller({"schedule": {"enabled": True, + "start_time": "09:00", + "end_time": "17:00"}, + "timezone": "UTC"}) + check_at(dc, "12:00") + assert dc._was_display_active is True + check_at(dc, "20:00") + assert dc._was_display_active is False + check_at(dc, "12:05") + assert dc._was_display_active is True + + +class TestDimSchedule: + def _config(self, start="20:00", end="07:00", **extra): + return {"dim_schedule": {"enabled": True, "start_time": start, + "end_time": end, "dim_brightness": 25, + **extra}, + "timezone": "UTC"} + + def test_disabled_by_default(self): + dc = make_controller({"dim_schedule": {"start_time": "20:00", + "end_time": "07:00"}, + "timezone": "UTC"}) + # Unlike the on/off schedule, dimming defaults to DISABLED when the + # enabled key is missing. + assert dim_at(dc, "23:00") == 90 + assert dc.is_dimmed is False + + def test_overnight_dim_window(self): + dc = make_controller(self._config()) + assert dim_at(dc, "23:00") == 25 + assert dc.is_dimmed is True + assert dim_at(dc, "03:00") == 25 + assert dim_at(dc, "12:00") == 90 + assert dc.is_dimmed is False + + def test_dim_brightness_defaults_to_30(self): + dc = make_controller({"dim_schedule": {"enabled": True, + "start_time": "20:00", + "end_time": "07:00"}, + "timezone": "UTC"}) + assert dim_at(dc, "23:00") == 30 + + def test_inactive_display_short_circuits_undimmed(self): + dc = make_controller(self._config()) + dc.is_display_active = False + dc.is_dimmed = True + assert dim_at(dc, "23:00") == 90 + assert dc.is_dimmed is False + + def test_per_day_mode(self): + dc = make_controller(self._config(mode="per-day", days={ + "monday": {"enabled": True, "start_time": "22:00", + "end_time": "06:00"}, + "tuesday": {"enabled": False}, + })) + assert dim_at(dc, "23:00", day="monday") == 25 + assert dim_at(dc, "21:00", day="monday") == 90 # before per-day start + assert dim_at(dc, "23:00", day="tuesday") == 90 # day disabled + assert dc.is_dimmed is False + + def test_no_legacy_inference_for_dim(self): + # Unlike _check_schedule, dim mode defaults to GLOBAL even when a + # days config exists — no legacy inference. + dc = make_controller(self._config(days={ + "monday": {"enabled": True, "start_time": "22:00", + "end_time": "06:00"}, + })) + # 21:00 is inside the global 20:00-07:00 window but outside monday's + # per-day 22:00 start; global mode wins. + assert dim_at(dc, "21:00", day="monday") == 25 + + def test_invalid_time_string_returns_normal(self): + dc = make_controller(self._config(start="late")) + assert dim_at(dc, "23:00") == 90 + + def test_was_dimmed_tracks_transitions(self): + dc = make_controller(self._config()) + dim_at(dc, "23:00") + assert dc._was_dimmed is True + dim_at(dc, "12:00") + assert dc._was_dimmed is False diff --git a/test/test_display_helper.py b/test/test_display_helper.py new file mode 100644 index 00000000..b8c9be63 --- /dev/null +++ b/test/test_display_helper.py @@ -0,0 +1,307 @@ +"""Tests for src/common/display_helper.py (DisplayHelper). + +Pure-PIL tests, no hardware or mocks required. Pixel assertions rely on +getbbox()/getpixel() rather than exact text pixel counts, because the +default-font metrics vary across Pillow versions. + +These tests pin the FIXED behaviors on this branch: +- draw_error_message / draw_no_data_message return a rendered image + (they previously crashed with AttributeError), +- draw_scorebug_layout draws period/status/clock as one combined top + line (previously overprinted at the same y), +- draw_ticker_layout draws at x=0 (previously started at + x=display_width, i.e. entirely off-canvas -> blank frames). +""" + +from PIL import Image, ImageDraw, ImageFont + +from src.common.display_helper import DisplayHelper + + +def default_font(): + return ImageFont.load_default() + + +def make_helper(width=128, height=32): + return DisplayHelper(width, height) + + +class TestCreateBaseImage: + def test_default_is_black_rgb_display_sized(self): + helper = make_helper() + img = helper.create_base_image() + assert img.size == (128, 32) + assert img.mode == 'RGB' + assert img.getpixel((0, 0)) == (0, 0, 0) + assert img.getpixel((127, 31)) == (0, 0, 0) + # Entirely black -> no bounding box in luminance + assert img.convert('L').getbbox() is None + + def test_custom_background_color(self): + helper = make_helper() + img = helper.create_base_image(background_color=(10, 20, 30)) + assert img.getpixel((0, 0)) == (10, 20, 30) + assert img.getpixel((64, 16)) == (10, 20, 30) + + def test_mode_rgba_is_honored(self): + helper = make_helper() + img = helper.create_base_image(mode='RGBA') + assert img.mode == 'RGBA' + assert img.size == (128, 32) + + +class TestCreateOverlay: + def test_overlay_is_transparent_rgba(self): + helper = make_helper() + overlay = helper.create_overlay() + assert overlay.mode == 'RGBA' + assert overlay.size == (128, 32) + assert overlay.getpixel((0, 0)) == (0, 0, 0, 0) + assert overlay.getpixel((127, 31)) == (0, 0, 0, 0) + + +class TestCompositeImages: + def test_rgb_inputs_are_upconverted_and_result_is_rgba(self): + helper = make_helper() + base = Image.new('RGB', (128, 32), (0, 0, 0)) + overlay = Image.new('RGB', (128, 32), (255, 0, 0)) + result = helper.composite_images(base, overlay) + assert result.mode == 'RGBA' + assert result.size == base.size + # RGB->RGBA conversion yields a fully opaque overlay + assert result.getpixel((0, 0)) == (255, 0, 0, 255) + + def test_transparent_overlay_leaves_base_visible(self): + helper = make_helper() + base = Image.new('RGB', (128, 32), (5, 6, 7)) + overlay = helper.create_overlay() + result = helper.composite_images(base, overlay) + assert result.mode == 'RGBA' + assert result.getpixel((64, 16)) == (5, 6, 7, 255) + + +class TestScorebugLayout: + def test_full_game_data_renders(self): + helper = make_helper() + font = default_font() + fonts = {'time': font, 'status': font, 'score': font, 'team': font} + game_data = { + 'home_score': 3, 'away_score': 2, + 'home_abbr': 'NYY', 'away_abbr': 'BOS', + 'status_text': 'LIVE', 'period_text': 'T9', 'clock': '2:30', + } + img = helper.draw_scorebug_layout(game_data, fonts) + assert img.mode == 'RGB' + assert img.size == (128, 32) + assert img.convert('L').getbbox() is not None + + def test_empty_game_data_uses_defaults_without_raising(self): + helper = make_helper() + font = default_font() + fonts = {'time': font, 'status': font, 'score': font, 'team': font} + img = helper.draw_scorebug_layout({}, fonts) + assert img.mode == 'RGB' + assert img.size == (128, 32) + # Defaults '0'/'HOME'/'AWAY' actually render something + assert img.convert('L').getbbox() is not None + + def test_empty_fonts_dict_falls_back_to_default_font(self): + # Pin: fonts={} must not raise — PIL falls back to the default + # font when font=None is passed through. + helper = make_helper() + img = helper.draw_scorebug_layout( + {'status_text': 'FINAL', 'period_text': 'Q4', 'clock': '0:00'}, {}) + assert img.size == (128, 32) + assert img.convert('L').getbbox() is not None + + def test_top_line_is_one_combined_centered_draw(self): + # FIXED behavior: period/status/clock are joined into a single + # top line drawn once at y=1 instead of three overprinted draws. + helper = make_helper() + calls = [] + original = helper._draw_centered_text + + def spy(draw, text, font, y_position): + calls.append({'text': text, 'y_position': y_position}) + original(draw, text, font, y_position) + + helper._draw_centered_text = spy + font = default_font() + fonts = {'time': font, 'status': font, 'score': font, 'team': font} + helper.draw_scorebug_layout( + {'period_text': 'Q4', 'status_text': 'LIVE', 'clock': '2:30'}, + fonts) + + top_calls = [c for c in calls if c['y_position'] == 1] + assert len(top_calls) == 1 + text = top_calls[0]['text'] + assert 'Q4' in text + assert 'LIVE' in text + assert '2:30' in text + + def test_no_top_line_when_all_parts_empty(self): + helper = make_helper() + calls = [] + original = helper._draw_centered_text + + def spy(draw, text, font, y_position): + calls.append(y_position) + original(draw, text, font, y_position) + + helper._draw_centered_text = spy + font = default_font() + helper.draw_scorebug_layout({}, {'score': font, 'team': font}) + assert 1 not in calls # no combined top line drawn + + def test_logo_positions_bleed_off_edges(self): + # Home logo pastes at x = width - logo.width + 10 (right edge, + # bleeding off-screen right); away at x = -10 (bleeding left). + helper = make_helper() + home_logo = Image.new('RGBA', (20, 20), (0, 0, 255, 255)) # blue + away_logo = Image.new('RGBA', (20, 20), (255, 0, 0, 255)) # red + # Empty abbrs/status so text can't land on the probed pixels. + game_data = {'home_abbr': '', 'away_abbr': ''} + font = default_font() + img = helper.draw_scorebug_layout(game_data, {'score': font}, + home_logo=home_logo, + away_logo=away_logo) + # center_y = 16; logos span y 6..25 -> probe y=16 at both edges. + assert img.getpixel((0, 16)) == (255, 0, 0) # away (left edge) + assert img.getpixel((127, 16)) == (0, 0, 255) # home (right edge) + # And the off-screen parts are truly clipped: image is still 128 wide + assert img.size == (128, 32) + + +class TestTickerLayout: + def test_frame_is_not_blank(self): + # FIXED behavior: text now starts at x=0. Previously it was drawn + # at x=display_width, entirely off-canvas, so frames were blank. + helper = make_helper() + img = helper.draw_ticker_layout('HELLO WORLD', default_font()) + assert img.size == (128, 32) + assert img.mode == 'RGB' + assert img.convert('L').getbbox() is not None + + def test_text_starts_at_left_edge(self): + helper = make_helper() + img = helper.draw_ticker_layout('HELLO', default_font()) + bbox = img.convert('L').getbbox() + assert bbox is not None + # Text is positioned at x=0 (outline extends 1px left, clipped), + # so ink begins hugging the left edge. Allow a couple of pixels of + # slack for font-dependent left-side bearing. + assert bbox[0] <= 2 + + def test_scroll_speed_does_not_affect_frame(self): + # Pin: scroll_speed is accepted for API compatibility only. + helper = make_helper() + font = default_font() + img1 = helper.draw_ticker_layout('SCROLLING', font, scroll_speed=1) + img5 = helper.draw_ticker_layout('SCROLLING', font, scroll_speed=5) + assert img1.tobytes() == img5.tobytes() + + def test_custom_colors(self): + helper = make_helper() + img = helper.draw_ticker_layout('X', default_font(), + background_color=(0, 0, 40), + text_color=(0, 255, 0)) + assert img.getpixel((127, 0)) == (0, 0, 40) # background corner + colors = {img.getpixel((x, y)) + for x in range(img.width) for y in range(img.height)} + # Text color appears somewhere (anti-aliasing may blend it, so + # check for a green-dominant pixel rather than the exact color). + assert any(g > 150 and r < 100 for (r, g, b) in colors) + + +class TestCenteredText: + def test_renders_centered_text_on_background(self): + helper = make_helper() + img = helper.draw_centered_text('HI', default_font(), + background_color=(0, 0, 60), + text_color=(255, 255, 0)) + assert img.size == (128, 32) + assert img.convert('L').getbbox() is not None + # Corners stay pure background + assert img.getpixel((0, 0)) == (0, 0, 60) + assert img.getpixel((127, 0)) == (0, 0, 60) + assert img.getpixel((0, 31)) == (0, 0, 60) + assert img.getpixel((127, 31)) == (0, 0, 60) + + +class TestErrorAndNoDataMessages: + def test_draw_error_message_returns_rendered_image(self): + # FIXED behavior: used to crash with AttributeError; now returns + # a rendered image on a dark red background. + helper = make_helper() + img = helper.draw_error_message('Boom') + assert img.size == (128, 32) + assert img.mode == 'RGB' + assert img.convert('L').getbbox() is not None + assert img.getpixel((0, 0)) == (50, 0, 0) # dark red background + + def test_draw_error_message_default_text(self): + helper = make_helper() + img = helper.draw_error_message() + assert img.size == (128, 32) + assert img.getpixel((127, 31)) == (50, 0, 0) + + def test_draw_no_data_message_returns_rendered_image(self): + helper = make_helper() + img = helper.draw_no_data_message() + assert img.size == (128, 32) + assert img.mode == 'RGB' + assert img.convert('L').getbbox() is not None + assert img.getpixel((0, 0)) == (0, 0, 0) # black background + + +class TestDrawTextWithOutline: + def test_fill_color_appears_in_output(self): + helper = make_helper() + img = Image.new('RGB', (40, 20), (0, 0, 255)) + draw = ImageDraw.Draw(img) + helper._draw_text_with_outline(draw, 'X', (5, 2), default_font(), + fill=(255, 0, 0)) + pixels = {img.getpixel((x, y)) + for x in range(img.width) for y in range(img.height)} + # Anti-aliased fonts blend edge pixels, so look for red-dominant + # (fill) and near-black (outline) pixels rather than exact colors. + assert any(r > 150 and g < 50 for (r, g, b) in pixels) # fill + assert any(max(p) < 80 for p in pixels) # outline + + def test_default_fill_is_white(self): + helper = make_helper() + img = Image.new('RGB', (40, 20), (0, 0, 255)) + draw = ImageDraw.Draw(img) + helper._draw_text_with_outline(draw, 'X', (5, 2), default_font()) + pixels = {img.getpixel((x, y)) + for x in range(img.width) for y in range(img.height)} + # White-dominant pixel present (exact white may be anti-aliased) + assert any(r > 200 and g > 200 for (r, g, b) in pixels) + + +class TestOrientationAndDimensions: + def test_landscape_display(self): + helper = DisplayHelper(128, 32) + assert helper.is_landscape() is True + assert helper.is_portrait() is False + + def test_portrait_display(self): + helper = DisplayHelper(32, 128) + assert helper.is_portrait() is True + assert helper.is_landscape() is False + + def test_square_display_is_neither(self): + # Pin: a square display is neither portrait nor landscape. + helper = DisplayHelper(64, 64) + assert helper.is_portrait() is False + assert helper.is_landscape() is False + + def test_get_center_position(self): + assert DisplayHelper(128, 32).get_center_position() == (64, 16) + + def test_get_center_position_floors_odd_dimensions(self): + assert DisplayHelper(65, 33).get_center_position() == (32, 16) + + def test_get_display_dimensions(self): + assert DisplayHelper(128, 32).get_display_dimensions() == (128, 32) + assert DisplayHelper(64, 64).get_display_dimensions() == (64, 64) diff --git a/test/test_dynamic_team_resolver.py b/test/test_dynamic_team_resolver.py new file mode 100644 index 00000000..c562d0d9 --- /dev/null +++ b/test/test_dynamic_team_resolver.py @@ -0,0 +1,259 @@ +""" +Tests for src/dynamic_team_resolver.py (DynamicTeamResolver). + +Covers dynamic team expansion (AP_TOP_5/10/25), order-preserving dedup, +unknown dynamic-name dropping, rankings parsing, the fixed genuinely +class-shared rankings cache (fetch and clear_cache write through +DynamicTeamResolver._rankings_cache / _cache_timestamp), TTL expiry, +network-failure resilience, and the resolve_dynamic_teams module function. + +No real network: src.dynamic_team_resolver.requests.get is always patched. +""" + +import types +from unittest.mock import MagicMock, patch + +import pytest +import requests + +import src.dynamic_team_resolver as dtr_module +from src.dynamic_team_resolver import DynamicTeamResolver, resolve_dynamic_teams + + +TOP_TEAMS = ['UGA', 'MICH', 'OSU', 'TEX', 'ALA', 'ORE', 'PSU', 'ND', 'FSU', 'OU'] + + +def _rankings_payload(teams=None): + teams = TOP_TEAMS if teams is None else teams + return { + 'rankings': [{ + 'name': 'AP Top 25', + 'ranks': [ + {'current': i + 1, 'team': {'abbreviation': abbr}} + for i, abbr in enumerate(teams) + ], + }] + } + + +def _make_response(payload): + response = MagicMock() + response.json.return_value = payload + response.raise_for_status.return_value = None + return response + + +@pytest.fixture(autouse=True) +def reset_class_cache(): + """Reset the CLASS-level shared cache between tests.""" + DynamicTeamResolver._rankings_cache = {} + DynamicTeamResolver._cache_timestamp = 0 + yield + DynamicTeamResolver._rankings_cache = {} + DynamicTeamResolver._cache_timestamp = 0 + + +@pytest.fixture +def mock_get(): + with patch('src.dynamic_team_resolver.requests.get') as m: + m.return_value = _make_response(_rankings_payload()) + yield m + + +@pytest.fixture +def resolver(): + return DynamicTeamResolver() + + +# --------------------------------------------------------------------------- +# resolve_teams basics +# --------------------------------------------------------------------------- + +class TestResolveTeamsBasics: + def test_empty_list_returns_empty_no_http(self, resolver, mock_get): + assert resolver.resolve_teams([]) == [] + mock_get.assert_not_called() + + def test_no_dynamic_names_passthrough_no_http(self, resolver, mock_get): + assert resolver.resolve_teams(['UGA', 'AUB', 'LSU']) == [ + 'UGA', 'AUB', 'LSU'] + mock_get.assert_not_called() + + def test_expansion_inserted_in_place_order_preserved( + self, resolver, mock_get): + result = resolver.resolve_teams(['UGA', 'AP_TOP_5', 'AUB']) + + # UGA is also ranked #1, so dedup keeps its first occurrence; the + # top-5 expansion lands where AP_TOP_5 appeared, AUB stays after. + assert result == ['UGA', 'MICH', 'OSU', 'TEX', 'ALA', 'AUB'] + + def test_order_preserving_dedup(self, resolver, mock_get): + result = resolver.resolve_teams(['UGA', 'AP_TOP_5', 'UGA']) + + assert result == ['UGA', 'MICH', 'OSU', 'TEX', 'ALA'] + assert result.count('UGA') == 1 + assert result[0] == 'UGA' + + +# --------------------------------------------------------------------------- +# AP_TOP_N slicing +# --------------------------------------------------------------------------- + +class TestSlicing: + def test_top_n_counts_and_order(self, resolver, mock_get): + teams_25 = [f'T{i:02d}' for i in range(1, 26)] + mock_get.return_value = _make_response(_rankings_payload(teams_25)) + + top5 = resolver.resolve_teams(['AP_TOP_5']) + top10 = resolver.resolve_teams(['AP_TOP_10']) + top25 = resolver.resolve_teams(['AP_TOP_25']) + + assert top5 == teams_25[:5] + assert top10 == teams_25[:10] + assert top25 == teams_25 + + +# --------------------------------------------------------------------------- +# Unknown dynamic-looking names +# --------------------------------------------------------------------------- + +class TestUnknownDynamicNames: + def test_unknown_dynamic_looking_names_dropped(self, resolver, mock_get): + result = resolver.resolve_teams( + ['AP_TOP_100', 'TOP_10', 'RANKED_ALL', 'PLAYOFF_TEAMS']) + + assert result == [] + mock_get.assert_not_called() + + def test_top_substring_hazard(self, resolver, mock_get): + # Hazard pin: _is_potential_dynamic_team matches the substring + # 'TOP_' anywhere in the (upper-cased) name, so a team literally + # named 'TOP_GUN' is dropped as an unknown dynamic team too. + assert resolver.resolve_teams(['TOP_GUN']) == [] + + +# --------------------------------------------------------------------------- +# Rankings parsing +# --------------------------------------------------------------------------- + +class TestRankingsParsing: + def test_drops_zero_rank_and_empty_abbreviation_sorts_ascending( + self, resolver, mock_get): + payload = { + 'rankings': [{ + 'name': 'AP Top 25', + 'ranks': [ + {'current': 3, 'team': {'abbreviation': 'C3'}}, + {'current': 1, 'team': {'abbreviation': 'A1'}}, + {'current': 0, 'team': {'abbreviation': 'ZERO'}}, + {'current': 4, 'team': {'abbreviation': ''}}, + {'current': 2, 'team': {'abbreviation': 'B2'}}, + ], + }] + } + mock_get.return_value = _make_response(payload) + + rankings = resolver._fetch_ncaa_fb_rankings() + + assert list(rankings.keys()) == ['A1', 'B2', 'C3'] + assert list(rankings.values()) == [1, 2, 3] + + def test_empty_rankings_returns_empty_and_caches_nothing( + self, resolver, mock_get): + mock_get.return_value = _make_response({'rankings': []}) + + assert resolver._fetch_ncaa_fb_rankings() == {} + # Nothing was cached, so the next call hits HTTP again. + assert resolver._fetch_ncaa_fb_rankings() == {} + assert mock_get.call_count == 2 + + +# --------------------------------------------------------------------------- +# Shared class cache (fixed behavior) +# --------------------------------------------------------------------------- + +class TestSharedCache: + def test_cache_shared_across_instances(self, mock_get): + resolver1 = DynamicTeamResolver() + resolver1.resolve_teams(['AP_TOP_5']) + assert mock_get.call_count == 1 + + resolver2 = DynamicTeamResolver() + result = resolver2.resolve_teams(['AP_TOP_5']) + + # Post-fix: the class-level cache serves the second instance with + # ZERO additional HTTP calls. + assert result == TOP_TEAMS[:5] + assert mock_get.call_count == 1 + + def test_ttl_expiry_refetches(self, resolver, mock_get, monkeypatch): + resolver.resolve_teams(['AP_TOP_5']) + assert mock_get.call_count == 1 + + stamp = DynamicTeamResolver._cache_timestamp + monkeypatch.setattr( + dtr_module, 'time', types.SimpleNamespace(time=lambda: stamp + 3601)) + + resolver.resolve_teams(['AP_TOP_5']) + assert mock_get.call_count == 2 + + def test_clear_cache_through_one_instance_affects_all(self, mock_get): + resolver1 = DynamicTeamResolver() + resolver1.resolve_teams(['AP_TOP_5']) + assert mock_get.call_count == 1 + + resolver2 = DynamicTeamResolver() + resolver2.clear_cache() + + # Post-fix: clear_cache writes through the class, so resolver1 + # must refetch even though resolver2 did the clearing. + resolver1.resolve_teams(['AP_TOP_5']) + assert mock_get.call_count == 2 + + def test_module_function_benefits_from_class_cache(self, mock_get): + # resolve_dynamic_teams constructs a fresh resolver per call, but + # the class-shared cache means only the first call hits HTTP. + first = resolve_dynamic_teams(['AP_TOP_5']) + second = resolve_dynamic_teams(['AP_TOP_5']) + + assert first == second == TOP_TEAMS[:5] + assert mock_get.call_count == 1 + + +# --------------------------------------------------------------------------- +# Failure handling +# --------------------------------------------------------------------------- + +class TestFailureHandling: + def test_network_failure_drops_dynamic_keeps_static_caches_nothing( + self, resolver, mock_get): + mock_get.side_effect = [ + requests.exceptions.RequestException('boom'), + _make_response(_rankings_payload()), + ] + + result = resolver.resolve_teams(['UGA', 'AP_TOP_5']) + + # Dynamic name silently dropped, static name kept, nothing raises. + assert result == ['UGA'] + + # Nothing was cached on failure: a subsequent call refetches and + # succeeds. + result = resolver.resolve_teams(['UGA', 'AP_TOP_5']) + assert result == ['UGA', 'MICH', 'OSU', 'TEX', 'ALA'] + assert mock_get.call_count == 2 + + +# --------------------------------------------------------------------------- +# sport argument +# --------------------------------------------------------------------------- + +class TestSportArgument: + def test_sport_arg_ignored_for_expansion(self, resolver, mock_get): + # Pin: the sport argument is effectively ignored — each pattern + # carries its own sport ('ncaa_fb'), so passing sport='nfl' still + # expands from the college-football rankings. + result = resolver.resolve_teams(['AP_TOP_5'], sport='nfl') + + assert result == TOP_TEAMS[:5] + assert mock_get.call_count == 1 diff --git a/test/test_logging_config.py b/test/test_logging_config.py new file mode 100644 index 00000000..e0fe7fff --- /dev/null +++ b/test/test_logging_config.py @@ -0,0 +1,274 @@ +""" +Tests for src/logging_config.py — the formatters, adapter, and setup used +by every logger in the system (BasePlugin uses get_logger, not stdlib +logging.getLogger). + +Includes regression guards for two fixed bugs: ContextualFormatter used to +mutate record.msg in place (double-prefixing with two handlers), and +log_error hardcoded exc_info=True so passing it explicitly raised +TypeError. +""" + +import json +import logging +import sys + +import pytest + +from src.logging_config import ( + ContextualFormatter, + PluginLoggerAdapter, + StructuredFormatter, + get_logger, + log_debug, + log_error, + log_info, + log_warning, + log_with_context, + setup_logging, +) + + +def make_record(msg="hello", level=logging.INFO, **extra): + record = logging.LogRecord( + name="test.logger", level=level, pathname=__file__, lineno=42, + msg=msg, args=(), exc_info=None) + for key, value in extra.items(): + setattr(record, key, value) + return record + + +class TestStructuredFormatter: + def test_emits_valid_json_with_base_keys(self): + out = json.loads(StructuredFormatter().format(make_record())) + assert set(out) == { + "timestamp", "level", "logger", "message", + "module", "function", "line", + } + assert out["level"] == "INFO" + assert out["message"] == "hello" + assert out["logger"] == "test.logger" + + def test_optional_keys_only_when_present(self): + record = make_record(context={"k": "v"}, plugin_id="clock", + operation_id="op-1") + out = json.loads(StructuredFormatter().format(record)) + assert out["context"] == {"k": "v"} + assert out["plugin_id"] == "clock" + assert out["operation_id"] == "op-1" + + def test_exception_key_when_exc_info_present(self): + try: + raise ValueError("kaboom") + except ValueError: + record = logging.LogRecord( + name="t", level=logging.ERROR, pathname=__file__, lineno=1, + msg="failed", args=(), exc_info=sys.exc_info()) + out = json.loads(StructuredFormatter().format(record)) + assert "kaboom" in out["exception"] + + def test_percent_args_formatted_into_message(self): + record = logging.LogRecord( + name="t", level=logging.INFO, pathname=__file__, lineno=1, + msg="count=%d", args=(7,), exc_info=None) + out = json.loads(StructuredFormatter().format(record)) + assert out["message"] == "count=7" + + +class TestContextualFormatter: + def test_context_prefix_prepended(self): + record = make_record(plugin_id="clock", operation_id="op-1", + context={"k": "v"}) + out = ContextualFormatter().format(record) + assert "[Plugin: clock] [Op: op-1] [k: v] hello" in out + + def test_include_context_false_leaves_message_bare(self): + record = make_record(plugin_id="clock") + out = ContextualFormatter(include_context=False).format(record) + assert "[Plugin:" not in out + assert "hello" in out + + def test_location_toggle(self): + record = make_record() + with_loc = ContextualFormatter(include_location=True).format(record) + without = ContextualFormatter(include_location=False).format(record) + assert f":{record.lineno}" in with_loc + assert f":{record.lineno}" not in without + + def test_record_not_mutated_no_double_prefix(self): + # Regression: a record is formatted once PER HANDLER. The formatter + # must not mutate record.msg, or the second handler's format call + # prepends the prefix again. + record = make_record(plugin_id="clock") + formatter = ContextualFormatter() + first = formatter.format(record) + second = formatter.format(record) + assert record.msg == "hello" # untouched + assert first.count("[Plugin: clock]") == 1 + assert second.count("[Plugin: clock]") == 1 + + def test_percent_args_still_format_after_copy(self): + record = logging.LogRecord( + name="t", level=logging.INFO, pathname=__file__, lineno=1, + msg="count=%d", args=(7,), exc_info=None) + record.plugin_id = "clock" + out = ContextualFormatter().format(record) + assert "[Plugin: clock] count=7" in out + + def test_exception_renders_through_two_handlers(self): + try: + raise ValueError("kaboom") + except ValueError: + record = logging.LogRecord( + name="t", level=logging.ERROR, pathname=__file__, lineno=1, + msg="failed", args=(), exc_info=sys.exc_info()) + record.plugin_id = "clock" + formatter = ContextualFormatter() + assert "kaboom" in formatter.format(record) + assert "kaboom" in formatter.format(record) # second handler's pass + + +class TestPluginLoggerAdapter: + def _capture(self, adapter): + records = [] + handler = logging.Handler() + handler.emit = records.append + adapter.logger.addHandler(handler) + adapter.logger.setLevel(logging.DEBUG) + return records + + def test_stamps_plugin_id_on_every_record(self): + adapter = get_logger("test.adapter1", plugin_id="clock") + records = self._capture(adapter) + adapter.info("x") + assert records[0].plugin_id == "clock" + + def test_explicit_extra_plugin_id_wins(self): + adapter = get_logger("test.adapter2", plugin_id="clock") + records = self._capture(adapter) + adapter.info("x", extra={"plugin_id": "other"}) + assert records[0].plugin_id == "other" + + def test_unrelated_extra_keys_preserved(self): + adapter = get_logger("test.adapter3", plugin_id="clock") + records = self._capture(adapter) + adapter.info("x", extra={"custom": 1}) + assert records[0].plugin_id == "clock" + assert records[0].custom == 1 + + +class TestGetLogger: + def test_plain_logger_without_plugin_id(self): + logger = get_logger("test.plain") + assert isinstance(logger, logging.Logger) + assert logger.name == "test.plain" + + def test_adapter_with_plugin_id(self): + adapter = get_logger("test.wrapped", plugin_id="clock") + assert isinstance(adapter, PluginLoggerAdapter) + assert adapter.logger.name == "test.wrapped" + + +class TestSetupLogging: + # conftest's autouse reset_logging restores root handlers after each test. + + def test_installs_single_stdout_handler(self): + setup_logging() + root = logging.getLogger() + assert len(root.handlers) == 1 + assert isinstance(root.handlers[0], logging.StreamHandler) + + def test_repeat_calls_do_not_accumulate_handlers(self): + setup_logging() + setup_logging() + assert len(logging.getLogger().handlers) == 1 + + def test_json_format_selects_structured_formatter(self): + setup_logging(format_type="json") + assert isinstance( + logging.getLogger().handlers[0].formatter, StructuredFormatter) + + def test_readable_format_selects_contextual_formatter(self): + setup_logging(format_type="readable") + assert isinstance( + logging.getLogger().handlers[0].formatter, ContextualFormatter) + + def test_log_file_adds_file_handler(self, tmp_path): + log_file = tmp_path / "test.log" + setup_logging(log_file=str(log_file)) + root = logging.getLogger() + file_handlers = [h for h in root.handlers + if isinstance(h, logging.FileHandler)] + assert len(file_handlers) == 1 + for h in file_handlers: + h.close() + + def test_unwritable_log_file_warns_and_keeps_console(self, tmp_path, capsys): + bad_path = tmp_path / "no-such-dir" / "test.log" + setup_logging(log_file=str(bad_path)) # must not raise + assert len(logging.getLogger().handlers) == 1 # console only + assert "Could not set up file logging" in capsys.readouterr().err + + def test_debug_env_true_enables_debug(self, monkeypatch): + monkeypatch.setenv("LEDMATRIX_DEBUG", "TRUE") + setup_logging() + assert logging.getLogger().level == logging.DEBUG + + def test_debug_env_other_values_stay_info(self, monkeypatch): + # Pinned: only the literal (case-insensitive) "true" enables debug; + # "1" does not. + monkeypatch.setenv("LEDMATRIX_DEBUG", "1") + setup_logging() + assert logging.getLogger().level == logging.INFO + + def test_explicit_level_wins_over_env(self, monkeypatch): + monkeypatch.setenv("LEDMATRIX_DEBUG", "true") + setup_logging(level=logging.WARNING) + assert logging.getLogger().level == logging.WARNING + + +class TestLogWithContext: + def _capture(self, name): + logger = logging.getLogger(name) + records = [] + handler = logging.Handler() + handler.emit = records.append + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + return logger, records + + def test_context_attrs_stamped(self): + logger, records = self._capture("test.lwc1") + log_with_context(logger, logging.INFO, "msg", + context={"k": "v"}, plugin_id="clock", + operation_id="op-1") + record = records[0] + assert record.context == {"k": "v"} + assert record.plugin_id == "clock" + assert record.operation_id == "op-1" + + def test_wrappers_use_their_levels(self): + logger, records = self._capture("test.lwc2") + log_debug(logger, "d") + log_info(logger, "i") + log_warning(logger, "w") + assert [r.levelno for r in records] == [ + logging.DEBUG, logging.INFO, logging.WARNING] + + def test_log_error_defaults_exc_info_true(self): + logger, records = self._capture("test.lwc3") + try: + raise ValueError("kaboom") + except ValueError: + log_error(logger, "failed") + assert records[0].levelno == logging.ERROR + assert records[0].exc_info is not None + + def test_log_error_accepts_explicit_exc_info(self): + # Regression: the old hardcoded exc_info=True raised + # "got multiple values for keyword argument 'exc_info'". + logger, records = self._capture("test.lwc4") + log_error(logger, "failed", exc_info=False) + # Falsy exc_info is stored verbatim on the record; the contract is + # simply "no traceback attached". + assert not records[0].exc_info diff --git a/test/test_saved_repositories.py b/test/test_saved_repositories.py new file mode 100644 index 00000000..39bed1e4 --- /dev/null +++ b/test/test_saved_repositories.py @@ -0,0 +1,240 @@ +""" +Tests for src/plugin_system/saved_repositories.py — pins the +SavedRepositoriesManager contract. + +Covers: the three accepted on-disk load shapes (bare list, wrapped +{"repositories": [...]}, anything else -> []) and that saves always write +the bare-list form; add/remove/has round trips through a fresh manager; +URL normalization post-fix (_clean_url strips only a TRAILING '.git' after +trailing slashes — the old unanchored .replace('.git', '') mangled URLs +like my.github.io); name derivation and registry-vs-single type +classification (the ledmatrix-plugins check is lowercased, the +plugins.json check is case-sensitive); and the post-fix rollback of the +in-memory list when _save_repositories() fails, so memory never diverges +from disk. +""" + +import json + +from src.plugin_system.saved_repositories import SavedRepositoriesManager + + +def make_manager(path): + return SavedRepositoriesManager(config_path=str(path)) + + +class TestLoading: + def test_missing_file_empty_and_not_created(self, tmp_path): + path = tmp_path / "repos.json" + manager = make_manager(path) + assert manager.get_all() == [] + assert not path.exists() + + def test_bare_list_shape(self, tmp_path): + path = tmp_path / "repos.json" + entries = [{'url': 'https://github.com/u/r', 'name': 'r', 'type': 'single'}] + path.write_text(json.dumps(entries)) + assert make_manager(path).get_all() == entries + + def test_wrapped_dict_shape(self, tmp_path): + path = tmp_path / "repos.json" + entries = [{'url': 'https://github.com/u/r', 'name': 'r', 'type': 'single'}] + path.write_text(json.dumps({'repositories': entries})) + assert make_manager(path).get_all() == entries + + def test_other_shape_yields_empty(self, tmp_path): + path = tmp_path / "repos.json" + path.write_text(json.dumps({'x': 1})) + assert make_manager(path).get_all() == [] + + def test_malformed_json_yields_empty_no_raise(self, tmp_path): + path = tmp_path / "repos.json" + path.write_text("not json {{") + assert make_manager(path).get_all() == [] + + +class TestSaveFormat: + def test_save_always_writes_bare_list(self, tmp_path): + # Even when loaded from the wrapped {"repositories": [...]} form, + # the next save normalizes the file to a bare JSON list. + path = tmp_path / "repos.json" + entries = [{'url': 'https://github.com/u/r', 'name': 'r', 'type': 'single'}] + path.write_text(json.dumps({'repositories': entries})) + manager = make_manager(path) + assert manager.add("https://github.com/u/r2") is True + on_disk = json.loads(path.read_text()) + assert isinstance(on_disk, list) + assert len(on_disk) == 2 + + +class TestAdd: + def test_round_trip_creates_parents_and_reloads(self, tmp_path): + path = tmp_path / "sub" / "repos.json" + manager = make_manager(path) + assert manager.add("https://github.com/user/repo") is True + assert path.exists() + entry = manager.get_all()[0] + assert entry['url'] == "https://github.com/user/repo" + assert entry['name'] == "repo" + assert entry['type'] == "single" + # A fresh manager on the same path sees the persisted entry. + fresh = make_manager(path) + assert fresh.get_all() == [entry] + + def test_duplicate_returns_false_file_unchanged(self, tmp_path): + path = tmp_path / "repos.json" + manager = make_manager(path) + assert manager.add("https://github.com/user/repo") is True + before = path.read_text() + assert manager.add("https://github.com/user/repo") is False + assert path.read_text() == before + assert len(manager.get_all()) == 1 + + def test_trailing_git_and_slash_stripped(self, tmp_path): + manager = make_manager(tmp_path / "repos.json") + assert manager.add("https://github.com/user/repo.git/") is True + assert manager.get_all()[0]['url'] == "https://github.com/user/repo" + + def test_interior_dot_git_not_mangled(self, tmp_path): + # Regression for the old unanchored .replace('.git', ''): a URL + # merely CONTAINING '.git' must be stored verbatim. + manager = make_manager(tmp_path / "repos.json") + url = "https://github.com/user/my.github.io" + assert manager.add(url) is True + assert manager.get_all()[0]['url'] == url + + +class TestNameExtraction: + def test_name_derived_from_last_path_segment(self, tmp_path): + manager = make_manager(tmp_path / "repos.json") + manager.add("https://github.com/user/football-scoreboard") + assert manager.get_all()[0]['name'] == "football-scoreboard" + + def test_explicit_name_preserved(self, tmp_path): + manager = make_manager(tmp_path / "repos.json") + manager.add("https://github.com/user/repo", name="My Repo") + assert manager.get_all()[0]['name'] == "My Repo" + + def test_url_without_slash_uses_whole_url(self, tmp_path): + manager = make_manager(tmp_path / "repos.json") + manager.add("standalone") + assert manager.get_all()[0]['name'] == "standalone" + + +class TestTypeClassification: + def _type_of(self, tmp_path, url): + manager = make_manager(tmp_path / "repos.json") + assert manager.add(url) is True + return manager.get_all()[0]['type'] + + def test_plugins_json_url_is_registry(self, tmp_path): + url = "https://raw.githubusercontent.com/x/main/plugins.json" + assert self._type_of(tmp_path, url) == "registry" + + def test_ledmatrix_plugins_check_is_case_insensitive(self, tmp_path): + url = "https://github.com/ChuckBuilds/LEDMATRIX-PLUGINS" + assert self._type_of(tmp_path, url) == "registry" + + def test_plugins_json_check_is_case_sensitive(self, tmp_path): + # Only the 'ledmatrix-plugins' check is lowercased; the + # 'plugins.json' substring check is case-sensitive. Pinned. + url = "https://example.com/PLUGINS.JSON" + assert self._type_of(tmp_path, url) == "single" + + def test_plain_repo_is_single(self, tmp_path): + assert self._type_of(tmp_path, "https://github.com/user/repo") == "single" + + def test_get_registry_repositories_filters(self, tmp_path): + manager = make_manager(tmp_path / "repos.json") + manager.add("https://github.com/user/repo") + manager.add("https://raw.githubusercontent.com/x/main/plugins.json") + registries = manager.get_registry_repositories() + assert len(registries) == 1 + assert registries[0]['type'] == "registry" + + +class TestRemove: + def test_remove_present_persists(self, tmp_path): + path = tmp_path / "repos.json" + manager = make_manager(path) + manager.add("https://github.com/user/repo") + assert manager.remove("https://github.com/user/repo") is True + assert manager.get_all() == [] + assert make_manager(path).get_all() == [] + + def test_remove_absent_false_no_write(self, tmp_path): + path = tmp_path / "repos.json" + manager = make_manager(path) + manager.add("https://github.com/user/repo") + before = path.read_text() + assert manager.remove("https://github.com/user/other") is False + assert path.read_text() == before + + def test_remove_with_dirty_url_matches_clean_stored(self, tmp_path): + manager = make_manager(tmp_path / "repos.json") + manager.add("https://github.com/user/repo") + assert manager.remove("https://github.com/user/repo.git/") is True + assert manager.get_all() == [] + + +class TestHas: + def test_has_applies_url_cleaning(self, tmp_path): + manager = make_manager(tmp_path / "repos.json") + manager.add("https://x/y") + assert manager.has("https://x/y.git/") is True + assert manager.has("https://x/z") is False + + +class TestSaveFailureRollback: + def test_add_rolls_back_on_save_failure(self, tmp_path, monkeypatch): + # Post-fix: a failed save must not leave a phantom in-memory entry. + path = tmp_path / "repos.json" + manager = make_manager(path) + monkeypatch.setattr(manager, "_save_repositories", lambda: False) + assert manager.add("https://github.com/user/repo") is False + assert manager.get_all() == [] + assert not path.exists() + + def test_remove_rolls_back_on_save_failure(self, tmp_path, monkeypatch): + path = tmp_path / "repos.json" + manager = make_manager(path) + manager.add("https://github.com/user/repo") # real save + monkeypatch.setattr(manager, "_save_repositories", lambda: False) + assert manager.remove("https://github.com/user/repo") is False + assert manager.get_all() == [ + {'url': 'https://github.com/user/repo', 'name': 'repo', 'type': 'single'} + ] + # Disk still has the entry too — memory and disk stay in sync. + assert len(json.loads(path.read_text())) == 1 + + def test_failed_write_leaves_existing_file_intact(self, tmp_path, monkeypatch): + # The save is atomic (temp file + os.replace): a write that dies + # mid-serialization must neither truncate the existing file nor + # leave a stray .tmp behind. + path = tmp_path / "repos.json" + manager = make_manager(path) + manager.add("https://github.com/user/repo") # real save + before = path.read_text() + + def boom(*args, **kwargs): + raise OSError("disk full") + monkeypatch.setattr(json, "dump", boom) + assert manager.add("https://github.com/user/other") is False + + assert path.read_text() == before + assert list(tmp_path.glob("*.tmp")) == [] + + +class TestGetAllCopy: + def test_get_all_is_shallow_copy(self, tmp_path): + # Characterization: get_all() copies the LIST but not the entry + # dicts, so mutating a returned entry mutates internal state. + # Appending to the returned list, however, does not. Do not "fix" + # without auditing callers that rely on list-copy semantics. + manager = make_manager(tmp_path / "repos.json") + manager.add("https://github.com/user/repo") + returned = manager.get_all() + returned.append({'url': 'x'}) + assert len(manager.get_all()) == 1 # list itself is copied + manager.get_all()[0]['name'] = 'hacked' + assert manager.get_all()[0]['name'] == 'hacked' # dicts are shared diff --git a/test/test_skin_runtime_cache.py b/test/test_skin_runtime_cache.py new file mode 100644 index 00000000..460e1902 --- /dev/null +++ b/test/test_skin_runtime_cache.py @@ -0,0 +1,377 @@ +"""Gap tests for src/skin_system/skin_runtime.py: the discovery cache, +module namespacing internals, API gating edge cases, and targeting. + +test/test_skin_system.py already covers discovery validation, load_skin +basics, and build_context — nothing here duplicates those. + +NOTE: every test uses a UNIQUE skin id. load_skin caches the entry +module in sys.modules per skin id and never re-executes it, so reusing +an id across tests would silently serve another test's module. +""" + +import builtins +import json +import os +import sys +import time +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +# skin_runtime -> skin_base can transitively reach hardware modules via +# sports imports in sibling tests' processes; stub the matrix driver +# before importing, matching test_skin_system.py. +sys.modules.setdefault("rgbmatrix", MagicMock()) + +from src.skin_system import skin_runtime +from src.skin_system.skin_base import SKIN_API_VERSION, ScoreboardSkin + + +DEFAULT_BODY = ( + "from src.skin_system.skin_base import ScoreboardSkin\n" + "class {cls}(ScoreboardSkin):\n" + " def render_live(self, ctx, game):\n" + " return True\n" +) + + +@pytest.fixture(autouse=True) +def _clean_runtime_state(): + """Clear the discovery cache and any skin modules this test creates.""" + skin_runtime._discovery_cache.clear() + before = {k for k in sys.modules if k.startswith("_skin_")} + yield + skin_runtime._discovery_cache.clear() + created = [k for k in sys.modules + if k.startswith("_skin_") and k not in before] + for k in created: + sys.modules.pop(k, None) + + +def make_skin(skins_dir: Path, skin_id: str, *, + api_version: str = SKIN_API_VERSION, + class_name: str = "TestSkin", + body: str = None, + extra_files: dict = None, + entry_point: str = None, + manifest_id: str = None, + manifest_extra: dict = None, + write_entry: bool = True) -> Path: + """Write a skin package directory and return its path.""" + skin_dir = skins_dir / skin_id + skin_dir.mkdir(parents=True, exist_ok=True) + manifest = { + "id": manifest_id or skin_id, + "name": skin_id, + "version": "1.0.0", + "skin_api_version": api_version, + "class_name": class_name, + } + if entry_point: + manifest["entry_point"] = entry_point + manifest.update(manifest_extra or {}) + (skin_dir / "skin.json").write_text(json.dumps(manifest)) + if write_entry: + entry_name = entry_point or "skin.py" + (skin_dir / entry_name).write_text( + body if body is not None else DEFAULT_BODY.format(cls=class_name)) + for name, content in (extra_files or {}).items(): + (skin_dir / name).write_text(content) + return skin_dir + + +def counting_read_manifest(monkeypatch): + """Wrap skin_runtime._read_manifest with a call counter.""" + original = skin_runtime._read_manifest + counter = {"count": 0} + + def wrapper(skin_dir): + counter["count"] += 1 + return original(skin_dir) + + monkeypatch.setattr(skin_runtime, "_read_manifest", wrapper) + return counter + + +def bump_mtime(path: Path, offset: float = 100.0): + """Set a distinct, strictly later mtime so the fingerprint changes.""" + t = time.time() + offset + os.utime(path, (t, t)) + + +# --------------------------------------------------------------------------- +# A. Discovery cache +# --------------------------------------------------------------------------- + +class TestDiscoveryCache: + def test_second_call_serves_cache(self, tmp_path, monkeypatch): + make_skin(tmp_path, "t01-cache-hit") + counter = counting_read_manifest(monkeypatch) + first = skin_runtime.discover_skins(tmp_path) + count_after_first = counter["count"] + assert count_after_first >= 1 + second = skin_runtime.discover_skins(tmp_path) + assert counter["count"] == count_after_first # no re-read + assert second == first + assert "t01-cache-hit" in second + + def test_manifest_edit_invalidates_without_force_refresh(self, tmp_path): + skin_dir = make_skin(tmp_path, "t02-edit") + skins = skin_runtime.discover_skins(tmp_path) + assert skins["t02-edit"]["name"] == "t02-edit" + + manifest_path = skin_dir / "skin.json" + manifest = json.loads(manifest_path.read_text()) + manifest["name"] = "renamed" + manifest_path.write_text(json.dumps(manifest)) + bump_mtime(manifest_path) + + skins = skin_runtime.discover_skins(tmp_path) # no force_refresh + assert skins["t02-edit"]["name"] == "renamed" + + def test_new_skin_dir_invalidates(self, tmp_path): + make_skin(tmp_path, "t03-first") + assert set(skin_runtime.discover_skins(tmp_path)) == {"t03-first"} + + new_dir = make_skin(tmp_path, "t03-second") + bump_mtime(new_dir / "skin.json") + bump_mtime(tmp_path) + + skins = skin_runtime.discover_skins(tmp_path) # no force_refresh + assert set(skins) == {"t03-first", "t03-second"} + + def test_py_file_change_does_not_invalidate(self, tmp_path, monkeypatch): + # PIN: the fingerprint only globs */skin.json — editing a skin's + # .py file alone does NOT invalidate the cache; the cached + # manifests are still served (a code change needs a restart). + skin_dir = make_skin(tmp_path, "t04-pyedit") + counter = counting_read_manifest(monkeypatch) + skin_runtime.discover_skins(tmp_path) + count_after_first = counter["count"] + + (skin_dir / "skin.py").write_text("# rewritten\n" + + DEFAULT_BODY.format(cls="TestSkin")) + bump_mtime(skin_dir / "skin.py") + + skins = skin_runtime.discover_skins(tmp_path) + assert counter["count"] == count_after_first # cache still served + assert "t04-pyedit" in skins + + def test_force_refresh_rereads_with_unchanged_fingerprint(self, tmp_path, + monkeypatch): + make_skin(tmp_path, "t05-force") + counter = counting_read_manifest(monkeypatch) + skin_runtime.discover_skins(tmp_path) + count_after_first = counter["count"] + skin_runtime.discover_skins(tmp_path, force_refresh=True) + assert counter["count"] > count_after_first + + def test_result_mapping_is_copy_but_manifests_shared(self, tmp_path): + make_skin(tmp_path, "t06-copy") + result = skin_runtime.discover_skins(tmp_path) + + # Mutating the returned mapping does not poison the cache... + del result["t06-copy"] + again = skin_runtime.discover_skins(tmp_path) # cache hit + assert "t06-copy" in again + + # ...but the inner manifest dicts ARE shared with the cache (pin). + again["t06-copy"]["name"] = "mutated-inner" + third = skin_runtime.discover_skins(tmp_path) # cache hit + assert third["t06-copy"]["name"] == "mutated-inner" + + def test_missing_directory_returns_empty_and_caches_nothing(self, tmp_path): + missing = tmp_path / "not-yet" + assert skin_runtime.discover_skins(missing) == {} + assert str(missing) not in skin_runtime._discovery_cache + + # Creating the directory later is picked up without force_refresh. + make_skin(missing, "t07-late") + skins = skin_runtime.discover_skins(missing) + assert "t07-late" in skins + + def test_hidden_underscore_and_plain_file_entries_skipped(self, tmp_path): + make_skin(tmp_path, ".hidden-skin") + make_skin(tmp_path, "_private-skin") + (tmp_path / "stray-file").write_text("not a directory") + make_skin(tmp_path, "t08-good") + skins = skin_runtime.discover_skins(tmp_path, force_refresh=True) + assert set(skins) == {"t08-good"} + + def test_manifest_id_mismatch_keys_by_manifest_id(self, tmp_path): + make_skin(tmp_path, "t09-dirname", manifest_id="t09-manifest-id") + skins = skin_runtime.discover_skins(tmp_path, force_refresh=True) + assert "t09-manifest-id" in skins + assert "t09-dirname" not in skins + assert skins["t09-manifest-id"]["_skin_dir"].endswith("t09-dirname") + + def test_falsy_required_field_drops_skin(self, tmp_path): + make_skin(tmp_path, "t10-empty-class", class_name="") + skins = skin_runtime.discover_skins(tmp_path, force_refresh=True) + assert skins == {} + + +# --------------------------------------------------------------------------- +# B. Module namespacing (_load_skin_module via load_skin) +# --------------------------------------------------------------------------- + +BODY_WITH_HELPERS = ( + "import helpers\n" + "from src.skin_system.skin_base import ScoreboardSkin\n" + "class TestSkin(ScoreboardSkin):\n" + " pass\n" +) + + +class TestModuleNamespacing: + def test_namespaced_sys_modules_keys(self, tmp_path): + make_skin(tmp_path, "t11-ns", body=BODY_WITH_HELPERS, + extra_files={"helpers.py": "VALUE = 11\n"}) + skin = skin_runtime.load_skin("t11-ns", skins_dir=tmp_path) + assert skin is not None + assert "_skin_t11-ns_skin" in sys.modules + assert "_skin_t11-ns_helpers" in sys.modules + + def test_preseeded_bare_name_restored(self, tmp_path, monkeypatch): + sentinel = object() + monkeypatch.setitem(sys.modules, "helpers", sentinel) + make_skin(tmp_path, "t12a-restore", body=BODY_WITH_HELPERS, + extra_files={"helpers.py": "VALUE = 'a'\n"}) + skin = skin_runtime.load_skin("t12a-restore", skins_dir=tmp_path) + assert skin is not None + assert sys.modules["helpers"] is sentinel + + def test_absent_bare_name_stays_absent(self, tmp_path): + saved = sys.modules.pop("helpers", None) + try: + assert "helpers" not in sys.modules + make_skin(tmp_path, "t12b-absent", body=BODY_WITH_HELPERS, + extra_files={"helpers.py": "VALUE = 'b'\n"}) + skin = skin_runtime.load_skin("t12b-absent", skins_dir=tmp_path) + assert skin is not None + assert "helpers" not in sys.modules + finally: + if saved is not None: + sys.modules["helpers"] = saved + + def test_stdlib_shadowing_sibling_leaves_real_module_intact(self, tmp_path): + real_json = sys.modules["json"] + make_skin(tmp_path, "t12c-json", + extra_files={"json.py": "SKIN_LOCAL = True\n"}) + skin = skin_runtime.load_skin("t12c-json", skins_dir=tmp_path) + assert skin is not None + assert sys.modules["json"] is real_json + assert not hasattr(sys.modules["json"], "SKIN_LOCAL") + assert json.loads('{"ok": 1}') == {"ok": 1} # stdlib still works + # The skin's copy lives only under its namespaced alias. + assert getattr(sys.modules["_skin_t12c-json_json"], "SKIN_LOCAL") is True + + def test_entry_module_executed_once_across_loads(self, tmp_path, + monkeypatch): + executions = [] + monkeypatch.setattr(builtins, "_t13_skin_executions", executions, + raising=False) + body = ( + "import builtins\n" + "builtins._t13_skin_executions.append(1)\n" + "from src.skin_system.skin_base import ScoreboardSkin\n" + "class TestSkin(ScoreboardSkin):\n" + " pass\n" + ) + make_skin(tmp_path, "t13-cached", body=body) + for _ in range(3): + skin = skin_runtime.load_skin("t13-cached", skins_dir=tmp_path) + assert skin is not None + assert len(executions) == 1 # module executed exactly once + + def test_sibling_import_failure_returns_none_and_restores_bare( + self, tmp_path, monkeypatch): + sentinel = object() + monkeypatch.setitem(sys.modules, "helpers", sentinel) + make_skin(tmp_path, "t14-sibfail", body=BODY_WITH_HELPERS, + extra_files={"helpers.py": "raise RuntimeError('sibling boom')\n"}) + assert skin_runtime.load_skin("t14-sibfail", skins_dir=tmp_path) is None + assert sys.modules["helpers"] is sentinel + + def test_missing_entry_point_file(self, tmp_path): + make_skin(tmp_path, "t15-noentry", write_entry=False) + assert skin_runtime.load_skin("t15-noentry", skins_dir=tmp_path) is None + + def test_custom_entry_point(self, tmp_path): + make_skin(tmp_path, "t16-custom", entry_point="render.py") + skin = skin_runtime.load_skin("t16-custom", skins_dir=tmp_path) + assert isinstance(skin, ScoreboardSkin) + assert "_skin_t16-custom_render" in sys.modules + assert "_skin_t16-custom_skin" not in sys.modules + + def test_class_name_pointing_at_unrelated_class(self, tmp_path): + body = "class NotASkin:\n pass\n" + make_skin(tmp_path, "t17a-wrongclass", body=body, + class_name="NotASkin") + assert skin_runtime.load_skin("t17a-wrongclass", + skins_dir=tmp_path) is None + + def test_class_name_pointing_at_instance(self, tmp_path): + body = ( + "from src.skin_system.skin_base import ScoreboardSkin\n" + "class MySkin(ScoreboardSkin):\n" + " pass\n" + "obj = MySkin({}, {})\n" + ) + make_skin(tmp_path, "t17b-instance", body=body, class_name="obj") + assert skin_runtime.load_skin("t17b-instance", + skins_dir=tmp_path) is None + + def test_constructor_raising_returns_none(self, tmp_path): + body = ( + "from src.skin_system.skin_base import ScoreboardSkin\n" + "class TestSkin(ScoreboardSkin):\n" + " def __init__(self, manifest, options):\n" + " raise ValueError('ctor boom')\n" + ) + make_skin(tmp_path, "t18-ctor", body=body) + assert skin_runtime.load_skin("t18-ctor", skins_dir=tmp_path) is None + + +# --------------------------------------------------------------------------- +# C. API gate + targeting +# --------------------------------------------------------------------------- + +class TestApiGateAndTargeting: + def test_same_major_higher_minor_loads(self, tmp_path): + make_skin(tmp_path, "t19-minor", api_version="1.9.0") + skin = skin_runtime.load_skin("t19-minor", skins_dir=tmp_path) + assert isinstance(skin, ScoreboardSkin) + + def test_malformed_api_version_refused(self, tmp_path): + make_skin(tmp_path, "t20-malformed", api_version="abc") + assert skin_runtime.load_skin("t20-malformed", + skins_dir=tmp_path) is None + + @pytest.mark.parametrize("manifest,sport,sport_key,expected", [ + # No targets key at all -> matches everything + ({"id": "x"}, "baseball", "mlb", True), + ({"id": "x"}, None, None, True), + # Empty targets dict -> matches everything + ({"id": "x", "targets": {}}, "hockey", None, True), + # sports family match + ({"id": "x", "targets": {"sports": ["baseball"]}}, + "baseball", None, True), + # sport_keys exact match + ({"id": "x", "targets": {"sport_keys": ["milb"]}}, + None, "milb", True), + # OR semantics: sport_keys matches even though sports excludes it + ({"id": "x", "targets": {"sports": ["hockey"], + "sport_keys": ["milb"]}}, + "baseball", "milb", True), + # Neither matches + ({"id": "x", "targets": {"sports": ["hockey"]}}, + "baseball", None, False), + ({"id": "x", "targets": {"sports": ["hockey"], + "sport_keys": ["nhl"]}}, + "baseball", "milb", False), + ]) + def test_skin_matches_target(self, manifest, sport, sport_key, expected): + assert skin_runtime.skin_matches_target( + manifest, sport, sport_key) is expected diff --git a/test/test_sports_capabilities.py b/test/test_sports_capabilities.py index f43a2b61..a3cd0f3f 100644 --- a/test/test_sports_capabilities.py +++ b/test/test_sports_capabilities.py @@ -878,3 +878,200 @@ class TestCapabilityExports: def test_rotation_strategy_base_requires_a_schedule(self): with pytest.raises(NotImplementedError): RotationStrategy().schedule([game("a")]) + + +# --------------------------------------------------------------------------- +# Celebrations: rendering + previously untested edges +# --------------------------------------------------------------------------- + +from PIL import Image, ImageDraw, ImageFont # noqa: E402 + + +class _RenderableLive(_FakeLive): + """A _FakeLive that can actually execute _draw_celebration_layout: + real fonts, a display manager holding a real PIL image, and the two + SportsCore drawing seams the mixin calls.""" + + def __init__(self, mode_config=None, favorite_teams=None, + width=128, height=32, with_matrix=True): + super().__init__(mode_config=mode_config, favorite_teams=favorite_teams) + font = ImageFont.load_default() + self.fonts = {"time": font, "status": font, "score": font} + self.display_width = width + self.display_height = height + dm = MagicMock() + if with_matrix: + dm.matrix.width = width + dm.matrix.height = height + else: + dm.matrix = None + dm.image = Image.new("RGB", (width, height)) + self.display_manager = dm + self.logo_calls = [] + + def _load_and_resize_logo(self, team_id, abbr, path, url): + self.logo_calls.append(abbr) + logo = Image.new("RGBA", (10, 10), (0, 200, 0, 255)) + return logo + + def _draw_text_with_outline(self, draw, text, position, font, fill=(255, 255, 255)): + draw.text(position, str(text), font=font, fill=fill) + + +class _RenderableCelebrating(CelebrationMixin, _RenderableLive): + pass + + +def _armed(manager, *, kind="score", side="home", started_ago=0.0): + manager._start_celebration( + game("g1", home_score=7, away_score=3), kind, + scored_side=side, team_abbr="HOM", away_score=3, home_score=7, + points=7, + ) + manager.active_celebration["started_at"] = time.time() - started_ago + return manager.active_celebration + + +class TestDrawCelebrationLayout: + """The takeover render path, executed for real (previously always + mocked out).""" + + def test_renders_and_hands_frame_to_display_manager(self): + manager = _RenderableCelebrating() + celebration = _armed(manager) + manager._draw_celebration_layout(celebration) + # The final frame was assigned and pushed. + assert isinstance(manager.display_manager.image, Image.Image) + assert manager.display_manager.image.mode == "RGB" + assert manager.display_manager.image.size == (128, 32) + manager.display_manager.update_display.assert_called_once() + assert manager.display_manager.image.convert("L").getbbox() is not None + + def test_force_clear_clears_display_first(self): + manager = _RenderableCelebrating() + celebration = _armed(manager) + manager._draw_celebration_layout(celebration, force_clear=True) + manager.display_manager.clear.assert_called_once() + + def test_flash_background_within_first_window(self): + # elapsed < 1.2 with int(elapsed/0.2) even -> flash color backdrop. + manager = _RenderableCelebrating() + celebration = _armed(manager, started_ago=0.05) + manager._draw_celebration_layout(celebration) + flash = manager.display_manager.image + # After the flash window: plain black backdrop. + celebration["started_at"] = time.time() - 5 + manager._draw_celebration_layout(celebration) + steady = manager.display_manager.image + # Corner pixels (away from logos/text) show the two backgrounds. + assert flash.getpixel((64, 30)) != steady.getpixel((64, 30)) or \ + flash.getpixel((3, 0)) != steady.getpixel((3, 0)) + + def test_matrix_dims_fallback_to_display_attrs(self): + manager = _RenderableCelebrating(width=96, height=48, with_matrix=False) + celebration = _armed(manager) + manager._draw_celebration_layout(celebration) + assert manager.display_manager.image.size == (96, 48) + + def test_highlight_color_alternates_with_elapsed(self): + manager = _RenderableCelebrating() + celebration = _armed(manager) + # int(elapsed*4) % 2 == 0 -> yellow; == 1 -> orange. Force each phase + # and diff the frames. + celebration["started_at"] = time.time() - 2.0 # 8 -> even + manager._draw_celebration_layout(celebration) + even = manager.display_manager.image.tobytes() + celebration["started_at"] = time.time() - 2.25 # 9 -> odd + manager._draw_celebration_layout(celebration) + odd = manager.display_manager.image.tobytes() + assert even != odd + + def test_logo_failure_still_renders_text(self): + manager = _RenderableCelebrating() + + def boom(*a, **k): + raise RuntimeError("disk gone") + + manager._load_and_resize_logo = boom + celebration = _armed(manager, started_ago=5) # steady background + manager._draw_celebration_layout(celebration) # must not raise + assert manager.display_manager.image.convert("L").getbbox() is not None + manager.display_manager.update_display.assert_called_once() + + +class TestCelebrationEdges: + def test_should_celebrate_for_three_way_branch(self, celebrating): + g = game("g1", home="FAV", away="OPP") + favored = celebrating(favorites=["FAV"]) + assert favored._should_celebrate_for(g, "home") is True # favorite + assert favored._should_celebrate_for(g, "away") is False # opponent + favored.celebrate_opponent_scores = True + assert favored._should_celebrate_for(g, "away") is True # opted in + unconfigured = celebrating(favorites=[]) + assert unconfigured._should_celebrate_for(g, "away") is True # no favs + + def test_active_celebration_boundary_is_strict(self, celebrating): + manager = celebrating(mode_config={"celebration_duration": 3}) + manager.active_celebration = {"started_at": time.time() - 3.0} + # elapsed == duration -> strictly-less-than comparison says done. + assert manager.has_active_celebration() is False + manager.active_celebration = None + assert manager.has_active_celebration() is False + + @pytest.mark.parametrize("value,expected", [ + ({"value": None}, None), # int(float(None)) TypeError -> caught + ({"value": "abc"}, None), + ({"other": 1}, 0), # neither key -> default 0 + ([3], None), # list -> TypeError -> caught + ("-4", None), # regex fallback finds digits -> 4? No: + ]) + def test_score_to_int_edges(self, value, expected): + result = CelebrationMixin._score_to_int(value) + if value == "-4": + # int(float("-4")) parses directly: -4. + assert result == -4 + else: + assert result == expected + + def test_both_teams_scoring_prefers_away(self, celebrating): + manager = celebrating(favorites=[]) + manager._check_for_score(game("g1", home_score=0, away_score=0)) + manager._check_for_score(game("g1", home_score=7, away_score=3)) + assert manager.active_celebration["scored_side"] == "away" + + def test_away_not_celebratable_falls_through_to_home(self, celebrating): + manager = celebrating(favorites=["HOM"]) # away is the opponent + manager._check_for_score(game("g1", home_score=0, away_score=0)) + manager._check_for_score(game("g1", home_score=7, away_score=3)) + assert manager.active_celebration["scored_side"] == "home" + + def test_coalesce_expired_celebration_fires_fresh(self, celebrating): + manager = celebrating(cls=_Coalescing, + mode_config={"celebration_duration": 1}) + manager._check_for_score(game("g1")) + manager._check_for_score(game("g1", home_score=6)) + first = manager.active_celebration + assert first is not None + first["started_at"] = time.time() - 2 # expired + manager._check_for_score(game("g1", home_score=7)) + # A new celebration replaced the expired one (coalescing only + # suppresses while one is actively on screen). + assert manager.active_celebration is not first + assert manager.active_celebration["home_score"] == 7 + + def test_disabled_win_check_preserves_baseline(self, celebrating): + manager = celebrating(favorites=["HOM"]) + manager._check_for_score(game("g1")) + assert "g1" in manager._score_baselines + manager.celebration_enabled = False + manager._check_for_win(game("g1", home_score=7)) + # Early return BEFORE consuming the baseline: re-enabling later can + # still fire for this game. + assert "g1" in manager._score_baselines + + def test_prune_drops_baselines_for_idless_live_games(self, celebrating): + manager = celebrating() + manager._score_baselines = {"g1": {"away": 0, "home": 0}} + manager.prune_score_baselines([{"no_id_here": True}]) + # live ids collapse to {None}; g1 is not live -> dropped. + assert manager._score_baselines == {} diff --git a/test/test_startup_validator.py b/test/test_startup_validator.py new file mode 100644 index 00000000..43cc2f38 --- /dev/null +++ b/test/test_startup_validator.py @@ -0,0 +1,293 @@ +""" +Tests for src/startup_validator.py — pins the StartupValidator contract. + +Covers: required-key/config error reporting (errors never propagate out of +validate_all), the load_config/get_config accessor split, cache-directory +error-vs-warning downgrade behavior, plugin discovery/manifest checks with +reserved config keys skipped, idempotent validate_all (fresh error/warning +lists each run — the pre-fix behavior duplicated messages), and the +exception classification precedence in raise_on_errors (config > cache > +plugin > fallback ConfigError). +""" + +import copy +import os +from unittest.mock import MagicMock + +import pytest + +from src.exceptions import CacheError, ConfigError, PluginError +from src.startup_validator import StartupValidator + +GOOD_CONFIG = { + 'display': {'hardware': {'rows': 32, 'cols': 64}}, + 'timezone': 'UTC', +} + + +def make_config_manager(config): + """Config manager whose load_config() and get_config() return `config`.""" + mgr = MagicMock() + mgr.load_config.return_value = copy.deepcopy(config) + mgr.get_config.return_value = copy.deepcopy(config) + return mgr + + +@pytest.fixture +def good_cache(monkeypatch, tmp_path): + """Patch CacheManager so cache validation sees an existing writable dir. + + _validate_cache_directory does `from src.cache_manager import CacheManager` + at call time, so patching the attribute on the module is picked up. + """ + mock_cls = MagicMock() + mock_cls.return_value.get_cache_dir.return_value = str(tmp_path) + monkeypatch.setattr("src.cache_manager.CacheManager", mock_cls) + return tmp_path + + +class TestValidateConfig: + """Configuration validation via load_config().""" + + def test_happy_path(self, good_cache): + validator = StartupValidator(make_config_manager(GOOD_CONFIG)) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is True + assert errors == [] + assert warnings == [] + + def test_missing_required_keys(self, good_cache): + validator = StartupValidator(make_config_manager({})) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is False + assert "Missing required configuration key: display" in errors + assert "Missing required configuration key: timezone" in errors + + def test_config_error_does_not_propagate(self, good_cache): + mgr = make_config_manager(GOOD_CONFIG) + mgr.load_config.side_effect = ConfigError("bad json") + validator = StartupValidator(mgr) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is False + config_errors = [e for e in errors if e.startswith("Configuration error:")] + assert len(config_errors) == 1 + assert "bad json" in config_errors[0] + + def test_unexpected_error_does_not_propagate(self, good_cache): + mgr = make_config_manager(GOOD_CONFIG) + mgr.load_config.side_effect = RuntimeError("kapow") + validator = StartupValidator(mgr) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is False + unexpected = [e for e in errors + if e.startswith("Unexpected error validating configuration:")] + assert len(unexpected) == 1 + assert "kapow" in unexpected[0] + + def test_accessor_split_get_config_failure_is_warning_only(self, good_cache): + # _validate_config uses load_config(); _validate_display_config uses + # get_config(). A broken get_config must degrade to a warning, not + # crash or produce a config error. + mgr = make_config_manager(GOOD_CONFIG) + mgr.get_config.side_effect = RuntimeError("accessor broken") + validator = StartupValidator(mgr) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is True + assert errors == [] + assert any(w.startswith("Could not validate display configuration:") + for w in warnings) + assert mgr.load_config.called + assert mgr.get_config.called + + +class TestDisplayConfig: + """Display hardware validation via get_config().""" + + def test_missing_hardware_section_is_error(self, good_cache): + config = {'display': {'runtime': {'gpio_slowdown': 2}}, 'timezone': 'UTC'} + validator = StartupValidator(make_config_manager(config)) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is False + assert "Display hardware configuration is missing" in errors + + def test_missing_rows_cols_are_warnings_not_errors(self, good_cache): + config = {'display': {'hardware': {'brightness': 90}}, 'timezone': 'UTC'} + validator = StartupValidator(make_config_manager(config)) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is True + assert errors == [] + assert "Display hardware setting 'rows' not specified, using default" in warnings + assert "Display hardware setting 'cols' not specified, using default" in warnings + + +class TestCacheDirectory: + """Cache directory validation error/warning split.""" + + def _patch_cache_dir(self, monkeypatch, cache_dir): + mock_cls = MagicMock() + mock_cls.return_value.get_cache_dir.return_value = cache_dir + monkeypatch.setattr("src.cache_manager.CacheManager", mock_cls) + + def test_nonexistent_cache_dir_is_error(self, monkeypatch, tmp_path): + missing = str(tmp_path / "does_not_exist") + self._patch_cache_dir(monkeypatch, missing) + validator = StartupValidator(make_config_manager(GOOD_CONFIG)) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is False + assert any("does not exist" in e and missing in e for e in errors) + + def test_writable_cache_dir_no_errors(self, monkeypatch, tmp_path): + self._patch_cache_dir(monkeypatch, str(tmp_path)) + validator = StartupValidator(make_config_manager(GOOD_CONFIG)) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is True + assert not any('cache' in e.lower() for e in errors) + + def test_unwritable_cache_dir_is_error(self, monkeypatch, tmp_path): + # Root (common in CI) can write anywhere, so chmod tricks don't + # work — force os.access to deny writes for the cache dir only. + cache_dir = str(tmp_path) + self._patch_cache_dir(monkeypatch, cache_dir) + real_access = os.access + + def fake_access(path, mode): + if str(path) == cache_dir and mode == os.W_OK: + return False + return real_access(path, mode) + + monkeypatch.setattr(os, "access", fake_access) + validator = StartupValidator(make_config_manager(GOOD_CONFIG)) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is False + assert any("is not writable" in e for e in errors) + + def test_none_cache_dir_is_warning_not_error(self, monkeypatch): + self._patch_cache_dir(monkeypatch, None) + validator = StartupValidator(make_config_manager(GOOD_CONFIG)) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is True + assert errors == [] + assert "Cache directory not available - caching will be disabled" in warnings + + def test_cache_manager_constructor_failure_is_warning(self, monkeypatch): + mock_cls = MagicMock(side_effect=RuntimeError("no disk")) + monkeypatch.setattr("src.cache_manager.CacheManager", mock_cls) + validator = StartupValidator(make_config_manager(GOOD_CONFIG)) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is True + assert errors == [] + assert any(w.startswith("Could not validate cache directory:") for w in warnings) + + +class TestPlugins: + """Plugin validation with a plugin manager present.""" + + def _config_with_plugins(self): + return { + 'display': {'hardware': {'rows': 32, 'cols': 64}}, + 'schedule': {'enabled': True}, # reserved key that LOOKS enabled + 'timezone': 'UTC', + 'plugin_system': {}, + 'known': {'enabled': True}, + 'ghost': {'enabled': True}, + } + + def test_ghost_plugin_warns_and_reserved_keys_skipped(self, good_cache, tmp_path): + pm = MagicMock() + pm.discover_plugins.return_value = ['known'] + known_dir = tmp_path / "known" + known_dir.mkdir() + (known_dir / "manifest.json").write_text("{}") + pm.get_plugin_directory.return_value = str(known_dir) + + validator = StartupValidator(make_config_manager(self._config_with_plugins()), pm) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is True + assert "Plugin 'ghost' is enabled but not found in plugins directory" in warnings + # Reserved sections are never treated as plugins, even when they + # contain an 'enabled' flag (schedule above). + for reserved in ('display', 'schedule', 'timezone', 'plugin_system'): + assert not any(f"'{reserved}'" in w for w in warnings) + + def test_enabled_plugin_missing_manifest_is_error(self, good_cache, tmp_path): + pm = MagicMock() + pm.discover_plugins.return_value = ['known'] + plugin_dir = tmp_path / "known" + plugin_dir.mkdir() # exists, but no manifest.json inside + pm.get_plugin_directory.return_value = str(plugin_dir) + + config = dict(GOOD_CONFIG, known={'enabled': True}) + validator = StartupValidator(make_config_manager(config), pm) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is False + assert "Plugin 'known' manifest.json not found" in errors + + def test_disabled_plugin_not_checked_for_manifest(self, good_cache, tmp_path): + pm = MagicMock() + pm.discover_plugins.return_value = ['known'] + pm.get_plugin_directory.return_value = str(tmp_path / "nowhere") + + config = dict(GOOD_CONFIG, known={'enabled': False}) + validator = StartupValidator(make_config_manager(config), pm) + is_valid, errors, warnings = validator.validate_all() + assert is_valid is True + assert errors == [] + + +class TestIdempotence: + """validate_all() resets error/warning state each run (the fixed bug).""" + + def test_repeated_runs_do_not_accumulate(self, good_cache): + validator = StartupValidator(make_config_manager({})) + first = validator.validate_all() + second = validator.validate_all() + assert first == second + assert len(second[1]) == len(first[1]) + assert len(second[2]) == len(first[2]) + + +class TestRaiseOnErrors: + """Exception classification and precedence in raise_on_errors().""" + + def _validator(self, errors): + validator = StartupValidator(make_config_manager(GOOD_CONFIG)) + validator.errors = list(errors) + return validator + + def test_no_errors_returns_none(self): + assert self._validator([]).raise_on_errors() is None + + def test_config_error(self): + msg = "Missing required configuration key: display" + with pytest.raises(ConfigError) as excinfo: + self._validator([msg]).raise_on_errors() + assert excinfo.value.message == "Configuration validation failed" + assert msg in excinfo.value.context['errors'] + + def test_cache_error(self): + msg = "Cache directory does not exist: /nope" + with pytest.raises(CacheError) as excinfo: + self._validator([msg]).raise_on_errors() + assert msg in excinfo.value.context['errors'] + + def test_plugin_error(self): + msg = "Plugin 'known' manifest.json not found" + with pytest.raises(PluginError) as excinfo: + self._validator([msg]).raise_on_errors() + assert msg in excinfo.value.context['errors'] + + def test_unclassified_error_falls_back_to_config_error(self): + msg = "Something entirely else went wrong" + with pytest.raises(ConfigError) as excinfo: + self._validator([msg]).raise_on_errors() + assert excinfo.value.message == "Startup validation failed" + assert msg in excinfo.value.context['errors'] + + def test_precedence_config_beats_cache(self): + # A message matching both 'config' and 'cache' substrings raises + # ConfigError because config classification is checked first. + msg = "config problem touching the cache layer" + with pytest.raises(ConfigError) as excinfo: + self._validator([msg]).raise_on_errors() + assert excinfo.value.message == "Configuration validation failed" + assert msg in excinfo.value.context['errors'] diff --git a/test/web_interface/test_api_v3_secret_roundtrip.py b/test/web_interface/test_api_v3_secret_roundtrip.py new file mode 100644 index 00000000..a4648824 --- /dev/null +++ b/test/web_interface/test_api_v3_secret_roundtrip.py @@ -0,0 +1,259 @@ +""" +End-to-end secret round-trips through the three api_v3 endpoints that +separate secrets from regular config (main-config save, plugin-config save, +plugin-config reset) — now backed by the canonical +src/web_interface/secret_helpers implementations. + +Unlike test_web_api.py (which mocks the config manager), these tests run a +REAL ConfigManager and a REAL SchemaManager over tmp_path files, so they +prove the whole chain: endpoint separation -> config_secrets.json write -> +atomic config.json save (strip) -> load_config (merge back), including the +array-item secret shape (accounts[].token) the inline copies never +supported. +""" + +import json +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from flask import Flask + +project_root = Path(__file__).parent.parent.parent +sys.path.insert(0, str(project_root)) + +from src.config_manager import ConfigManager # noqa: E402 +from src.plugin_system.schema_manager import SchemaManager # noqa: E402 +from web_interface.blueprints.api_v3 import api_v3 # noqa: E402 + + +PLUGIN_ID = "testplugin" + +SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "enabled": {"type": "boolean", "default": True}, + "display_duration": {"type": "number", "default": 15}, + "api_key": {"type": "string", "x-secret": True, "default": ""}, + "city": {"type": "string", "default": "Austin"}, + "accounts": { + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "token": {"type": "string", "x-secret": True}, + }, + }, + }, + }, +} + + +@pytest.fixture +def env(tmp_path): + """Real ConfigManager + SchemaManager over tmp_path, wired onto the + api_v3 blueprint with the remaining managers mocked.""" + config_file = tmp_path / "config.json" + config_file.write_text("{}") + + plugins_dir = tmp_path / "plugins" + plugin_dir = plugins_dir / PLUGIN_ID + plugin_dir.mkdir(parents=True) + (plugin_dir / "config_schema.json").write_text(json.dumps(SCHEMA)) + (plugin_dir / "manifest.json").write_text(json.dumps({ + "id": PLUGIN_ID, "name": "Test Plugin", "version": "1.0.0", + })) + + config_manager = ConfigManager( + config_path=str(config_file), + secrets_path=str(tmp_path / "config_secrets.json")) + config_manager.template_path = str(tmp_path / "no-template.json") + + schema_manager = SchemaManager(plugins_dir=plugins_dir, + project_root=tmp_path) + + plugin_manager = MagicMock() + plugin_manager.plugin_manifests = {PLUGIN_ID: {"id": PLUGIN_ID}} + plugin_manager.plugins_dir = plugins_dir + plugin_manager.get_plugin.return_value = None + + api_v3.config_manager = config_manager + api_v3.schema_manager = schema_manager + api_v3.plugin_manager = plugin_manager + api_v3.plugin_store_manager = MagicMock() + api_v3.saved_repositories_manager = MagicMock() + api_v3.operation_queue = MagicMock() + api_v3.plugin_state_manager = MagicMock() + api_v3.operation_history = MagicMock() + api_v3.cache_manager = MagicMock() + + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(api_v3, url_prefix="/api/v3") + + class Env: + pass + + e = Env() + e.client = app.test_client() + e.config_manager = config_manager + e.config_file = config_file + e.secrets_file = tmp_path / "config_secrets.json" + e.tmp_path = tmp_path + + def fresh_load(): + """Load via a NEW ConfigManager, as the next request/process would. + + The endpoint's manager serves its post-save in-memory config via the + mtime fast path, and that copy predates the secrets it just + separated out — a pre-existing quirk that applies to scalar secrets + too. On-disk truth is what these tests care about. + """ + fresh = ConfigManager(config_path=str(config_file), + secrets_path=str(e.secrets_file)) + fresh.template_path = str(tmp_path / "no-template.json") + return fresh.load_config() + + e.fresh_load = fresh_load + return e + + +def _on_disk(path): + return json.loads(path.read_text()) + + +class TestSaveMainConfig: + """Site A: POST /config/main with a plugin-id key.""" + + def test_array_and_scalar_secrets_routed_to_secrets_file(self, env): + resp = env.client.post("/api/v3/config/main", json={ + PLUGIN_ID: { + "city": "Dallas", + "api_key": "s3cret-key", + "accounts": [ + {"name": "a", "token": "s3cret-a"}, + {"name": "b"}, + ], + }, + }) + assert resp.status_code == 200, resp.get_json() + + on_disk = _on_disk(env.config_file) + assert on_disk[PLUGIN_ID]["city"] == "Dallas" + assert "api_key" not in on_disk[PLUGIN_ID] + assert on_disk[PLUGIN_ID]["accounts"] == [{"name": "a"}, {"name": "b"}] + assert "s3cret" not in env.config_file.read_text() + + secrets = _on_disk(env.secrets_file) + assert secrets[PLUGIN_ID]["api_key"] == "s3cret-key" + assert secrets[PLUGIN_ID]["accounts"] == [{"token": "s3cret-a"}, {}] + + def test_load_config_merges_secrets_back(self, env): + env.client.post("/api/v3/config/main", json={ + PLUGIN_ID: {"accounts": [{"name": "a", "token": "s3cret-a"}]}, + }) + merged = env.fresh_load() + assert merged[PLUGIN_ID]["accounts"] == [ + {"name": "a", "token": "s3cret-a"}] + + +class TestSavePluginConfig: + """Site B: POST /plugins/config (JSON body).""" + + def _save(self, env, config): + return env.client.post("/api/v3/plugins/config", json={ + "plugin_id": PLUGIN_ID, "config": config, + }) + + def test_round_trip_with_array_secrets(self, env): + resp = self._save(env, { + "enabled": True, + "city": "Houston", + "api_key": "s3cret-key", + "accounts": [ + {"name": "a", "token": "s3cret-a"}, + {"name": "b", "token": "s3cret-b"}, + ], + }) + assert resp.status_code == 200, resp.get_json() + + assert "s3cret" not in env.config_file.read_text() + on_disk = _on_disk(env.config_file) + assert on_disk[PLUGIN_ID]["accounts"] == [{"name": "a"}, {"name": "b"}] + + secrets = _on_disk(env.secrets_file) + assert secrets[PLUGIN_ID]["accounts"] == [ + {"token": "s3cret-a"}, {"token": "s3cret-b"}] + + merged = env.fresh_load() + assert merged[PLUGIN_ID]["accounts"][1]["token"] == "s3cret-b" + + def test_secret_count_message_counts_top_level_keys(self, env): + # Pinned: the "(N secret field(s))" message counts TOP-LEVEL keys of + # the separated secrets dict. Here that is 2: the posted accounts + # array (all its item tokens count as ONE key) plus the schema's + # api_key default ("") that merge_with_defaults adds before + # separation. + resp = self._save(env, { + "accounts": [{"name": "a", "token": "t"}], + }) + message = resp.get_json()["message"] + assert "(2 secret field(s) saved to config_secrets.json)" in message + + def test_resave_replaces_stored_secrets_list_wholesale(self, env): + # Characterized: api_v3's deep_merge intentionally replaces lists, + # so a re-save's parallel secrets list is authoritative. + self._save(env, {"accounts": [ + {"name": "a", "token": "old-a"}, + {"name": "b", "token": "old-b"}, + ]}) + self._save(env, {"accounts": [{"name": "only", "token": "new-only"}]}) + + secrets = _on_disk(env.secrets_file) + assert secrets[PLUGIN_ID]["accounts"] == [{"token": "new-only"}] + merged = env.fresh_load() + assert merged[PLUGIN_ID]["accounts"] == [ + {"name": "only", "token": "new-only"}] + + +class TestResetPluginConfig: + """Site C: POST /plugins/config/reset.""" + + def _seed(self, env): + env.client.post("/api/v3/plugins/config", json={ + "plugin_id": PLUGIN_ID, + "config": {"city": "Houston", "api_key": "s3cret-key", + "accounts": [{"name": "a", "token": "s3cret-a"}]}, + }) + + def test_reset_preserving_secrets(self, env): + self._seed(env) + resp = env.client.post("/api/v3/plugins/config/reset", json={ + "plugin_id": PLUGIN_ID, "preserve_secrets": True, + }) + assert resp.status_code == 200, resp.get_json() + + on_disk = _on_disk(env.config_file) + assert on_disk[PLUGIN_ID]["city"] == "Austin" # schema default + assert on_disk[PLUGIN_ID]["accounts"] == [] # schema default + + # Existing secrets survive (top-level-only preserve merge, pinned). + secrets = _on_disk(env.secrets_file) + assert secrets[PLUGIN_ID]["api_key"] == "s3cret-key" + assert secrets[PLUGIN_ID]["accounts"] == [{"token": "s3cret-a"}] + + def test_reset_without_preserving_secrets(self, env): + self._seed(env) + resp = env.client.post("/api/v3/plugins/config/reset", json={ + "plugin_id": PLUGIN_ID, "preserve_secrets": False, + }) + assert resp.status_code == 200, resp.get_json() + + secrets = _on_disk(env.secrets_file) + # Replaced with schema-default secrets — the schema declares no + # secret defaults, so the plugin's secrets are emptied. + assert secrets[PLUGIN_ID] in ({}, {"api_key": ""}) diff --git a/test/web_interface/test_secret_separation_parity.py b/test/web_interface/test_secret_separation_parity.py index 65c5576e..91367234 100644 --- a/test/web_interface/test_secret_separation_parity.py +++ b/test/web_interface/test_secret_separation_parity.py @@ -1,18 +1,13 @@ """ -Drift guard for the duplicated secret-separation logic. +Drift guard: api_v3 must use the canonical secret helpers. -src/web_interface/secret_helpers.py is the canonical implementation of -find_secret_fields / separate_secrets, but web_interface/blueprints/api_v3.py -still carries THREE inline nested-function copies of each (in the plugin -config GET, POST, and reset endpoints). The copies lack the canonical -module's array-item support (`accounts[].token`), so migrating an endpoint -onto the module is a behavior change that must be made deliberately. - -This file guards two things: -1. The copy count can only go DOWN. A fourth copy appearing means someone - re-implemented the logic again instead of importing secret_helpers. -2. The known behavioral gap is documented as an executable fact, so whoever - migrates the endpoints knows exactly what changes. +Historically web_interface/blueprints/api_v3.py carried THREE inline +nested-function copies of ``find_secret_fields``/``separate_secrets`` (in the +main-config save, plugin-config save, and plugin-config reset endpoints). +They lacked the canonical module's array-item secret support and drifted from +each other. They have been migrated onto +``src/web_interface/secret_helpers`` — this file now guards against copies +REAPPEARING, and keeps the canonical array-item behavior executable. """ import re @@ -23,77 +18,44 @@ from src.web_interface.secret_helpers import find_secret_fields, separate_secret API_V3_PATH = (Path(__file__).resolve().parents[2] / "web_interface" / "blueprints" / "api_v3.py") -# Update DOWNWARD as endpoints migrate onto src/web_interface/secret_helpers. -EXPECTED_INLINE_COPIES = 3 +# The migration is complete: any inline reimplementation is a regression. +EXPECTED_INLINE_COPIES = 0 -class TestInlineCopyCount: +class TestNoInlineCopies: def _count(self, name: str) -> int: source = API_V3_PATH.read_text(encoding="utf-8") return len(re.findall(rf"^\s*def {name}\(", source, flags=re.MULTILINE)) - def test_find_secret_fields_copy_count(self): + def test_no_inline_find_secret_fields(self): count = self._count("find_secret_fields") assert count == EXPECTED_INLINE_COPIES, ( f"api_v3.py has {count} inline find_secret_fields definitions, " - f"expected {EXPECTED_INLINE_COPIES}. New code must import it from " - f"src/web_interface/secret_helpers instead of re-implementing it; " - f"if you migrated an endpoint, lower EXPECTED_INLINE_COPIES." + f"expected {EXPECTED_INLINE_COPIES}. Import it from " + f"src/web_interface/secret_helpers instead of re-implementing it." ) - def test_separate_secrets_copy_count(self): + def test_no_inline_separate_secrets(self): count = self._count("separate_secrets") assert count == EXPECTED_INLINE_COPIES, ( f"api_v3.py has {count} inline separate_secrets definitions, " - f"expected {EXPECTED_INLINE_COPIES}. New code must import it from " - f"src/web_interface/secret_helpers instead of re-implementing it; " - f"if you migrated an endpoint, lower EXPECTED_INLINE_COPIES." + f"expected {EXPECTED_INLINE_COPIES}. Import it from " + f"src/web_interface/secret_helpers instead of re-implementing it." ) - def test_inline_copies_lack_array_item_support(self): - """The documented gap: no inline copy recurses into array `items` - schemas, so array-item secrets (accounts[].token) are NOT routed to - config_secrets.json by these endpoints. The canonical module handles - them. When an endpoint migrates onto the module that behavior - changes (a fix, but a deliberate one). - - If this fails, an inline copy has grown array support — duplicating - the canonical module even harder. Migrate the endpoint onto - src/web_interface/secret_helpers instead. - """ - for body in self._inline_bodies("find_secret_fields"): - # Array handling requires checking type == 'array'; no inline - # copy does. (Can't grep bare "items" — properties.items() the - # dict method appears legitimately.) - assert "'array'" not in body and '"array"' not in body - - @staticmethod - def _inline_bodies(name: str): - """Extract each inline def's body from api_v3.py by indentation.""" - lines = API_V3_PATH.read_text(encoding="utf-8").splitlines() - bodies = [] - i = 0 - while i < len(lines): - match = re.match(rf"^(\s+)def {name}\(", lines[i]) - if not match: - i += 1 - continue - indent = len(match.group(1)) - body = [lines[i]] - i += 1 - while i < len(lines): - line = lines[i] - if line.strip() and (len(line) - len(line.lstrip())) <= indent: - break - body.append(line) - i += 1 - bodies.append("\n".join(body)) - assert bodies, f"no inline {name} definitions found" - return bodies + def test_canonical_import_present(self): + # Tripwire: the endpoints still need the helpers, so removing the + # import means either dead secret handling or a new local copy. + source = API_V3_PATH.read_text(encoding="utf-8") + assert re.search( + r"from src\.web_interface\.secret_helpers import .*find_secret_fields", + source, + ), "api_v3.py no longer imports the canonical secret helpers" class TestCanonicalArrayItemBehavior: - """Executable documentation of what migrating endpoints will change.""" + """Executable documentation of the array-item secret contract the + endpoints now inherit from the canonical module.""" SCHEMA = { "accounts": { diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 4c80ed07..68e01408 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -21,6 +21,7 @@ logger = logging.getLogger(__name__) # Import new infrastructure from src.web_interface.api_helpers import success_response, error_response, validate_request_json from src.web_interface.errors import ErrorCode +from src.web_interface.secret_helpers import find_secret_fields, separate_secrets from src.plugin_system.operation_types import OperationType from src.web_interface.validators import ( validate_file_upload @@ -1196,18 +1197,6 @@ def save_main_config(): plugins_dir = PROJECT_ROOT / plugins_dir_name schema_path = plugins_dir / plugin_id / 'config_schema.json' - def find_secret_fields(properties, prefix=''): - """Recursively find fields marked with x-secret: true""" - fields = set() - for field_name, field_props in properties.items(): - full_path = f"{prefix}.{field_name}" if prefix else field_name - if field_props.get('x-secret', False): - fields.add(full_path) - # Check nested objects - if field_props.get('type') == 'object' and 'properties' in field_props: - fields.update(find_secret_fields(field_props['properties'], full_path)) - return fields - if schema_path.exists(): try: with open(schema_path, 'r', encoding='utf-8') as f: @@ -1218,24 +1207,6 @@ def save_main_config(): logger.debug("Error reading schema for secret detection: %s", e) # Separate secrets from regular config (same logic as save_plugin_config) - def separate_secrets(config, secrets_set, prefix=''): - """Recursively separate secret fields from regular config""" - regular = {} - secrets = {} - for key, value in config.items(): - full_path = f"{prefix}.{key}" if prefix else key - if isinstance(value, dict): - nested_regular, nested_secrets = separate_secrets(value, secrets_set, full_path) - if nested_regular: - regular[key] = nested_regular - if nested_secrets: - secrets[key] = nested_secrets - elif full_path in secrets_set: - secrets[key] = value - else: - regular[key] = value - return regular, secrets - regular_config, secrets_config = separate_secrets(plugin_config, secret_fields) # PRE-PROCESSING: Preserve 'enabled' state if not in regular_config @@ -4276,6 +4247,11 @@ def deep_merge(base_dict, update_dict): """ Deep merge update_dict into base_dict. For nested dicts, recursively merge. For other types, update_dict takes precedence. + + Lists are intentionally REPLACED wholesale, never index-merged: form posts + carry complete arrays, and index-merging would resurrect items the user + deleted. This also applies to the parallel secrets lists produced by + separate_secrets — a newly saved secrets list is authoritative. """ result = base_dict.copy() for key, value in update_dict.items(): @@ -5290,23 +5266,9 @@ def save_plugin_config(): # Default to True on error to avoid disabling plugins plugin_config['enabled'] = True - # Find secret fields (supports nested schemas) + # Find secret fields (supports nested schemas and array-item secrets) secret_fields = set() - def find_secret_fields(properties, prefix=''): - """Recursively find fields marked with x-secret: true""" - fields = set() - if not isinstance(properties, dict): - return fields - for field_name, field_props in properties.items(): - full_path = f"{prefix}.{field_name}" if prefix else field_name - if isinstance(field_props, dict) and field_props.get('x-secret', False): - fields.add(full_path) - # Check nested objects - if isinstance(field_props, dict) and field_props.get('type') == 'object' and 'properties' in field_props: - fields.update(find_secret_fields(field_props['properties'], full_path)) - return fields - if schema and 'properties' in schema: secret_fields = find_secret_fields(schema['properties']) @@ -5621,29 +5583,8 @@ def save_plugin_config(): status_code=400 ) - # Separate secrets from regular config (handles nested configs) - def separate_secrets(config, secrets_set, prefix=''): - """Recursively separate secret fields from regular config""" - regular = {} - secrets = {} - - for key, value in config.items(): - full_path = f"{prefix}.{key}" if prefix else key - - if isinstance(value, dict): - # Recursively handle nested dicts - nested_regular, nested_secrets = separate_secrets(value, secrets_set, full_path) - if nested_regular: - regular[key] = nested_regular - if nested_secrets: - secrets[key] = nested_secrets - elif full_path in secrets_set: - secrets[key] = value - else: - regular[key] = value - - return regular, secrets - + # Separate secrets from regular config (handles nested configs and + # array-item secrets — see src/web_interface/secret_helpers.py) regular_config, secrets_config = separate_secrets(plugin_config, secret_fields) # Get current configs @@ -5891,41 +5832,10 @@ def reset_plugin_config(): schema = schema_mgr.load_schema(plugin_id, use_cache=True) secret_fields = set() - def find_secret_fields(properties, prefix=''): - """Recursively find fields marked with x-secret: true""" - fields = set() - if not isinstance(properties, dict): - return fields - for field_name, field_props in properties.items(): - full_path = f"{prefix}.{field_name}" if prefix else field_name - if isinstance(field_props, dict) and field_props.get('x-secret', False): - fields.add(full_path) - if isinstance(field_props, dict) and field_props.get('type') == 'object' and 'properties' in field_props: - fields.update(find_secret_fields(field_props['properties'], full_path)) - return fields - if schema and 'properties' in schema: secret_fields = find_secret_fields(schema['properties']) # Separate defaults into regular and secret configs - def separate_secrets(config, secrets_set, prefix=''): - """Recursively separate secret fields from regular config""" - regular = {} - secrets = {} - for key, value in config.items(): - full_path = f"{prefix}.{key}" if prefix else key - if isinstance(value, dict): - nested_regular, nested_secrets = separate_secrets(value, secrets_set, full_path) - if nested_regular: - regular[key] = nested_regular - if nested_secrets: - secrets[key] = nested_secrets - elif full_path in secrets_set: - secrets[key] = value - else: - regular[key] = value - return regular, secrets - default_regular, default_secrets = separate_secrets(defaults, secret_fields) # Update main config with defaults From 6287acd591fde7db99a6a219b32bd0731dfe6089 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:17:28 -0400 Subject: [PATCH 09/53] fix(vegas): stop capping plugin width by default (#445) * fix(vegas): stop capping plugin width by default Vegas plugins read as "cut off early" or "starting in the middle". That was the per-plugin width budget, not the scroll engine: overflow_mode=rotate is designed to resume mid-content on each appearance, so the symptom was the feature working as specified. Measured over a 17-plugin fleet on a 512px panel, the cap was a bad trade. Only four plugins were ever wide enough to hit the 3.0 default -- leaderboard 11,518px, news 10,021px, odds-ticker 4,643px, hockey 1,508px. Weather is 650px and flights 512px; the cap never touched them or the other eleven. So it bought nothing on thirteen plugins while costing two visible faults on four: content entering mid-item (a news ticker started at column 6027 of its own strip), and a final rotation window of whatever happened to be left -- 348px of an 1,840px stocks ticker, seven seconds of panel time. Default max_plugin_width_ratio to 0 (uncapped), so every plugin contributes all of its content and is always entered at its beginning. The cap remains available, and vegas_max_width_screens still caps an individual plugin -- which is where the knob belongs, since a genuinely long ticker is a property of that plugin rather than of the fleet. Verified on a live 512px device: 327 budget crops in the preceding six hours, none after. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 * test(vegas): cover the width cap still working when asked for Defaulting the cap off must not quietly remove it. Asserts that an explicit ratio is honoured and validates, alongside the existing check that omitting it means uncapped. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 --------- Co-authored-by: Claude Opus 5 (1M context) --- config/config.template.json | 2 +- docs/CONFIG_REFERENCE.md | 2 +- src/vegas_mode/config.py | 22 ++++++++++++++----- test/test_vegas_density.py | 14 ++++++++++++ .../templates/v3/partials/display.html | 4 ++-- 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/config/config.template.json b/config/config.template.json index 433039f7..9edbec18 100644 --- a/config/config.template.json +++ b/config/config.template.json @@ -149,7 +149,7 @@ "min_plugin_width": 8, "lead_in_width": 0, "plugins_per_cycle": 6, - "max_plugin_width_ratio": 3.0, + "max_plugin_width_ratio": 0.0, "overflow_mode": "rotate", "dynamic_duration_enabled": true, "min_cycle_duration": 60, diff --git a/docs/CONFIG_REFERENCE.md b/docs/CONFIG_REFERENCE.md index a7bd81df..ff623439 100644 --- a/docs/CONFIG_REFERENCE.md +++ b/docs/CONFIG_REFERENCE.md @@ -127,7 +127,7 @@ Read by `src/vegas_mode/config.py` (`VegasScrollConfig.from_config`). See | `min_plugin_width` | int, `8` | | `lead_in_width` | int, `0` | | `plugins_per_cycle` | int, `6` | -| `max_plugin_width_ratio` | float, `3.0` | +| `max_plugin_width_ratio` | float, `0.0` | | `overflow_mode` | string, `"rotate"` | | `dynamic_duration_enabled` | bool, `true` | | `min_cycle_duration` | int, `60` | diff --git a/src/vegas_mode/config.py b/src/vegas_mode/config.py index 14786481..551da4d8 100644 --- a/src/vegas_mode/config.py +++ b/src/vegas_mode/config.py @@ -104,10 +104,22 @@ class VegasModeConfig: overflow_mode: str = "rotate" # Cap on one plugin's share of a cycle, as a multiple of display width. - # A single ticker returning 7,000px would otherwise hold the panel for over - # two minutes. Overflow is deferred to later cycles rather than discarded. - # 0 disables the cap. - max_plugin_width_ratio: float = 3.0 + # 0 (the default) disables the cap, so every plugin contributes all of its + # content and is always entered at its beginning. + # + # Capping was the default until it proved to cost more than it bought. + # Measured over a 17-plugin fleet on a 512px panel, only four plugins were + # ever wide enough to hit a 3.0 cap; for those four it produced two visible + # faults. Content resumed mid-item on each appearance (a news ticker entered + # at column 6027 of its own strip), and the final window of a rotation was + # whatever happened to be left — 348px of a 1840px stocks ticker, seven + # seconds of panel time. Both read as the display being broken rather than + # as deferral working. + # + # A wide plugin does hold the panel for a long time uncapped: set the cap + # per plugin with vegas_max_width_screens where that matters, rather than + # globally where it mostly hurts plugins that were never the problem. + max_plugin_width_ratio: float = 0.0 # Plugin management plugin_order: List[str] = field(default_factory=list) @@ -159,7 +171,7 @@ class VegasModeConfig: lead_in_width=int(vegas_config.get('lead_in_width', 0)), plugins_per_cycle=int(vegas_config.get('plugins_per_cycle', 6)), max_plugin_width_ratio=float( - vegas_config.get('max_plugin_width_ratio', 3.0)), + vegas_config.get('max_plugin_width_ratio', 0.0)), overflow_mode=str(vegas_config.get('overflow_mode', 'rotate')), plugin_order=list(vegas_config.get('plugin_order', [])), excluded_plugins=set(vegas_config.get('excluded_plugins', [])), diff --git a/test/test_vegas_density.py b/test/test_vegas_density.py index f0c1fd72..4fc2db12 100644 --- a/test/test_vegas_density.py +++ b/test/test_vegas_density.py @@ -781,6 +781,20 @@ class TestNewConfigKeys: assert cfg.render_width_pct == 100 assert cfg.min_content_separation == 24 + def test_width_cap_is_off_by_default(self): + # Capping made wide plugins resume mid-content on every appearance and + # emit runt final windows; it is now opt-in per plugin instead. + assert VegasModeConfig().max_plugin_width_ratio == 0.0 + assert VegasModeConfig.from_config({}).max_plugin_width_ratio == 0.0 + + def test_width_cap_is_still_available_when_asked_for(self): + # Defaulting the cap off must not remove it: a user who sets a ratio + # still gets one, and 0 still means uncapped. + cfg = VegasModeConfig.from_config( + {'display': {'vegas_scroll': {'max_plugin_width_ratio': 3.0}}}) + assert cfg.max_plugin_width_ratio == 3.0 + assert cfg.validate() == [] + @pytest.mark.parametrize('overrides,bad_key', [ ({'render_width_pct': 5}, 'render_width_pct'), ({'render_width_pct': 101}, 'render_width_pct'), diff --git a/web_interface/templates/v3/partials/display.html b/web_interface/templates/v3/partials/display.html index 2fa31027..1e2c423a 100644 --- a/web_interface/templates/v3/partials/display.html +++ b/web_interface/templates/v3/partials/display.html @@ -556,11 +556,11 @@
- + Date: Mon, 10 Aug 2026 08:17:57 -0400 Subject: [PATCH 10/53] test(harness): flag a mode that draws nothing without reporting it (#447) The controller skips a mode whose display() returns False and treats anything else -- including None -- as "content was shown". A mode that draws nothing and does not return False is therefore never skipped, and because a mode switch clears the panel first, it sits on a blank screen for its whole display duration. Two sports plugins shipped exactly that. The harness rendered those modes and passed them, because it called display() and discarded the result. Capture it, and warn when a render produced no lit pixels while claiming content. Warn-only by default, and deliberately so: a scroll mode's first frame is legitimately its blank scroll-in buffer, which is 42 of these on the F1 scoreboard alone. Plugins whose modes are known to draw on their fixture data can opt into failing via harness.json {"empty_check": "strict"}, matching how the fill check is staged. Worth being clear about the limit: this only sees what the fixtures render. It would not have caught the sports bug, whose fixture seeds games so the empty path never renders -- that needs the source-level gate in the plugins repo. What it does catch is the same mistake in any plugin whose empty state the harness does happen to reach, which is coverage there was none of before. Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 Co-authored-by: Claude Opus 5 (1M context) --- scripts/check_plugin.py | 15 ++++ src/plugin_system/testing/harness.py | 61 ++++++++++++++-- test/test_harness_empty_claimed.py | 103 +++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 test/test_harness_empty_claimed.py diff --git a/scripts/check_plugin.py b/scripts/check_plugin.py index 997bf6d8..3f111d4d 100644 --- a/scripts/check_plugin.py +++ b/scripts/check_plugin.py @@ -41,6 +41,7 @@ from src.plugin_system.testing.loading import ( # noqa: E402 ) from src.plugin_system.testing.harness import ( # noqa: E402 RenderResult, render_plugin_matrix, compare_to_goldens, write_goldens, + check_empty_claimed, check_scale_up, ) from src.plugin_system.testing.sizes import ( # noqa: E402 @@ -115,6 +116,11 @@ def check_one(plugin_id: str, search_dirs: List[str], sizes, mock_data: Dict, declared = load_manifest(plugin_dir).get("display", {}).get("design_size", {}) design_size = (int(declared.get("width", 128)), int(declared.get("height", 32))) fill_strict = spec.get("fill_check") == "strict" + # A mode that renders nothing without returning False is never skipped by + # the display controller, so it holds a blank panel for its whole duration. + # Warn-only by default: a scroll mode's first frame is legitimately its + # blank scroll-in buffer. + empty_strict = spec.get("empty_check") == "strict" # Every run: the base config, plus one per harness.json "variant" — # a config overlay with its own golden dir (e.g. adaptive layout mode @@ -142,6 +148,7 @@ def check_one(plugin_id: str, search_dirs: List[str], sizes, mock_data: Dict, compare_to_goldens(results, golden_dir) check_scale_up(results, design_size=design_size, strict=fill_strict) + check_empty_claimed(results, strict=empty_strict) # Tag variant runs so the report and PNG dumps stay distinguishable. if variant_name: @@ -178,6 +185,9 @@ def print_report(all_results: Dict[str, List[RenderResult]]) -> bool: # warn-only underfill: big panel left mostly empty ex, ey = r.fill_extent detail += f" (fill warn: extent {ex:.0%}x{ey:.0%})" + if r.empty_claimed and r.empty_ok is None: + detail += (f" (empty warn: drew nothing but display() returned" + f" {r.display_returned!r}, so the mode is not skipped)") else: everything_ok = False if r.error is not None: @@ -191,6 +201,11 @@ def print_report(all_results: Dict[str, List[RenderResult]]) -> bool: ex, ey = r.fill_extent or (0.0, 0.0) status = "FAIL" detail = f" fill: extent {ex:.0%}x{ey:.0%} below required coverage" + elif r.empty_ok is False: + status = "FAIL" + detail = (f" drew nothing but display() returned" + f" {r.display_returned!r}; return False so the" + f" controller skips the mode") else: status, detail = "FAIL", "" print(f" [{status}] {r.size_label:>7} {r.mode}{detail}") diff --git a/src/plugin_system/testing/harness.py b/src/plugin_system/testing/harness.py index ee4d6613..51c688a2 100644 --- a/src/plugin_system/testing/harness.py +++ b/src/plugin_system/testing/harness.py @@ -73,6 +73,11 @@ class RenderResult: golden_ok: Optional[bool] = None golden_diff_pixels: int = 0 golden_max_delta: int = 0 + # what display() handed back; the controller skips a mode only on False + display_returned: Any = None + # empty-frame check: rendered nothing while not reporting "no content" + empty_claimed: Optional[bool] = None # True when that happened + empty_ok: Optional[bool] = None # False only in strict mode # fill / scale-up check (populated only for sizes >= 2x the design size) fill_checked: bool = False fill_ok: Optional[bool] = None # False only in strict mode @@ -92,6 +97,8 @@ class RenderResult: return False if self.fill_ok is False: return False + if self.empty_ok is False: + return False return True @@ -132,21 +139,25 @@ def _instantiate(plugin_id: str, manifest: Dict[str, Any], plugin_dir: Path, return plugin_instance -def _render_mode(plugin_instance: Any, mode: str) -> None: +def _render_mode(plugin_instance: Any, mode: str) -> Any: """Render a specific screen. Prefer an explicit display_mode kwarg; otherwise drive the plugin's internal mode state machine (first display() call renders - modes[current_mode_index] when current_display_mode is None).""" + modes[current_mode_index] when current_display_mode is None). + + Returns whatever display() returned. The display controller skips a mode + whose display() returns False, so that value decides whether an empty mode + is rotated past or sat on -- which makes it worth reporting rather than + discarding.""" sig = inspect.signature(plugin_instance.display) if "display_mode" in sig.parameters: - plugin_instance.display(force_clear=True, display_mode=mode) - return + return plugin_instance.display(force_clear=True, display_mode=mode) modes = getattr(plugin_instance, "modes", None) if modes and mode in modes: plugin_instance.current_mode_index = list(modes).index(mode) if hasattr(plugin_instance, "current_display_mode"): plugin_instance.current_display_mode = None - plugin_instance.display(force_clear=False) + return plugin_instance.display(force_clear=False) def _freeze(freeze_time: Optional[str]): @@ -234,7 +245,7 @@ def _render_size(plugin_id, manifest, plugin_dir, config, mock_data, logger.warning("update() raised a non-connectivity error for %s [%s]: %s", plugin_id, mode, e) if result.error is None: - _render_mode(inst, mode) + result.display_returned = _render_mode(inst, mode) result.image = dm.get_image() result.overflow = dm.check_overflow() except Exception as e: # noqa: BLE001 — a display crash is a real failure @@ -341,6 +352,44 @@ def fill_metrics(image: Image.Image) -> Tuple[float, float, float]: return (extent_x, extent_y, ink) +def check_empty_claimed(results: List[RenderResult], + strict: bool = False) -> List[RenderResult]: + """Flag a mode that rendered nothing without reporting "no content". + + The display controller skips a mode whose ``display()`` returns False, and + treats anything else -- including None -- as "content was shown". A mode + that draws nothing and does not return False therefore holds whatever is on + the panel for its whole display duration. Since a mode switch clears first, + that is a blank screen. Two sports plugins shipped exactly this: their + ``display()`` returned None on every path, so an out-of-season league sat + blank for its full duration rather than being rotated past. + + Warn-only by default, because a blank frame is not automatically wrong: a + scroll mode whose first frame is its blank scroll-in buffer renders empty + and is behaving correctly. ``strict=True`` sets ``empty_claimed`` such that + ``RenderResult.ok`` fails -- opt in per plugin via harness.json + ``{"empty_check": "strict"}`` once its modes are known to draw on the + fixture data. + + Note this can only catch what the fixtures actually render. A plugin whose + harness fixture seeds content never exercises its empty path here; the + source-level gate in the plugins repo covers that case. + """ + for r in results: + if r.image is None or r.error is not None: + continue + # An explicit False is the plugin correctly saying "nothing to show". + if r.display_returned is False: + continue + if r.image.convert("L").point( + lambda p: 255 if p > _LIT_THRESHOLD else 0).getbbox() is not None: + continue + r.empty_claimed = True + if strict: + r.empty_ok = False + return results + + def check_scale_up(results: List[RenderResult], design_size: Tuple[int, int] = (128, 32), min_extent: float = _MIN_FILL_EXTENT, diff --git a/test/test_harness_empty_claimed.py b/test/test_harness_empty_claimed.py new file mode 100644 index 00000000..2791516a --- /dev/null +++ b/test/test_harness_empty_claimed.py @@ -0,0 +1,103 @@ +"""Tests for the harness empty-frame check (src/plugin_system/testing/harness.py). + +The display controller skips a mode whose display() returns False and treats +anything else -- including None -- as "content was shown". A mode that draws +nothing without returning False is therefore never skipped, and since a mode +switch clears the panel first, it sits on a blank screen for its whole display +duration. + +Two sports plugins shipped exactly that: their display() returned None on every +path, so an out-of-season league held a blank panel instead of being rotated +past. The harness rendered those modes and passed them, because it discarded +the return value entirely. +""" + +from PIL import Image + +from src.plugin_system.testing.harness import RenderResult, check_empty_claimed + + +def _blank(w=64, h=32): + return Image.new("RGB", (w, h), (0, 0, 0)) + + +def _drawn(w=64, h=32): + img = _blank(w, h) + img.paste(Image.new("RGB", (10, 10), (255, 255, 255)), (5, 5)) + return img + + +def _result(image, returned=None, **kw): + return RenderResult("p", 64, 32, "mode", image=image, + display_returned=returned, **kw) + + +class TestCheckEmptyClaimed: + def test_blank_frame_returning_none_is_flagged(self): + # The shape that shipped: nothing drawn, nothing reported. + r = _result(_blank(), returned=None) + check_empty_claimed([r]) + assert r.empty_claimed is True + + def test_blank_frame_returning_true_is_flagged(self): + # Just as broken, and more explicit about it. + r = _result(_blank(), returned=True) + check_empty_claimed([r]) + assert r.empty_claimed is True + + def test_blank_frame_returning_false_is_fine(self): + # The plugin correctly said "no content"; the controller will skip it. + r = _result(_blank(), returned=False) + check_empty_claimed([r]) + assert r.empty_claimed is None + + def test_a_drawn_frame_is_fine_whatever_it_returns(self): + for returned in (None, True, False): + r = _result(_drawn(), returned=returned) + check_empty_claimed([r]) + assert r.empty_claimed is None, returned + + def test_near_black_still_counts_as_drawn(self): + # Guard the threshold: content dim enough to look black to the eye is + # still content, and flagging it would train people to ignore this. + img = _blank() + img.paste(Image.new("RGB", (4, 4), (60, 60, 60)), (2, 2)) + r = _result(img, returned=None) + check_empty_claimed([r]) + assert r.empty_claimed is None + + +class TestWarnVersusStrict: + def test_warn_only_by_default(self): + # A scroll mode's first frame is legitimately its blank scroll-in + # buffer, so this must not fail a run unless opted in. + r = _result(_blank(), returned=None) + check_empty_claimed([r]) + assert r.empty_ok is None + assert r.ok is True + + def test_strict_fails_the_result(self): + r = _result(_blank(), returned=None) + check_empty_claimed([r], strict=True) + assert r.empty_ok is False + assert r.ok is False + + def test_strict_still_allows_an_honest_false(self): + r = _result(_blank(), returned=False) + check_empty_claimed([r], strict=True) + assert r.empty_ok is None + assert r.ok is True + + +class TestSkippedResults: + def test_a_crashed_render_is_left_alone(self): + # error already fails the result; adding a second reason just muddies + # the report. + r = _result(None, returned=None, error="boom") + check_empty_claimed([r], strict=True) + assert r.empty_claimed is None + + def test_a_result_with_no_image_is_left_alone(self): + r = _result(None, returned=None) + check_empty_claimed([r], strict=True) + assert r.empty_claimed is None From ca26c1b83be4100778e92a54fd0deb4a2bd0b636 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Mon, 10 Aug 2026 09:41:25 -0400 Subject: [PATCH 11/53] fix(vegas): stop the width cap emitting fragments and stale windows (#446) * fix(vegas): stop the width cap emitting fragments and stale windows Two defects in the rotation that narrows an oversized plugin to its width budget. Both were found while investigating "cut off early / starts in the middle" reports and are the reason the cap is no longer on by default; they still bite anyone who sets one. A rotation's last window was whatever happened to be left over. Windows are placed by walking forward from the previous one, with nothing looking at the remainder, so a 1,840px stocks ticker against a 1,536px budget split 1,492 + 348 -- every other appearance showed seven seconds and cut. Absorb a remainder below half a budget into the window before it. That overruns the budget by at most half, which is the better trade: the budget guards against one plugin holding the panel for minutes, not against a 20% overshoot. The floor is measured against the budget rather than the panel because snapping to item boundaries already lands an ordinary window short of it -- a 512px budget over 182px-pitch items yields 348px windows, so an absolute floor merges windows that were never fragments. The stored offset also outlived the content it was recorded against. It was a pixel column, reused verbatim after the plugin re-rendered, so once anything ahead of it changed width the window pointed at unrelated items -- observed as news refreshing 9,793px -> 9,505px mid-rotation. Track the rotation as an index into the strip's item boundaries instead, since the Nth boundary survives a digit appearing in a price, and record alongside it what the offset indexes into: a row list, a boundary list, or a column in a gapless image. A mismatch restarts the rotation rather than reinterpreting the number, which also closes the case where one plugin's row index was read back as a pixel column after its content changed from several rows to one wide strip. Replaying the four plugins that actually hit the cap on a live 512px panel: no window is now a fragment, none exceeds 1.5 budgets, and every rotation still covers the whole strip. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 * fix(vegas): apply the runt floor to the multi-row rotation too The floor only guarded the single-image path. I had reasoned the row path could not produce a runt because it wraps, which is wrong: wrapping only helps when the row wrapped to actually fits. Rows of 450, 450 and 100 against a 512px budget give the 100 a pass of its own -- two seconds against nine, which is the symptom this branch exists to remove. Reproduced before changing anything: pass 1: 450px pass 2: 450px pass 3: 100px A window may now overrun the budget while it is still shorter than the floor, bounded at the same 1.5 budgets the single-image path allows, so the short row is carried with its neighbour instead of standing alone. pass 1: 450px pass 2: 450px pass 3: 550px A next row too wide to absorb within that cap still leaves a short window standing -- rows of 900 and 100 keep alternating. Merging them would mean a window of nearly two budgets, and the rule that always shows an oversized first row already makes the same trade. Three regression tests: the reported shape, that the overrun stays bounded when a row cannot be absorbed, and that absorbing never drops a row from the rotation. The single-image path is untouched -- the four plugins that actually hit the cap on a live panel replay identically. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 --------- Co-authored-by: Claude Opus 5 (1M context) --- src/vegas_mode/plugin_adapter.py | 173 +++++++++++++++++++++--- test/test_vegas_density.py | 225 +++++++++++++++++++++++++++++++ 2 files changed, 378 insertions(+), 20 deletions(-) diff --git a/src/vegas_mode/plugin_adapter.py b/src/vegas_mode/plugin_adapter.py index 0da51ce2..0ea71571 100644 --- a/src/vegas_mode/plugin_adapter.py +++ b/src/vegas_mode/plugin_adapter.py @@ -68,6 +68,21 @@ class PluginAdapter: # always the same opening items. self._item_offsets: dict = {} + # What the matching entry in _item_offsets is an offset *into*, as + # (kind, size). An offset only means anything against the content it + # was derived from, and there are three incompatible kinds: + # + # ('rows', n) index into a list of n images + # ('cuts', n) index into the n item boundaries of one image + # ('cols', w) pixel column in a w-wide image with no item boundaries + # + # Without this the offsets were reused across kinds — a plugin that + # returned one wide image on one fetch and several rows on the next had + # a pixel column of 1400 read back as a row index — and across content + # changes, where a column recorded against a 9,793px news strip pointed + # into unrelated headlines once the strip refreshed to 9,505px. + self._offset_shapes: dict = {} + logger.info( "PluginAdapter initialized: display=%dx%d", self.display_width, self.display_height @@ -398,6 +413,88 @@ class PluginAdapter: return 0 return int(self.display_width * ratio) + def _resume_offset(self, plugin_id: str, shape: Tuple[str, int]) -> int: + """ + The plugin's stored rotation offset, if it still applies. + + An offset is only meaningful against content shaped the way it was + when the offset was recorded. When the shape has changed — a different + number of rows, a re-rendered strip with different item boundaries — + the stored value points somewhere arbitrary, so rotation restarts. + + Args: + plugin_id: Plugin identifier + shape: (kind, size) describing what an offset would index into now + + Returns: + The stored offset, or 0 when it no longer applies + """ + if self._offset_shapes.get(plugin_id) != shape: + if plugin_id in self._item_offsets: + logger.info( + "[%s] Content is %s now, was %s — restarting the rotation " + "rather than resuming at a position that no longer means " + "anything", plugin_id, shape, + self._offset_shapes.get(plugin_id)) + self._item_offsets.pop(plugin_id, None) + self._offset_shapes[plugin_id] = shape + return 0 + return self._item_offsets.get(plugin_id, 0) + + def _record_offset( + self, plugin_id: str, offset: int, shape: Tuple[str, int] + ) -> None: + """Store where the next window should resume, with what it indexes.""" + if offset: + self._item_offsets[plugin_id] = offset + self._offset_shapes[plugin_id] = shape + else: + # A wrapped-to-zero rotation is the same as no state at all, and + # keeping the key would report a window as active when the next + # pass starts from the top anyway. + self._item_offsets.pop(plugin_id, None) + self._offset_shapes.pop(plugin_id, None) + + def _clear_offset(self, plugin_id: str) -> None: + """Forget any rotation state for a plugin.""" + self._item_offsets.pop(plugin_id, None) + self._offset_shapes.pop(plugin_id, None) + + def _merge_trailing_runt(self, end: int, width: int, budget: int) -> int: + """ + Extend a window to the end of the content when what would be left over + is too small to be worth its own pass. + + Windows were placed by walking forward from the last one, which makes + the final window whatever happens to remain. Measured on a live panel + that produced a 1,840px stocks ticker splitting 1,492 + 348 — the + second pass showing seven seconds of content before cutting, which + reads as the display failing rather than as a rotation. + + Absorbing the remainder overruns the budget by less than one window + floor, which is a better trade than a fragment: the budget is a guard + against one plugin holding the panel for minutes, not a hard limit. + + Args: + end: Column the window would otherwise end at + width: Full content width + budget: Width budget being applied + + Returns: + ``end``, or ``width`` when the remainder is below the floor + """ + remainder = width - end + # Measured against the budget rather than the panel: snapping to item + # boundaries means an ordinary window already lands short of the budget + # (a 512px budget over 182px-pitch items yields 348px windows), so an + # absolute floor would merge windows that were never fragments. Half a + # budget separates "a short last pass" from "a sliver", and caps the + # overrun this can cause at 1.5 budgets. + floor = budget // 2 + if 0 < remainder < floor: + return width + return end + def _apply_width_budget( self, images: List[Image.Image], plugin_id: str, plugin: Optional['BasePlugin'] = None @@ -435,32 +532,48 @@ class PluginAdapter: if not budget or total <= budget: # Fits, so reset rotation — the whole segment is being shown. - self._item_offsets.pop(plugin_id, None) + self._clear_offset(plugin_id) return images if len(images) == 1: return [self._crop_to_budget(images[0], budget, plugin_id, mode)] + shape = ('rows', len(images)) if mode == 'truncate': # Ordered content: always show from the top. Deliberately does not # advance the offset, so the same opening items appear every time # rather than the viewer being shown the middle of a ranked list. start = 0 else: - start = self._item_offsets.get(plugin_id, 0) % len(images) + start = self._resume_offset(plugin_id, shape) % len(images) selected: List[Image.Image] = [] used = 0 consumed = 0 # Walk forward from the rotation offset, taking whole items only, so a # cut never lands in the middle of one. + # + # A window may overrun the budget while it is still shorter than the + # runt floor, for the same reason _merge_trailing_runt exists on the + # single-image path: a pass far shorter than its neighbours reads as + # the display failing rather than as a rotation. Rows of 450, 450 and + # 100 against a 512px budget used to give the 100 a pass of its own -- + # two seconds against nine. Wrapping does not prevent that, because it + # only helps when the row wrapped to actually fits. + floor = budget // 2 for step in range(len(images)): img = images[(start + step) % len(images)] cost = img.width if selected: cost += self._row_gap(selected[-1], img) if selected and used + cost > budget: - break + # Keep the overrun bounded at the same 1.5 budgets the + # single-image path allows. A next row too wide to absorb + # leaves a short window standing -- better than a window of + # 1.9 budgets, and the same trade the always-take-the-first + # rule below already makes. + if used >= floor or used + cost > budget + floor: + break selected.append(img) used += cost consumed += 1 @@ -472,7 +585,8 @@ class PluginAdapter: plugin_id, budget, len(selected), len(images), used ) else: - self._item_offsets[plugin_id] = (start + consumed) % len(images) + self._record_offset( + plugin_id, (start + consumed) % len(images), shape) logger.info( "[%s] Width budget %dpx: showing %d of %d row(s) (%dpx incl. gaps) " "from offset %d; remainder deferred to a later cycle", @@ -490,16 +604,13 @@ class PluginAdapter: The cut is snapped to the nearest blank column so it does not slice through a glyph or logo and leave half a character at the panel edge. - """ - if mode == 'truncate': - # Always the start of the strip, so a ranked table is never entered - # from the middle. - offset = 0 - else: - offset = self._item_offsets.get(plugin_id, 0) - if offset >= img.width: - offset = 0 + Rotation is tracked as an index into the strip's item boundaries rather + than as a pixel column, because a ticker re-renders between fetches. A + column recorded against one render points at unrelated content in the + next as soon as anything ahead of it changes width — a digit in a + price, a shorter headline. The Nth boundary stays the Nth boundary. + """ # Cut only where the plugin left a real gap between items. Snapping to # any blank column used to pick the single-column gaps between # characters, splitting a word and orphaning its tail into the next @@ -514,9 +625,17 @@ class PluginAdapter: # budget exactly. The gap rule exists to protect discrete items # (words, ticker entries); it would be wrong to let a solid image # escape the cap in its name. - end = min(offset + budget, img.width) + # + # With no items to index, the offset here has to stay a column, so + # it is only reusable while the image keeps its width. + shape = ('cols', img.width) + offset = 0 if mode == 'truncate' else self._resume_offset( + plugin_id, shape) + end = self._merge_trailing_runt( + min(offset + budget, img.width), img.width, budget) if mode != 'truncate': - self._item_offsets[plugin_id] = 0 if end >= img.width else end + self._record_offset( + plugin_id, 0 if end >= img.width else end, shape) logger.info( "[%s] Width budget %dpx: cropped continuous %dpx image to " "[%d:%d] (no item gaps of %dpx+ to align to)%s", @@ -528,8 +647,15 @@ class PluginAdapter: # Cut mid-gap so the content either side keeps some breathing room. cuts = sorted({0, img.width} | {(a + b) // 2 for a, b in gaps}) - start = max((c for c in cuts if c <= offset), default=0) - later = [c for c in cuts if c > start] + shape = ('cuts', len(cuts)) + index = 0 if mode == 'truncate' else self._resume_offset( + plugin_id, shape) + # Clamped rather than wrapped: a stale index past the end means the + # strip shrank, and restarting reads better than landing near the end. + start_index = index if 0 <= index < len(cuts) - 1 else 0 + start = cuts[start_index] + + later = cuts[start_index + 1:] if not later: end = img.width else: @@ -537,15 +663,22 @@ class PluginAdapter: # No boundary inside the budget: take the next one and overrun, # because the alternative is cutting through an item. end = max(within) if within else min(later) + end = self._merge_trailing_runt(end, img.width, budget) + # Every candidate for `end` came from `cuts` (which includes img.width), + # so this always resolves; the fallback is defensive only. + end_index = cuts.index(end) if end in cuts else len(cuts) - 1 if mode != 'truncate': - # Next cycle resumes where this one stopped; wrap when the strip ends. - self._item_offsets[plugin_id] = 0 if end >= img.width else end + # Next cycle resumes at the boundary this one stopped on; wrap when + # the strip ends. + self._record_offset( + plugin_id, 0 if end >= img.width else end_index, shape) logger.info( "[%s] Width budget %dpx: cropped single %dpx image to [%d:%d] " - "(%dpx) at item boundaries, %s", + "(%dpx) at item boundaries %d-%d of %d, %s", plugin_id, budget, img.width, start, end, end - start, + start_index, end_index, len(cuts) - 1, "showing the start only (overflow=truncate)" if mode == 'truncate' else "window advances next cycle" ) diff --git a/test/test_vegas_density.py b/test/test_vegas_density.py index 4fc2db12..cd6db2bc 100644 --- a/test/test_vegas_density.py +++ b/test/test_vegas_density.py @@ -1657,3 +1657,228 @@ class TestPerPluginWidthBudget: strip = canvas([(0, 5000)], width=5000) adapter.get_content(NativePlugin([strip]), 'ticker') assert adapter._item_offsets.get('ticker', 0) > 0 + + +def ticker(item_widths, gap=32, height=DISPLAY_H): + """ + A strip of discrete items separated by real gaps, like a news or stocks + ticker. Wide enough gaps that blank_runs() sees item boundaries, which is + what puts _crop_to_budget on its item-aligned path rather than treating the + strip as one continuous block. + """ + width = sum(item_widths) + gap * (len(item_widths) - 1) + spans, x = [], 0 + for w in item_widths: + spans.append((x, x + w)) + x += w + gap + return canvas(spans, width=width, height=height) + + +class TestTrailingRuntWindow: + """ + A rotation's last window used to be whatever happened to be left over. + Measured on a live 512px panel, a 1,840px stocks ticker against a 1,536px + budget split 1,492 + 348 — the second pass showed seven seconds and cut. + """ + + def test_a_barely_oversized_strip_is_shown_whole(self): + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0) + # 1.2 budgets wide: splitting it can only ever produce a fragment. + strip = ticker([180] * 12) # 2160 + 352 gaps = 2512px vs 512 budget + assert strip.width > DISPLAY_W + + adapter = adapter_with(content_padding=0, + max_plugin_width_ratio=strip.width / DISPLAY_W * 0.9) + shown = adapter.get_content(NativePlugin([strip]), 'stocks')[0] + assert shown.width == strip.width, "should absorb the runt, not split" + assert 'stocks' not in adapter._item_offsets + + def test_no_window_in_a_rotation_is_a_fragment(self): + # Walk a long ticker all the way round; every pass must be worth + # showing rather than one of them being a leftover sliver. + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0) + strip = ticker([150] * 40) + plugin = NativePlugin([strip]) + + widths, seen_offsets = [], set() + for _ in range(20): + adapter.invalidate_cache('news') + widths.append(adapter.get_content(plugin, 'news')[0].width) + offset = adapter._item_offsets.get('news', 0) + if offset in seen_offsets: + break + seen_offsets.add(offset) + + assert len(widths) > 1, "a strip this long must take several passes" + # Item snapping means an ordinary window lands short of the budget, so + # the bar is "not a sliver" rather than "a full budget". + assert min(widths) >= DISPLAY_W // 2, ( + "no window should be a fragment, got %r" % widths) + assert max(widths) <= DISPLAY_W * 1.5, ( + "absorbing a runt must stay bounded, got %r" % widths) + + def test_a_continuous_image_also_absorbs_its_runt(self): + # The no-item-gaps path had the same leftover problem. + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0) + solid = canvas([(0, 700)], width=700) # 512 budget -> 512 + 188 runt + first = adapter.get_content(NativePlugin([solid]), 'chart')[0] + assert first.width == 700, "188px tail is not worth its own pass" + assert 'chart' not in adapter._item_offsets + + def test_the_reported_stocks_case(self): + # The exact numbers logged on a 512px panel: an 1,840px stocks ticker + # against a 1,536px budget split 1,492 + 348, so every other appearance + # showed seven seconds of stocks and cut. It should now come through in + # one piece, 20% over budget being the better of the two outcomes. + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=3.0) + # 10 items of 152px with 32px gaps = 1520 + 288 = 1808, near enough. + strip = ticker([152] * 10) + assert DISPLAY_W * 3 < strip.width < DISPLAY_W * 4 + + widths = [] + for _ in range(3): + adapter.invalidate_cache('stocks') + widths.append(adapter.get_content( + NativePlugin([strip]), 'stocks')[0].width) + + assert widths == [strip.width] * 3, ( + "a strip this close to the budget should be shown whole every " + "time, not split into a big pass and a sliver; got %r" % widths) + + def test_a_short_final_row_window_is_not_left_alone(self): + # The multi-row path has the same fault as the single-image one, and + # wrapping does not save it: rows of 450/450/100 against a 512px budget + # gave the 100 a pass of its own, two seconds against nine, because the + # row it wrapped to did not fit either. + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0, + intra_plugin_gap=0, min_content_separation=0) + rows = [canvas([(0, 450)], width=450), + canvas([(0, 450)], width=450), + canvas([(0, 100)], width=100)] + + widths = [] + for _ in range(6): + adapter.invalidate_cache('rows') + shown = adapter.get_content(NativePlugin(list(rows)), 'rows') + widths.append(sum(img.width for img in shown)) + + assert min(widths) >= DISPLAY_W // 2, ( + "a row window should not be a sliver, got %r" % widths) + assert max(widths) <= DISPLAY_W * 1.5, ( + "absorbing a short row must stay bounded, got %r" % widths) + + def test_row_rotation_still_covers_every_row(self): + # Absorbing a short tail must not drop rows from the rotation. + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0, + intra_plugin_gap=0, min_content_separation=0) + rows = [canvas([(0, 450)], width=450), + canvas([(0, 450)], width=450), + canvas([(0, 100)], width=100)] + + seen = set() + for _ in range(8): + adapter.invalidate_cache('rows') + for img in adapter.get_content(NativePlugin(list(rows)), 'rows'): + seen.add(img.width) + assert seen == {450, 100}, "rotation never showed every row: %r" % seen + + def test_a_row_too_wide_to_absorb_still_bounds_the_overrun(self): + # When the next row cannot be taken without blowing past 1.5 budgets, + # a short window is the lesser evil — the same trade the always-show- + # the-first-row rule already makes. + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0, + intra_plugin_gap=0, min_content_separation=0) + rows = [canvas([(0, 900)], width=900), canvas([(0, 100)], width=100)] + for _ in range(4): + adapter.invalidate_cache('wide') + shown = adapter.get_content(NativePlugin(list(rows)), 'wide') + assert sum(i.width for i in shown) <= 900, ( + "must not merge a row that overruns the cap") + + def test_a_genuinely_long_strip_still_gets_capped(self): + # Absorbing runts must not become "never cap anything". + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0) + strip = ticker([150] * 60) + shown = adapter.get_content(NativePlugin([strip]), 'long')[0] + assert shown.width < strip.width + assert shown.width <= DISPLAY_W * 2 + + +class TestOffsetOutlivesItsContent: + """ + A rotation offset only means something against the content it was recorded + against. news re-rendered 9,793px -> 9,505px mid-rotation while its stored + column kept advancing, so the window pointed into unrelated headlines. + """ + + def test_rotation_survives_items_changing_width(self): + # Same items, each a little wider — a price gaining a digit. The window + # should resume at the same *item*, not at a now-meaningless column. + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0) + adapter.get_content(NativePlugin([ticker([150] * 40)]), 'stocks') + first = adapter._item_offsets.get('stocks') + assert first, "the first pass should leave a resume point" + + adapter.invalidate_cache('stocks') + adapter.get_content(NativePlugin([ticker([158] * 40)]), 'stocks') + assert adapter._item_offsets.get('stocks', 0) > first, ( + "same item count means the offset still applies and should advance") + + def test_rotation_restarts_when_the_item_count_changes(self): + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0) + adapter.get_content(NativePlugin([ticker([150] * 40)]), 'news') + assert adapter._item_offsets.get('news', 0) > 0 + + # A fresh headline set with fewer entries: the old position is + # meaningless, so the next pass starts at the top. + adapter.invalidate_cache('news') + shown = adapter.get_content(NativePlugin([ticker([150] * 25)]), 'news')[0] + expected = adapter.get_content( + NativePlugin([ticker([150] * 25)]), 'fresh')[0] + assert shown.width == expected.width + + def test_a_row_index_is_never_read_back_as_a_pixel_column(self): + # The unit collision: _apply_width_budget stores an index into a list + # of rows, _crop_to_budget a column in one image, under the same key. + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0, + intra_plugin_gap=0, min_content_separation=0) + rows = [canvas([(0, 200)], width=200) for _ in range(8)] + adapter.get_content(NativePlugin(rows), 'mixed') + assert adapter._item_offsets.get('mixed', 0) > 0 + assert adapter._offset_shapes['mixed'][0] == 'rows' + + # Now the same plugin returns one wide strip instead. The row index + # must not be read as a column into it: the strip is entered at the + # top, exactly as it would be for a plugin with no history at all. + strip = ticker([150] * 40) + adapter.invalidate_cache('mixed') + carried = adapter.get_content(NativePlugin([strip]), 'mixed')[0] + assert adapter._offset_shapes['mixed'][0] == 'cuts' + + clean = adapter_with(content_padding=0, max_plugin_width_ratio=1.0, + intra_plugin_gap=0, min_content_separation=0) + assert carried.tobytes() == clean.get_content( + NativePlugin([strip]), 'clean')[0].tobytes() + + def test_a_stale_index_past_the_end_restarts(self): + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0) + strip = ticker([150] * 40) + adapter.get_content(NativePlugin([strip]), 'news') + # Force an index far beyond anything the current strip has, keeping the + # shape intact so the guard does not catch it first. + shape = adapter._offset_shapes['news'] + adapter._item_offsets['news'] = 10_000 + adapter.invalidate_cache('news') + shown = adapter.get_content(NativePlugin([strip]), 'news')[0] + assert shown.width > 0 + assert adapter._offset_shapes['news'] == shape + + def test_content_that_fits_clears_both_offset_and_shape(self): + adapter = adapter_with(content_padding=0, max_plugin_width_ratio=1.0) + adapter.get_content(NativePlugin([ticker([150] * 40)]), 'shrink') + assert 'shrink' in adapter._offset_shapes + + adapter.invalidate_cache('shrink') + adapter.get_content(NativePlugin([canvas([(0, 100)], width=100)]), 'shrink') + assert 'shrink' not in adapter._item_offsets + assert 'shrink' not in adapter._offset_shapes From 44f59ede071da0d9f4b5651c61bfd8431bc9764e Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:56:16 -0400 Subject: [PATCH 12/53] fix(web): say what actually went wrong instead of "unknown" (#448) * fix(web): say what actually went wrong instead of "unknown" Every failing endpoint returned "An error occurred; see logs for details" and nothing else. That is survivable until the logs are the thing you cannot reach: a device whose SD card was failing answered the restart action, /system/status and /logs with that same sentence -- the log viewer included, because journalctl could not be executed -- while the exception underneath said [Errno 5] Input/output error: 'systemctl' which names the fault outright. The only endpoint that helped was /health, and only because it happens to pass a subprocess's stderr through. Diagnosis came down to guessing which endpoint leaked something. Add describe_exception(), returning "TypeName: message" on one line, and populate the `details` field that the response schema has always had and nothing ever filled. The type alone carries information -- a bare PermissionError says more than any generic sentence. Exception text is not automatically safe to echo: a requests error quotes the URL it failed on, and plugins that authenticate by query string put their key there. Credential values are redacted while the parameter name is kept, since knowing which credential was involved is part of the diagnosis. Length is capped and newlines collapsed so a parser's context cannot flood a JSON field. Nine handlers in api_v3 bound the exception and never used it, so the promised log entry was never written either -- "see logs for details" was false, not merely unhelpful. Those now log with a traceback and carry the detail. The other 60 already logged and are unchanged; they can adopt the helper as they are touched. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 * fix(web): redact auth headers and URL userinfo, and cover every handler Three review findings. The sanitizer missed two credential shapes that requests puts in its exception text verbatim: `Authorization: Bearer ` and `https://user:password@host`. Both would have gone straight into a response. The auth-scheme name and the username are kept -- they say which credential and whose without being the secret. The AST test only asked whether *something* had been logged, so a `logger.info("failed")` satisfied it while discarding the exception just as completely. It now requires an error-level record carrying exc_info and `describe_exception()` called on the handler's own bound exception. Enforcing that revealed the first cut had scoped itself wrongly. I had converted the nine handlers that logged nothing and left the sixty that logged, reasoning their detail was at least in the journal. But /system/status is one of the sixty, and on the failing device it told me nothing -- the journal was exactly what could not be read. Splitting them left most of the diagnostic surface unhelpful for the case this change exists for, so all sixty-nine now carry the detail. Two handlers had no bound exception name, and three passed the message through a variable rather than a literal; both shapes needed doing by hand. Full suite: 2383 passed, one pre-existing unrelated failure. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 * fix(web): stop reporting client errors as server faults Werkzeug's HTTPExceptions subclass Exception, so the catch-all handler saw them too and turned every 405, 400, 413 and 415 into a 500 UNKNOWN_ERROR. A GET on a POST-only route answered "an error occurred; see logs for details", which tells the caller nothing and blames the wrong side -- found while probing a device whose POST-only config endpoints did exactly that. Hand HTTPExceptions back as themselves, with their own status and description. A genuine server fault still reports as one, with the detail this branch adds. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 * fix(web): redact any auth scheme, and require the detail in the response Two review findings. The auth-header pattern listed Bearer, Basic, Digest and Token, so `Authorization: ApiKey SECRET` or `Negotiate SECRET` went to the client intact. A fixed list silently leaks whatever it does not name, and plugin APIs invent their own schemes, so match any scheme name and keep it while redacting the credential. The AST test accepted a describe_exception(e) call anywhere in the handler, which a handler could satisfy by computing the detail and dropping it before returning the generic message. It now requires the call inside every return expression, which is where it has to be to reach the caller. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 --------- Co-authored-by: Claude Opus 5 (1M context) --- src/web_interface/error_handler.py | 73 +++++++++ test/test_web_error_detail.py | 248 +++++++++++++++++++++++++++++ web_interface/app.py | 36 ++++- web_interface/blueprints/api_v3.py | 159 ++++++++++-------- 4 files changed, 445 insertions(+), 71 deletions(-) create mode 100644 test/test_web_error_detail.py diff --git a/src/web_interface/error_handler.py b/src/web_interface/error_handler.py index 0af53732..ea6423a4 100644 --- a/src/web_interface/error_handler.py +++ b/src/web_interface/error_handler.py @@ -4,6 +4,7 @@ Centralized error handling for web interface. Provides helpers for consistent error responses across API endpoints. """ +import re from typing import Any, Optional from flask import jsonify @@ -16,6 +17,78 @@ from src.logging_config import get_logger logger = get_logger(__name__) +# Credentials that turn up inside exception text. A requests error quotes the +# URL it failed on, and plugins that authenticate by query string put their key +# there, so echoing an exception verbatim can hand out an API key. Redact the +# value, keep the parameter name -- knowing *which* credential was involved is +# part of the diagnosis. +_REDACT_CREDENTIAL = re.compile( + r'((?:api[_-]?key|access[_-]?token|auth|apikey|key|passwd|password|pwd|' + r'secret|sig|signature|token)["\']?\s*[=:]\s*["\']?)([^\s&"\'<>,}]+)', + re.IGNORECASE, +) + +# `Authorization: `. The scheme name is kept because it +# says which kind of credential failed; the credential goes. Any scheme +# matches, not a fixed list: ApiKey, Negotiate, NTLM, AWS4-HMAC-SHA256 and +# whatever a plugin's API invents next are all credentials, and a list would +# silently leak the ones nobody thought of. Not covered by the generic pattern +# above, whose value part stops at whitespace and so would keep the credential +# once a space follows the scheme. +_REDACT_AUTH_HEADER = re.compile( + r'((?:proxy-)?authorization["\']?\s*[=:]\s*["\']?\s*' + r'(?:[A-Za-z][\w.+-]*[ \t]+)?)' # optional scheme name, kept + r'([^\s,"\'<>}]+)', # the credential, redacted + re.IGNORECASE, +) + +# Credentials embedded in a URL: https://user:password@host. requests quotes +# the full URL in its exceptions, so this is a realistic leak. The username is +# kept -- it identifies which account failed without being the secret. +_REDACT_URL_USERINFO = re.compile(r'([a-z][a-z0-9+.-]*://[^/\s:@]+:)([^/\s@]+)(@)', + re.IGNORECASE) + +# Long enough for an errno string with a path, short enough not to dump a +# parser's worth of context into a JSON field. +_MAX_DETAIL_LENGTH = 400 + + +def describe_exception(exc: BaseException, + max_length: int = _MAX_DETAIL_LENGTH) -> str: + """ + One-line, safe-to-return description of an exception. + + The generic "an error occurred; see logs for details" tells a user nothing + and, when the failure is bad enough, the logs are unreachable too: a device + whose storage was failing returned that message from every endpoint + *including* the log viewer, because journalctl could not be executed. The + underlying `[Errno 5] Input/output error` named the fault immediately. + + Returns "TypeName: message", credentials redacted and length capped. The + type alone is worth carrying -- a bare PermissionError says more than any + generic sentence. + + Args: + exc: The exception to describe + max_length: Truncate beyond this many characters + + Returns: + A single-line description, never empty + """ + message = str(exc).strip() + text = f"{type(exc).__name__}: {message}" if message else type(exc).__name__ + # Order matters: the URL and header forms are more specific than the + # generic key=value pattern, which would otherwise chew the scheme. + text = _REDACT_URL_USERINFO.sub(r'\1\3', text) + text = _REDACT_AUTH_HEADER.sub(r'\1', text) + text = _REDACT_CREDENTIAL.sub(r'\1', text) + # Collapse newlines/tabs so the detail stays one line in a JSON field. + text = ' '.join(text.split()) + if len(text) > max_length: + text = text[:max_length - 1].rstrip() + '…' + return text + + def create_error_response( error_code: ErrorCode, message: str, diff --git a/test/test_web_error_detail.py b/test/test_web_error_detail.py new file mode 100644 index 00000000..5362943a --- /dev/null +++ b/test/test_web_error_detail.py @@ -0,0 +1,248 @@ +"""Tests for surfacing the underlying error in web responses. + +Regression under test: every failing endpoint returned "An error occurred; see +logs for details" and nothing else. On a device whose storage was failing that +sentence came back from the restart action, from /system/status, and from +/logs -- the log viewer itself -- because journalctl could not be executed. The +exception underneath said `[Errno 5] Input/output error: 'systemctl'`, which +names the fault outright, and nine handlers were discarding it entirely rather +than even logging it. +""" + +import pytest + +from src.web_interface.error_handler import describe_exception + + +class TestDescribeException: + def test_names_the_type_and_message(self): + detail = describe_exception(OSError(5, "Input/output error", "systemctl")) + assert detail == "OSError: [Errno 5] Input/output error: 'systemctl'" + + def test_the_reported_failure_is_legible(self): + # The whole point: this string is the diagnosis. + assert "Input/output error" in describe_exception( + OSError(5, "Input/output error", "systemctl")) + + def test_a_bare_exception_still_names_its_type(self): + # A PermissionError with no message still says more than "unknown". + assert describe_exception(PermissionError()) == "PermissionError" + assert describe_exception(Exception()) == "Exception" + + def test_message_is_kept_when_present(self): + assert describe_exception(ValueError("bad port")) == "ValueError: bad port" + + +class TestCredentialRedaction: + """Exception text quotes URLs, and plugins authenticate by query string.""" + + @pytest.mark.parametrize("secret_text,leaked", [ + ("failed: https://api.x.com/v1?api_key=SEC123&city=Tampa", "SEC123"), + ("token=abcdef123456 was rejected", "abcdef123456"), + ("connect failed password=hunter2", "hunter2"), + ("GET /?access_token=zzz999", "zzz999"), + ('{"secret": "topsecret"}', "topsecret"), + # requests quotes the URL it failed on, and both of these forms turn + # up in real client exceptions. + ("401 for https://user:hunter2@example.com/api", "hunter2"), + ("headers: {'Authorization': 'Bearer eyJ.SECRET.sig'}", "eyJ.SECRET.sig"), + ("Authorization: Basic dXNlcjpwYXNzd29yZA==", "dXNlcjpwYXNzd29yZA=="), + ("Proxy-Authorization: Bearer ptok999", "ptok999"), + # Any scheme, not a fixed list -- a list silently leaks whatever it + # does not name, and plugin APIs invent their own. + ("Authorization: ApiKey SECRET123", "SECRET123"), + ("Authorization: Negotiate YIIZnegotiateblob", "YIIZnegotiateblob"), + ("Authorization: NTLM TlRMTVNTUAAB", "TlRMTVNTUAAB"), + ("authorization: barecredential", "barecredential"), + ]) + def test_credentials_never_reach_the_response(self, secret_text, leaked): + detail = describe_exception(RuntimeError(secret_text)) + assert leaked not in detail + assert "" in detail + + def test_the_parameter_name_survives_redaction(self): + # Knowing *which* credential was involved is part of the diagnosis. + detail = describe_exception(RuntimeError("https://x/y?api_key=SEC123")) + assert "api_key" in detail + + def test_unknown_schemes_keep_their_name(self): + for scheme in ("ApiKey", "Negotiate", "NTLM", "AWS4-HMAC-SHA256"): + detail = describe_exception( + RuntimeError("Authorization: %s SECRETVALUE" % scheme)) + assert scheme in detail, detail + assert "SECRETVALUE" not in detail, detail + + def test_auth_scheme_and_username_survive(self): + # Which kind of credential, and whose, without the credential itself. + assert "Bearer" in describe_exception( + RuntimeError("Authorization: Bearer eyJ.SECRET.sig")) + assert "user" in describe_exception( + RuntimeError("https://user:hunter2@example.com")) + + def test_non_secret_context_is_preserved(self): + detail = describe_exception(RuntimeError("https://api.x.com/v1?city=Tampa")) + assert "city=Tampa" in detail + assert "" not in detail + + +class TestBounds: + def test_long_messages_are_truncated(self): + detail = describe_exception(ValueError("x" * 5000)) + assert len(detail) <= 400 + + def test_newlines_are_collapsed_to_one_line(self): + detail = describe_exception(ValueError("line one\nline two\tthree")) + assert "\n" not in detail and "\t" not in detail + assert detail == "ValueError: line one line two three" + + def test_custom_length_is_honoured(self): + assert len(describe_exception(ValueError("y" * 500), max_length=50)) <= 50 + + +class TestHandlersCarryDetail: + """The response shape callers actually see.""" + + def test_no_api_v3_handler_discards_its_exception(self): + """Every generic-message handler must log a traceback and return detail. + + Nine of them bound `e` and never used it, so the promised log entry was + never written either. Checking merely that *something* was logged is + too weak -- a `logger.info("failed")` would satisfy it while throwing + the exception away just as completely, so this asserts the two things + that actually make the failure diagnosable: an error-level record with + the traceback, and the sanitized detail in the response. + """ + import ast + + src = open("web_interface/blueprints/api_v3.py").read() + tree = ast.parse(src) + generic = "An error occurred; see logs for details" + + def logs_a_traceback(handler): + """An error/exception-level log call carrying exc_info.""" + for call in [n for n in ast.walk(handler) if isinstance(n, ast.Call)]: + func = call.func + if not isinstance(func, ast.Attribute): + continue + if func.attr == "exception": # implies exc_info + return True + if func.attr not in ("error", "critical"): + continue + if any(kw.arg == "exc_info" and getattr(kw.value, "value", False) is True + for kw in call.keywords): + return True + return False + + def describes_this_exception(node, bound): + """A describe_exception() call anywhere under `node`.""" + for call in [n for n in ast.walk(node) if isinstance(n, ast.Call)]: + if not (isinstance(call.func, ast.Name) + and call.func.id == "describe_exception"): + continue + if bound is None: + return True # bare `except:` cannot name it; accept + if any(isinstance(a, ast.Name) and a.id == bound + for a in call.args): + return True + return False + + def returns_the_detail(handler): + """The detail must be inside what the handler actually returns. + + Looking anywhere in the handler is too weak: a handler could + compute describe_exception(e), drop it on the floor, and return the + generic message with no details field, while still passing. So the + call has to appear within a `return` expression. + """ + returns = [n for n in ast.walk(handler) if isinstance(n, ast.Return)] + if not returns: + return False + return all(describes_this_exception(r, handler.name) for r in returns) + + offenders = [] + for h in [n for n in ast.walk(tree) if isinstance(n, ast.ExceptHandler)]: + seg = ast.get_source_segment(src, h) or "" + if generic not in seg: + continue + missing = [] + if not logs_a_traceback(h): + missing.append("error-level log with exc_info") + if not returns_the_detail(h): + missing.append("describe_exception(e) in the response") + if missing: + offenders.append((h.lineno, missing)) + + assert not offenders, ( + "handlers returning the generic message without %s: %r" + % ("both a traceback log and the detail", offenders)) + + def test_client_errors_keep_their_own_status(self): + """A 405 must not be reported as a server-side UNKNOWN_ERROR. + + Werkzeug's HTTPExceptions subclass Exception, so the catch-all saw them + too: a GET on a POST-only route came back 500 "an error occurred", + which tells the caller nothing and blames the wrong side. Found while + probing a device whose POST-only config endpoints answered every GET + with UNKNOWN_ERROR. + """ + from flask import Flask, jsonify + from werkzeug.exceptions import HTTPException + + app = Flask(__name__) + + @app.errorhandler(Exception) + def handle(error): + if isinstance(error, HTTPException): + return jsonify({ + "status": "error", + "error_code": (error.name or "HTTP_ERROR").upper().replace(" ", "_"), + "message": error.description, + }), error.code or 500 + return jsonify({ + "status": "error", + "error_code": "UNKNOWN_ERROR", + "message": "An error occurred; see logs for details", + "details": describe_exception(error), + }), 500 + + @app.route("/only-post", methods=["POST"]) + def only_post(): + return jsonify({"ok": True}) + + @app.route("/boom") + def boom(): + raise OSError(5, "Input/output error", "systemctl") + + client = app.test_client() + + resp = client.get("/only-post") + assert resp.status_code == 405, "a wrong method must stay a 405" + assert resp.get_json()["error_code"] == "METHOD_NOT_ALLOWED" + + # A genuine server fault still reports as one, with its detail. + resp = client.get("/boom") + assert resp.status_code == 500 + assert "Input/output error" in resp.get_json()["details"] + + def test_global_handler_reports_the_underlying_error(self): + from flask import Flask, jsonify + + app = Flask(__name__) + + @app.errorhandler(Exception) + def handle(error): + return jsonify({ + "status": "error", + "error_code": "UNKNOWN_ERROR", + "message": "An error occurred; see logs for details", + "details": describe_exception(error), + }), 500 + + @app.route("/boom") + def boom(): + raise OSError(5, "Input/output error", "systemctl") + + client = app.test_client() + body = client.get("/boom").get_json() + assert body["error_code"] == "UNKNOWN_ERROR" + assert "Input/output error" in body["details"] diff --git a/web_interface/app.py b/web_interface/app.py index 74ebcfe0..8ed9cb8c 100644 --- a/web_interface/app.py +++ b/web_interface/app.py @@ -16,6 +16,8 @@ from datetime import datetime, timedelta sys.path.insert(0, str(Path(__file__).parent.parent)) from src.config_manager import ConfigManager +from src.web_interface.error_handler import describe_exception +from werkzeug.exceptions import HTTPException from src.exceptions import ConfigError from src.plugin_system.plugin_manager import PluginManager from src.plugin_system.store_manager import PluginStoreManager @@ -391,15 +393,42 @@ def internal_error(error): import logging logger = logging.getLogger('web_interface') logger.error("Internal server error", exc_info=True) - return jsonify({ + payload = { 'status': 'error', 'error_code': 'INTERNAL_ERROR', 'message': 'An internal error occurred; see logs for details', - }), 500 + } + # Flask hands the original exception over as `error.original_exception` + # when propagation is off; without it there is nothing to describe. + original = getattr(error, 'original_exception', None) or ( + error if isinstance(error, BaseException) else None) + if original is not None: + payload['details'] = describe_exception(original) + return jsonify(payload), 500 @app.errorhandler(Exception) def handle_exception(error): - """Handle all unhandled exceptions.""" + """Handle all unhandled exceptions. + + Returning only "see logs for details" is fine until the logs are exactly + what you cannot reach. A device with failing storage answered every + endpoint with that sentence -- including the log viewer, because journalctl + could not be executed -- while the exception underneath said + `[Errno 5] Input/output error`. Naming the error costs nothing here and is + frequently the whole diagnosis, so include it alongside the log pointer. + """ + # Werkzeug's HTTPExceptions subclass Exception, so this catch-all sees + # them too and was reporting every 405, 400, 413 and 415 as a server-side + # UNKNOWN_ERROR 500. A GET on a POST-only route came back as "an error + # occurred" rather than "method not allowed", which tells the caller + # nothing and blames the wrong side. Hand those back as themselves. + if isinstance(error, HTTPException): + return jsonify({ + 'status': 'error', + 'error_code': (error.name or 'HTTP_ERROR').upper().replace(' ', '_'), + 'message': error.description, + }), error.code or 500 + import logging logger = logging.getLogger('web_interface') logger.error("Unhandled exception", exc_info=True) @@ -407,6 +436,7 @@ def handle_exception(error): 'status': 'error', 'error_code': 'UNKNOWN_ERROR', 'message': 'An error occurred; see logs for details', + 'details': describe_exception(error), }), 500 # Captive portal redirect middleware diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 68e01408..6062b85d 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) from src.web_interface.api_helpers import success_response, error_response, validate_request_json from src.web_interface.errors import ErrorCode from src.web_interface.secret_helpers import find_secret_fields, separate_secrets +from src.web_interface.error_handler import describe_exception from src.plugin_system.operation_types import OperationType from src.web_interface.validators import ( validate_file_upload @@ -272,7 +273,7 @@ def get_main_config(): return jsonify({'status': 'success', 'data': config}) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/config/schedule', methods=['GET']) def get_schedule_config(): @@ -290,9 +291,11 @@ def get_schedule_config(): return success_response(data=schedule_config) except Exception as e: + logger.error("%s failed", request.path, exc_info=True) return error_response( ErrorCode.CONFIG_LOAD_FAILED, "An error occurred; see logs for details", + details=describe_exception(e), status_code=500 ) @@ -468,7 +471,7 @@ def save_schedule_config(): ErrorCode.CONFIG_SAVE_FAILED, "An error occurred; see logs for details", - status_code=500 + status_code=500, details=describe_exception(e) ) @api_v3.route('/config/dim-schedule', methods=['GET']) @@ -516,14 +519,14 @@ def get_dim_schedule_config(): return error_response( ErrorCode.CONFIG_LOAD_FAILED, "An error occurred; see logs for details", - status_code=500 + status_code=500, details=describe_exception(e) ) except Exception as e: logging.error(f"[DIM SCHEDULE] Unexpected error loading config: {e}", exc_info=True) return error_response( ErrorCode.CONFIG_LOAD_FAILED, "An error occurred; see logs for details", - status_code=500 + status_code=500, details=describe_exception(e) ) @api_v3.route('/config/dim-schedule', methods=['POST']) @@ -687,7 +690,7 @@ def save_dim_schedule_config(): ErrorCode.CONFIG_SAVE_FAILED, "An error occurred; see logs for details", - status_code=500 + status_code=500, details=describe_exception(e) ) @api_v3.route('/config/main', methods=['POST']) @@ -1314,7 +1317,7 @@ def save_main_config(): return error_response( ErrorCode.CONFIG_SAVE_FAILED, "An error occurred; see logs for details", - status_code=500 + status_code=500, details=describe_exception(e) ) @api_v3.route('/config/secrets', methods=['GET']) @@ -1328,7 +1331,7 @@ def get_secrets_config(): return jsonify({'status': 'success', 'data': config}) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/config/raw/main', methods=['POST']) def save_raw_main_config(): @@ -1361,6 +1364,7 @@ def save_raw_main_config(): return error_response( ErrorCode.CONFIG_SAVE_FAILED, error_message, + details=describe_exception(e), context={'config_path': e.config_path} if hasattr(e, 'config_path') and e.config_path else None, status_code=500 @@ -1370,6 +1374,7 @@ def save_raw_main_config(): return error_response( ErrorCode.UNKNOWN_ERROR, error_message, + details=describe_exception(e), status_code=500 ) @@ -1409,7 +1414,8 @@ def save_raw_secrets_config(): else: error_message = 'An error occurred; see logs for details' - return jsonify({'status': 'error', 'message': error_message}), 500 + return jsonify({'status': 'error', 'message': error_message, + 'details': describe_exception(e)}), 500 @api_v3.route('/system/status', methods=['GET']) def get_system_status(): @@ -1497,7 +1503,7 @@ def get_system_status(): return jsonify({'status': 'success', 'data': status}) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/health', methods=['GET']) def get_health(): @@ -1596,9 +1602,11 @@ def get_health(): return jsonify({'status': 'success', 'data': health_status}) except Exception as e: + logger.error("%s failed", request.path, exc_info=True) return jsonify({ 'status': 'error', 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e), 'data': {'status': 'unhealthy'} }), 500 @@ -2368,7 +2376,7 @@ def get_display_current(): return jsonify({'status': 'success', 'data': display_data}) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/display/on-demand/status', methods=['GET']) def get_on_demand_status(): @@ -2392,7 +2400,7 @@ def get_on_demand_status(): }) except Exception as exc: logger.error('Error in get_on_demand_status', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 @api_v3.route('/display/on-demand/start', methods=['POST']) def start_on_demand_display(): @@ -2495,7 +2503,7 @@ def start_on_demand_display(): return jsonify({'status': 'success', 'data': response_data}) except Exception as exc: logger.error('Error in start_on_demand_display', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 @api_v3.route('/display/on-demand/stop', methods=['POST']) def stop_on_demand_display(): @@ -2531,7 +2539,7 @@ def stop_on_demand_display(): }) except Exception as exc: logger.error('Error in stop_on_demand_display', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 @api_v3.route('/plugins/installed', methods=['GET']) def get_installed_plugins(): @@ -2679,7 +2687,7 @@ def get_installed_plugins(): return jsonify({'status': 'success', 'data': {'plugins': plugins}}) except Exception as e: logger.error('Error in get_installed_plugins', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 def _installed_plugin_ids(): """Best-effort list of installed plugin IDs for the web process. @@ -2745,7 +2753,7 @@ def get_plugin_health(): }) except Exception as e: logger.error('Error in get_plugin_health', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/health/', methods=['GET']) def get_plugin_health_single(plugin_id): @@ -2770,7 +2778,7 @@ def get_plugin_health_single(plugin_id): }) except Exception as e: logger.error('Error in get_plugin_health_single', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/health//reset', methods=['POST']) def reset_plugin_health(plugin_id): @@ -2795,7 +2803,7 @@ def reset_plugin_health(plugin_id): }) except Exception as e: logger.error('Error in reset_plugin_health', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/metrics', methods=['GET']) def get_plugin_metrics(): @@ -2835,7 +2843,7 @@ def get_plugin_metrics(): }) except Exception as e: logger.error('Error in get_plugin_metrics', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/metrics/', methods=['GET']) def get_plugin_metrics_single(plugin_id): @@ -2860,7 +2868,7 @@ def get_plugin_metrics_single(plugin_id): }) except Exception as e: logger.error('Error in get_plugin_metrics_single', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/metrics//reset', methods=['POST']) def reset_plugin_metrics(plugin_id): @@ -2885,7 +2893,7 @@ def reset_plugin_metrics(plugin_id): }) except Exception as e: logger.error('Error in reset_plugin_metrics', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/limits/', methods=['GET', 'POST']) def manage_plugin_limits(plugin_id): @@ -2940,7 +2948,7 @@ def manage_plugin_limits(plugin_id): }) except Exception as e: logger.error('Error in manage_plugin_limits', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/toggle', methods=['POST']) def toggle_plugin(): @@ -3949,7 +3957,7 @@ def install_plugin(): except Exception as e: logger.error('Error in install_plugin', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/install-from-url', methods=['POST']) def install_plugin_from_url(): @@ -4004,7 +4012,7 @@ def install_plugin_from_url(): except Exception as e: logger.error('Error in install_plugin_from_url', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/registry-from-url', methods=['POST']) def get_registry_from_url(): @@ -4036,7 +4044,7 @@ def get_registry_from_url(): except Exception as e: logger.error('Error in get_registry_from_url', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/saved-repositories', methods=['GET']) def get_saved_repositories(): @@ -4049,7 +4057,7 @@ def get_saved_repositories(): return jsonify({'status': 'success', 'data': {'repositories': repositories}}) except Exception as e: logger.error('Error in get_saved_repositories', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/saved-repositories', methods=['POST']) def add_saved_repository(): @@ -4080,7 +4088,7 @@ def add_saved_repository(): }), 400 except Exception as e: logger.error('Error in add_saved_repository', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/saved-repositories', methods=['DELETE']) def remove_saved_repository(): @@ -4110,7 +4118,7 @@ def remove_saved_repository(): }), 404 except Exception as e: logger.error('Error in remove_saved_repository', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/store/list', methods=['GET']) def list_plugin_store(): @@ -4163,7 +4171,7 @@ def list_plugin_store(): return jsonify({'status': 'success', 'data': {'plugins': formatted_plugins}}) except Exception as e: logger.error('Error in list_plugin_store', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/store/github-status', methods=['GET']) def get_github_auth_status(): @@ -4214,7 +4222,7 @@ def get_github_auth_status(): }) except Exception as e: logger.error('Error in get_github_auth_status', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/store/refresh', methods=['POST']) def refresh_plugin_store(): @@ -4241,7 +4249,7 @@ def refresh_plugin_store(): }) except Exception as e: logger.error('Error in refresh_plugin_store', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 def deep_merge(base_dict, update_dict): """ @@ -5763,7 +5771,7 @@ def get_plugin_schema(): return jsonify({'status': 'success', 'data': {'schema': default_schema}}) except Exception as e: logger.error('Error in get_plugin_schema', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/skins', methods=['GET']) def list_skins(): @@ -5798,9 +5806,9 @@ def list_skins(): 'has_preview': bool(preview and (skin_dir / preview).is_file()), }) return jsonify({'status': 'success', 'data': {'skins': payload}}) - except Exception: + except Exception as e: logger.error('Error in list_skins', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/config/reset', methods=['POST']) def reset_plugin_config(): @@ -5880,7 +5888,7 @@ def reset_plugin_config(): }) except Exception as e: logger.error('Error in reset_plugin_config', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/action', methods=['POST']) def execute_plugin_action(): @@ -6140,7 +6148,7 @@ sys.exit(proc.returncode) logger.error("Error executing action step 1", exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) }), 500 else: # Simple script execution @@ -6190,7 +6198,7 @@ sys.exit(proc.returncode) return jsonify({'status': 'error', 'message': 'Action timed out'}), 408 except Exception as e: logger.error('Error in execute_plugin_action', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/authenticate/spotify', methods=['POST']) def authenticate_spotify(): @@ -6323,12 +6331,12 @@ sys.exit(proc.returncode) logger.error("Error getting Spotify auth URL", exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) }), 500 except Exception as e: logger.error('Error in authenticate_spotify', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/authenticate/ytm', methods=['POST']) def authenticate_ytm(): @@ -6378,7 +6386,7 @@ def authenticate_ytm(): return jsonify({'status': 'error', 'message': 'Authentication timed out'}), 408 except Exception as e: logger.error('Error in authenticate_ytm', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/fonts/catalog', methods=['GET']) def get_fonts_catalog(): @@ -6473,7 +6481,10 @@ def get_fonts_catalog(): return jsonify({'status': 'success', 'data': {'catalog': catalog}}) except Exception as e: - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + logger.error("%s failed", request.path, exc_info=True) + return jsonify({'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e)}), 500 @api_v3.route('/fonts/tokens', methods=['GET']) def get_font_tokens(): @@ -6492,7 +6503,7 @@ def get_font_tokens(): return jsonify({'status': 'success', 'data': {'tokens': tokens}}) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/fonts/overrides', methods=['GET']) def get_fonts_overrides(): @@ -6504,7 +6515,7 @@ def get_fonts_overrides(): return jsonify({'status': 'success', 'data': {'overrides': overrides}}) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/fonts/overrides', methods=['POST']) def save_fonts_overrides(): @@ -6518,7 +6529,7 @@ def save_fonts_overrides(): return jsonify({'status': 'success', 'message': 'Font overrides saved'}) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/fonts/overrides/', methods=['DELETE']) def delete_font_override(element_key): @@ -6528,7 +6539,7 @@ def delete_font_override(element_key): return jsonify({'status': 'success', 'message': f'Font override for {element_key} deleted'}) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/fonts/upload', methods=['POST']) def upload_font(): @@ -6593,7 +6604,7 @@ def upload_font(): }) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/fonts/preview', methods=['GET']) @@ -6738,7 +6749,7 @@ def get_font_preview() -> tuple[Response, int] | Response: }) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/fonts/', methods=['DELETE']) @@ -6826,7 +6837,7 @@ def delete_font(font_family: str) -> tuple[Response, int] | Response: }) except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/assets/upload', methods=['POST']) @@ -6974,7 +6985,7 @@ def upload_plugin_asset(): except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/of-the-day/json/upload', methods=['POST']) def upload_of_the_day_json(): @@ -7124,7 +7135,7 @@ def upload_of_the_day_json(): except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/of-the-day/json/delete', methods=['POST']) def delete_of_the_day_json(): @@ -7171,7 +7182,7 @@ def delete_of_the_day_json(): except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins//static/', methods=['GET']) def serve_plugin_static(plugin_id, file_path): @@ -7217,7 +7228,7 @@ def serve_plugin_static(plugin_id, file_path): except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/calendar/upload-credentials', methods=['POST']) @@ -7299,7 +7310,7 @@ def upload_calendar_credentials(): except Exception as e: logger.error('Error in upload_calendar_credentials', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/assets/delete', methods=['POST']) def delete_plugin_asset(): @@ -7342,7 +7353,7 @@ def delete_plugin_asset(): except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins/assets/list', methods=['GET']) def list_plugin_assets(): @@ -7370,7 +7381,7 @@ def list_plugin_assets(): except Exception as e: logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/display/current-status', methods=['GET']) def get_current_display_status(): @@ -7391,9 +7402,9 @@ def get_current_display_status(): 'last_updated': None, } return jsonify({'status': 'success', 'data': state}) - except Exception: + except Exception as e: logger.error('Error in get_current_display_status', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/logs', methods=['GET']) def get_logs(): @@ -7432,9 +7443,11 @@ def get_logs(): 'message': 'Timeout while fetching logs' }), 500 except Exception as e: + logger.error("%s failed", request.path, exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) }), 500 # Multi-Display Sync Endpoints @@ -7499,9 +7512,11 @@ def get_wifi_status(): } }) except Exception as e: + logger.error("%s failed", request.path, exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) }), 500 @api_v3.route('/wifi/scan', methods=['GET']) @@ -7623,7 +7638,7 @@ def connect_wifi(): logger.error("Error connecting to WiFi", exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) }), 500 @api_v3.route('/wifi/disconnect', methods=['POST']) @@ -7649,7 +7664,7 @@ def disconnect_wifi(): logger.error("Error disconnecting from WiFi", exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) }), 500 @api_v3.route('/wifi/ap/enable', methods=['POST']) @@ -7674,9 +7689,11 @@ def enable_ap_mode(): 'message': message }), 400 except Exception as e: + logger.error("%s failed", request.path, exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) }), 500 @api_v3.route('/wifi/ap/disable', methods=['POST']) @@ -7699,9 +7716,11 @@ def disable_ap_mode(): 'message': message }), 400 except Exception as e: + logger.error("%s failed", request.path, exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) }), 500 @api_v3.route('/wifi/ap/auto-enable', methods=['GET']) @@ -7720,9 +7739,11 @@ def get_auto_enable_ap_mode(): } }) except Exception as e: + logger.error("%s failed", request.path, exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) }), 500 @api_v3.route('/wifi/ap/auto-enable', methods=['POST']) @@ -7752,9 +7773,11 @@ def set_auto_enable_ap_mode(): } }) except Exception as e: + logger.error("%s failed", request.path, exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) }), 500 @api_v3.route('/wifi/radio', methods=['GET']) @@ -7774,7 +7797,7 @@ def get_wifi_radio(): logger.error("Error getting WiFi radio state", exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) }), 500 @api_v3.route('/wifi/radio', methods=['POST']) @@ -7822,7 +7845,7 @@ def set_wifi_radio(): logger.error("Error setting WiFi radio state", exc_info=True) return jsonify({ 'status': 'error', - 'message': 'An error occurred; see logs for details' + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) }), 500 @api_v3.route('/cache/list', methods=['GET']) @@ -7847,7 +7870,7 @@ def list_cache_files(): }) except Exception as e: logger.error('Error in list_cache_files', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/cache/delete', methods=['POST']) def delete_cache_file(): @@ -7873,7 +7896,7 @@ def delete_cache_file(): }) except Exception as e: logger.error('Error in delete_cache_file', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 # ============================================================================= From 8159afca431f14b8dee32771e813698461dc62a6 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:56:53 -0400 Subject: [PATCH 13/53] fix(odds): stop a stalled ESPN taking the whole plugin update with it (#449) Odds are fetched per live game from inside SportsLive.update(), with show_odds defaulting on, and the plugin executor kills an operation at 30s. The odds request timeout was also 30s, so a single stalled request consumed the entire budget and the update carrying every game's score was killed. Out of season that is invisible: preseason week 1 returns one game. A Sunday slate is around sixteen, so the odds of at least one slow request rise sharply just as the cost of losing the update does. Shorten the request timeout to 5s, and after a network failure skip the network for 60s. The timeout alone is not enough -- sixteen consecutive 5s timeouts still blow through -- and when ESPN is unreachable it is unreachable for the whole slate, so the first failure already answers the question for the rest of the pass. before: one stalled request = 30s = the entire budget after : 5s, the rest of the slate skipped, retry after 60s The stale-cache fallback is unchanged: the cache is consulted before any of this, and the failing request still falls back to it. An earlier version of this branch also jittered the cache TTL to stagger expiry across a slate. That has been dropped: CacheManager.set() stores ttl for compatibility but the read path expires entries by a per-type max_age (1800s for odds), so the jitter was inert. Making the read path honour a per-entry ttl is a real fix but changes a contract 48 plugin call sites already rely on, which is not a change to make two weeks before the season. Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 Co-authored-by: Claude Opus 5 (1M context) --- src/base_odds_manager.py | 39 +++++++++- test/test_base_odds_manager.py | 8 +- test/test_odds_request_budget.py | 122 +++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 test/test_odds_request_budget.py diff --git a/src/base_odds_manager.py b/src/base_odds_manager.py index f35c4d54..50b6fe63 100644 --- a/src/base_odds_manager.py +++ b/src/base_odds_manager.py @@ -12,6 +12,8 @@ Follows LEDMatrix configuration management patterns: """ import logging +import time + import requests import json from typing import Dict, Any, Optional, List @@ -45,7 +47,14 @@ class BaseOddsManager: # Configuration with defaults self.update_interval = 3600 # 1 hour default - self.request_timeout = 30 # 30 seconds default + # Well under the plugin executor's 30s operation budget. At 30s a + # single stalled ESPN request consumed the entire budget and the whole + # update() was killed -- and odds are fetched per live game, inside the + # live update loop, with show_odds defaulting on. Losing one game's + # odds beats losing the update that carries every game's score. + self.request_timeout = 5 + # Set when a request fails; until then, skip the network entirely. + self._skip_network_until = 0.0 self.cache_ttl = 1800 # 30 minutes default # Load configuration if available @@ -73,6 +82,14 @@ class BaseOddsManager: except Exception as e: self.logger.warning(f"Failed to load BaseOddsManager configuration: {e}") + # After a network failure, stop trying for this long and serve cache only. + # A short per-request timeout bounds one stall, but a full Sunday slate is + # ~16 games fetched in a loop, so 16 consecutive timeouts still blow the + # budget. When ESPN is unreachable it is unreachable for all of them, so + # the first failure is enough to know: skip the rest of this pass and try + # again shortly. + _FAILURE_COOLDOWN = 60.0 + def get_odds(self, sport: str | None, league: str | None, event_id: str, update_interval_seconds: int = None) -> Optional[Dict[str, Any]]: """ @@ -101,8 +118,18 @@ class BaseOddsManager: self.logger.info(f"Using cached odds from ESPN for {cache_key}") return cached_data + if time.monotonic() < self._skip_network_until: + # A recent request failed, so ESPN is very likely still unreachable. + # Returning now keeps the caller's update inside its time budget + # instead of paying the timeout again for every remaining game. + self.logger.debug( + "Skipping odds fetch for %s: a recent request failed, holding off " + "for another %.0fs", cache_key, + self._skip_network_until - time.monotonic()) + return None + self.logger.info(f"Cache miss - fetching fresh odds from ESPN for {cache_key}") - + try: # Map league names to ESPN API format league_mapping = { @@ -121,6 +148,8 @@ class BaseOddsManager: response.raise_for_status() raw_data = response.json() + self._skip_network_until = 0.0 # reachable again + self.logger.debug(f"Received raw odds data from ESPN: {json.dumps(raw_data, indent=2)}") odds_data = self._extract_espn_data(raw_data) @@ -140,7 +169,11 @@ class BaseOddsManager: return odds_data except requests.exceptions.RequestException as e: - self.logger.error(f"Error fetching odds from ESPN API for {cache_key}: {e}") + self._skip_network_until = time.monotonic() + self._FAILURE_COOLDOWN + self.logger.error( + "Error fetching odds from ESPN API for %s: %s. Holding off on odds " + "for %.0fs so a slate of games does not pay this timeout each.", + cache_key, e, self._FAILURE_COOLDOWN) except json.JSONDecodeError: self.logger.error(f"Error decoding JSON response from ESPN API for {cache_key}.") diff --git a/test/test_base_odds_manager.py b/test/test_base_odds_manager.py index b08183ad..3eb591e3 100644 --- a/test/test_base_odds_manager.py +++ b/test/test_base_odds_manager.py @@ -87,7 +87,11 @@ class TestGetOdds: assert '/events/401/competitions/401/odds' in url assert url == ('https://sports.core.api.espn.com/v2/sports/football/' 'leagues/nfl/events/401/competitions/401/odds') - assert mock_get.call_args.kwargs['timeout'] == 30 + # The number matters less than the property: a single stalled request + # must not be able to consume the plugin executor's 30s operation + # budget, since odds are fetched per live game inside update(). + assert mock_get.call_args.kwargs['timeout'] == 5 + assert mock_get.call_args.kwargs['timeout'] < 30 def test_ncaa_fb_maps_to_college_football(self, manager, mock_get): manager.get_odds('football', 'ncaa_fb', '401') @@ -355,5 +359,5 @@ class TestLoadConfiguration: manager = BaseOddsManager(cache_manager, config_manager=config_manager) assert manager.update_interval == 3600 - assert manager.request_timeout == 30 + assert manager.request_timeout == 5 assert manager.cache_ttl == 1800 diff --git a/test/test_odds_request_budget.py b/test/test_odds_request_budget.py new file mode 100644 index 00000000..233780fb --- /dev/null +++ b/test/test_odds_request_budget.py @@ -0,0 +1,122 @@ +"""Tests that a slow ESPN cannot take a whole plugin update with it. + +Odds are fetched per live game from inside SportsLive.update(), with show_odds +defaulting on, and the plugin executor kills an operation at 30s. The odds +request timeout was also 30s, so one stalled request consumed the entire budget +and the update carrying every game's score was killed: + + 00:43:43 ERROR plugin football-scoreboard operation timed out after 30.0s + 01:43:43 ERROR plugin football-scoreboard operation timed out after 30.0s + +Invisible out of season -- preseason week 1 returns a single game -- and a +Sunday slate is around sixteen. +""" + +from unittest.mock import Mock + +from src.base_odds_manager import BaseOddsManager + +PLUGIN_BUDGET = 30.0 # PluginExecutor(default_timeout=30.0) + + +def _manager(cache=None): + cache = cache or Mock() + cache.get_with_auto_strategy.return_value = None + return BaseOddsManager(cache_manager=cache, config_manager=None) + + +class TestRequestTimeout: + def test_leaves_room_in_the_operation_budget(self): + assert _manager().request_timeout < PLUGIN_BUDGET / 2 + + def test_the_timeout_is_the_one_actually_used(self): + m = _manager() + import src.base_odds_manager as mod + real = mod.requests.get + try: + mod.requests.get = Mock(side_effect=mod.requests.exceptions.Timeout("x")) + m.get_odds("football", "nfl", "401") + assert mod.requests.get.call_args.kwargs["timeout"] == m.request_timeout + finally: + mod.requests.get = real + + +class TestSlowEspnCannotKillTheUpdate: + def test_one_failure_stops_the_rest_of_the_slate_hitting_the_network(self): + m = _manager() + import src.base_odds_manager as mod + real = mod.requests.get + calls = {"n": 0} + + def timeout(*a, **k): + calls["n"] += 1 + raise mod.requests.exceptions.Timeout("timed out") + + try: + mod.requests.get = timeout + for i in range(16): # a full slate, one game at a time + m.get_odds("football", "nfl", "4018730%02d" % i) + finally: + mod.requests.get = real + + assert calls["n"] == 1, ( + "%d games each paid the timeout; the breaker should have stopped " + "after the first" % calls["n"]) + + def test_worst_case_slate_stays_inside_the_budget(self): + m = _manager() + assert m.request_timeout * 1 < PLUGIN_BUDGET + + def test_recovery_is_automatic(self): + m = _manager() + import src.base_odds_manager as mod + real_get, real_monotonic = mod.requests.get, mod.time.monotonic + clock = {"t": 1000.0} + try: + mod.time.monotonic = lambda: clock["t"] + mod.requests.get = Mock( + side_effect=mod.requests.exceptions.Timeout("timed out")) + m.get_odds("football", "nfl", "401") + assert m._skip_network_until > clock["t"], "breaker did not open" + + clock["t"] += 1 + before = mod.requests.get.call_count + m.get_odds("football", "nfl", "402") + assert mod.requests.get.call_count == before, "should not have retried" + + clock["t"] += m._FAILURE_COOLDOWN + m.get_odds("football", "nfl", "403") + assert mod.requests.get.call_count > before, "never retried" + finally: + mod.requests.get, mod.time.monotonic = real_get, real_monotonic + + def test_a_healthy_fetch_clears_the_breaker(self): + m = _manager() + m._skip_network_until = 0.0 + m._extract_espn_data = Mock(return_value=None) + import src.base_odds_manager as mod + real = mod.requests.get + try: + resp = Mock() + resp.json.return_value = {} + resp.raise_for_status.return_value = None + mod.requests.get = Mock(return_value=resp) + m.get_odds("football", "nfl", "401") + finally: + mod.requests.get = real + assert m._skip_network_until == 0.0 + + def test_the_stale_cache_fallback_still_works(self): + # The failing request must still hand back whatever was cached; only + # the *subsequent* games skip the network. + cache = Mock() + cache.get_with_auto_strategy.side_effect = [None, {"details": "stale"}] + m = BaseOddsManager(cache_manager=cache, config_manager=None) + import src.base_odds_manager as mod + real = mod.requests.get + try: + mod.requests.get = Mock( + side_effect=mod.requests.exceptions.Timeout("timed out")) + assert m.get_odds("football", "nfl", "401") == {"details": "stale"} + finally: + mod.requests.get = real From bb1a1671ec752e5be7617521dbbade729eb240b8 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:14:25 -0400 Subject: [PATCH 14/53] fix(cache): make the ttl parameter actually control expiry (#450) CacheManager.set(key, data, ttl=...) stored the number and no read path ever consulted it. Expiry came from a max_age inferred from substrings in the key -- "live", "odds", "stock" -- so all 52 callers passing a ttl were writing a value that did nothing. The docstring said so outright: "stored for compatibility but expiration is still controlled via max_age when reading". It is easier to read that as a note than as a defect, which is presumably how it survived. Both cache layers already hold the record when they decide, so each now prefers an explicit ttl and falls back to max_age when there is none. The caller that wrote the record knows what its data is; a substring guess is a reasonable default for records that never said, and a poor override for records that did. Measured against a device's real cache of 8,875 entries carrying a ttl, the inferred and intended values disagreed nearly everywhere: stocks max_age 600 vs ttl 1800 4903 entries news max_age 3600 vs ttl 600 1770 entries odds max_age 1800 vs ttl 3600 1301 entries images max_age 300 vs ttl 2592000 20 entries In every case the ttl matches what the plugin plainly intended: stock quotes cached for half an hour rather than ten minutes, headlines refreshed every ten minutes rather than hourly, bird photographs that never change kept for a month rather than five minutes. Two things make this safe to land now. No sports_live entry carries a ttl at all -- the live-score path does not use set(ttl=) -- so live freshness is untouched, which matters with a season two weeks out. And replaying the change against that real cache, 997 currently-expired entries become live while not one live entry becomes expired, so there is no invalidation spike on deploy. Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 Co-authored-by: Claude Opus 5 (1M context) --- src/cache/disk_cache.py | 16 ++++ src/cache/memory_cache.py | 10 +++ src/cache_manager.py | 6 +- test/test_cache_ttl_honoured.py | 130 ++++++++++++++++++++++++++++++++ 4 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 test/test_cache_ttl_honoured.py diff --git a/src/cache/disk_cache.py b/src/cache/disk_cache.py index 64b72f07..e50b3a85 100644 --- a/src/cache/disk_cache.py +++ b/src/cache/disk_cache.py @@ -112,6 +112,22 @@ class DiskCache: record_ts = None now = time.time() + + # An explicit per-entry ttl wins over the caller's max_age. The + # caller that wrote the record knows what its data is; max_age is + # inferred from substrings in the key ("live", "odds", "stock") and + # is only a fallback for records that never said. Until now the ttl + # was stored and ignored, so `set(key, data, ttl=...)` did nothing + # at all -- 48 plugin call sites and 4 in the core were writing a + # number no read path consulted. + effective_max_age = max_age + if isinstance(record, dict): + stored_ttl = record.get('ttl') + if isinstance(stored_ttl, (int, float)) and not isinstance(stored_ttl, bool) \ + and stored_ttl >= 0: + effective_max_age = stored_ttl + max_age = effective_max_age + # max_age=None means "never expires" (mirrors MemoryCache and the # cache_manager docstring). Guard it explicitly — otherwise the # comparison below raises TypeError and the record is treated as a diff --git a/src/cache/memory_cache.py b/src/cache/memory_cache.py index 33c9963b..a13e3ccf 100644 --- a/src/cache/memory_cache.py +++ b/src/cache/memory_cache.py @@ -57,6 +57,16 @@ class MemoryCache: if timestamp is None: return None + # An explicit per-entry ttl wins over the caller's max_age, matching + # DiskCache. max_age is inferred from substrings in the key and is + # only a fallback for records that did not say what they wanted. + record = self._cache[key] + if isinstance(record, dict): + stored_ttl = record.get('ttl') + if isinstance(stored_ttl, (int, float)) and not isinstance(stored_ttl, bool) \ + and stored_ttl >= 0: + max_age = stored_ttl + # Check expiration if max_age is not None and (now - timestamp) > max_age: # Expired - remove it diff --git a/src/cache_manager.py b/src/cache_manager.py index c76c530d..3eaab7fa 100644 --- a/src/cache_manager.py +++ b/src/cache_manager.py @@ -594,8 +594,10 @@ class CacheManager: Args: key: Cache key data: Data to cache - ttl: Optional time-to-live in seconds (stored for compatibility but - expiration is still controlled via max_age when reading) + ttl: Time-to-live in seconds for this entry. Takes precedence over + the max_age a reader would otherwise apply, which is inferred + from the key and is only a fallback for entries that did not + say. Omit it to keep that inferred behaviour. """ cache_data = { 'data': data, diff --git a/test/test_cache_ttl_honoured.py b/test/test_cache_ttl_honoured.py new file mode 100644 index 00000000..080cbe2e --- /dev/null +++ b/test/test_cache_ttl_honoured.py @@ -0,0 +1,130 @@ +"""Tests that a per-entry ttl actually controls expiry. + +Regression under test: `CacheManager.set(key, data, ttl=...)` stored the value +and no read path ever consulted it. Expiry came from a `max_age` inferred from +substrings in the key ("live", "odds", "stock"), so every caller passing `ttl=` +-- 48 sites across the plugins and 4 in the core -- was writing a number that +did nothing. The old docstring admitted as much: "stored for compatibility but +expiration is still controlled via max_age when reading". + +Measured against a real device's cache (8,873 entries carrying a ttl), the +inferred value and the intended one disagreed almost everywhere: + + stocks max_age 600 vs ttl 1800 4903 entries + news max_age 3600 vs ttl 600 1770 entries + odds max_age 1800 vs ttl 3600 1301 entries + images max_age 300 vs ttl 2592000 20 entries + +No `sports_live` entry carries a ttl, so live scores keep their inferred +30-second freshness either way. +""" + +import time + +import pytest + +from src.cache.memory_cache import MemoryCache +from src.cache.disk_cache import DiskCache + + +@pytest.fixture +def disk(tmp_path): + return DiskCache(cache_dir=str(tmp_path)) + + +def _record(ttl=None, age=0.0): + rec = {"data": {"v": 1}, "timestamp": time.time() - age} + if ttl is not None: + rec["ttl"] = ttl + return rec + + +class TestDiskCacheHonoursTtl: + def test_ttl_longer_than_max_age_keeps_the_entry(self, disk): + # The odds case: written wanting an hour, expired at 30 minutes. + disk.set("odds_espn_football_nfl_401", _record(ttl=3600, age=1900)) + assert disk.get("odds_espn_football_nfl_401", max_age=1800) is not None + + def test_ttl_shorter_than_max_age_expires_the_entry(self, disk): + # The news case: written wanting 10 minutes, kept for an hour. + disk.set("news_NHL_1", _record(ttl=600, age=900)) + assert disk.get("news_NHL_1", max_age=3600) is None + + def test_without_a_ttl_max_age_still_applies(self, disk): + disk.set("plain_key", _record(age=400)) + assert disk.get("plain_key", max_age=300) is None + disk.set("plain_key2", _record(age=100)) + assert disk.get("plain_key2", max_age=300) is not None + + def test_a_fresh_entry_within_its_ttl_survives(self, disk): + disk.set("k", _record(ttl=600, age=10)) + assert disk.get("k", max_age=30) is not None + + def test_ttl_zero_expires_immediately(self, disk): + # 0 means zero seconds, not "forever" -- max_age=None is how a caller + # asks for no expiry. + disk.set("k", _record(ttl=0, age=1)) + assert disk.get("k", max_age=99999) is None + + @pytest.mark.parametrize("bad", ["600", None, True, False, -5, {"a": 1}]) + def test_a_nonsense_ttl_falls_back_to_max_age(self, disk, bad): + # Including bools: True is an int in Python and must not become a 1s ttl. + rec = _record(age=400) + rec["ttl"] = bad + disk.set("k_%s" % type(bad).__name__, rec) + assert disk.get("k_%s" % type(bad).__name__, max_age=300) is None + + +class TestMemoryCacheHonoursTtl: + def test_ttl_longer_than_max_age_keeps_the_entry(self): + m = MemoryCache() + m.set("k", _record(ttl=3600)) + m._timestamps["k"] = time.time() - 1900 + assert m.get("k", max_age=1800) is not None + + def test_ttl_shorter_than_max_age_expires_the_entry(self): + m = MemoryCache() + m.set("k", _record(ttl=600)) + m._timestamps["k"] = time.time() - 900 + assert m.get("k", max_age=3600) is None + + def test_without_a_ttl_max_age_still_applies(self): + m = MemoryCache() + m.set("k", _record()) + m._timestamps["k"] = time.time() - 400 + assert m.get("k", max_age=300) is None + + def test_both_layers_agree(self, tmp_path): + """A record must not be live in one layer and expired in the other.""" + rec = _record(ttl=3600, age=1900) + d = DiskCache(cache_dir=str(tmp_path)) + d.set("k", rec) + m = MemoryCache() + m.set("k", rec) + m._timestamps["k"] = rec["timestamp"] + assert (d.get("k", max_age=1800) is not None) == (m.get("k", max_age=1800) is not None) + + +class TestEndToEnd: + def test_set_then_get_respects_the_ttl(self, tmp_path, monkeypatch): + """The behaviour a caller of CacheManager.set(ttl=...) expects.""" + from src.cache_manager import CacheManager + + cm = CacheManager() + cm._disk_cache_component = DiskCache(cache_dir=str(tmp_path)) + cm._memory_cache_component = MemoryCache() + + cm.set("odds_espn_football_nfl_401", {"spread": 6.5}, ttl=3600) + + # Age the stored record past the inferred max_age for odds (1800s) but + # within the ttl the caller asked for. + path = cm._disk_cache_component.get_cache_path("odds_espn_football_nfl_401") + import json + rec = json.load(open(path)) + rec["timestamp"] = time.time() - 1900 + json.dump(rec, open(path, "w")) + cm._memory_cache_component.clear() if hasattr( + cm._memory_cache_component, "clear") else None + + got = cm.get_with_auto_strategy("odds_espn_football_nfl_401") + assert got is not None, "the ttl the caller asked for was ignored" From 2add759f40024bfd209d3b9a767a31a01bdb2bd2 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:10:56 -0400 Subject: [PATCH 15/53] fix(odds): identify the odds requests to ESPN (#451) The odds fetch used a bare requests.get, so it went out as python-requests/x.y -- the one agent ESPN is known to reject. Around 2026-08-04 it began 403ing browser strings and bare custom tokens alike; what it accepts is a token carrying a URL that says who is calling. Every other ESPN caller in the tree already sends that header (src/common/api_helper.py, src/base_classes/data_sources.py); this path was simply missed. It is the worst one to miss. Odds are fetched per live game from inside the live update loop, so its failures are the ones that cost the caller its whole update budget -- the same path the 5s timeout and the cooldown were added to protect. Sent via a session rather than per-call, which also reuses the connection across a slate. Deliberately no retry adapter, unlike api_helper: retries multiply request_timeout, which is 5s precisely to stay inside the 30s operation budget. The existing tests patched the module's requests.get, which this change bypasses -- test_base_odds_manager was consequently reaching the real ESPN and taking 404s. Both files now patch the session, and the new tests pin the agent against api_helper's live value so the two cannot drift apart the next time ESPN moves the goalposts. Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 Co-authored-by: Claude Opus 5 (1M context) --- src/base_odds_manager.py | 20 ++++- test/test_base_odds_manager.py | 6 +- test/test_odds_request_budget.py | 138 ++++++++++++++++++++----------- 3 files changed, 112 insertions(+), 52 deletions(-) diff --git a/src/base_odds_manager.py b/src/base_odds_manager.py index 50b6fe63..4b455069 100644 --- a/src/base_odds_manager.py +++ b/src/base_odds_manager.py @@ -44,6 +44,24 @@ class BaseOddsManager: self.config_manager = config_manager self.logger = logging.getLogger(__name__) self.base_url = "https://sports.core.api.espn.com/v2/sports" + + # This path used a bare requests.get, so it identified itself as + # python-requests/x.y -- the one thing ESPN is known to reject. Around + # 2026-08-04 it began 403ing browser strings and bare custom tokens + # alike; what it accepts is a token with a URL that says who is + # calling. Every other ESPN caller in the tree already sends this + # (src/common/api_helper.py, src/base_classes/data_sources.py); the + # odds path was simply missed, and it is the one whose failures cost + # the caller its whole update budget. + # + # Deliberately no retry adapter, unlike api_helper: retries multiply + # request_timeout, which is set to 5s precisely to stay inside that + # budget. One try, then the cooldown below. + self.session = requests.Session() + self.session.headers.update({ + 'User-Agent': 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)', + 'Accept': 'application/json', + }) # Configuration with defaults self.update_interval = 3600 # 1 hour default @@ -144,7 +162,7 @@ class BaseOddsManager: url = f"{self.base_url}/{sport}/leagues/{espn_league}/events/{event_id}/competitions/{event_id}/odds" self.logger.info(f"Requesting odds from URL: {url}") - response = requests.get(url, timeout=self.request_timeout) + response = self.session.get(url, timeout=self.request_timeout) response.raise_for_status() raw_data = response.json() diff --git a/test/test_base_odds_manager.py b/test/test_base_odds_manager.py index 3eb591e3..e53a5036 100644 --- a/test/test_base_odds_manager.py +++ b/test/test_base_odds_manager.py @@ -8,7 +8,9 @@ is_odds_available's ML-blind truth table, the fixed format_odds_summary gate (money-line-only odds now format), get_odds_for_games, and configuration loading. -No real network: src.base_odds_manager.requests.get is always patched. +No real network: requests.Session.get is always patched. The odds path sends +its requests through a session so it can identify itself to ESPN, so patching +the module-level requests.get would no longer intercept anything. """ from unittest.mock import MagicMock, patch @@ -59,7 +61,7 @@ def manager(cache_manager): @pytest.fixture def mock_get(): - with patch('src.base_odds_manager.requests.get') as m: + with patch('src.base_odds_manager.requests.Session.get') as m: m.return_value = _make_response({'items': [dict(FULL_ITEM)]}) yield m diff --git a/test/test_odds_request_budget.py b/test/test_odds_request_budget.py index 233780fb..6289a18f 100644 --- a/test/test_odds_request_budget.py +++ b/test/test_odds_request_budget.py @@ -10,10 +10,15 @@ and the update carrying every game's score was killed: Invisible out of season -- preseason week 1 returns a single game -- and a Sunday slate is around sixteen. + +The request now goes through a session that identifies the caller, so the +tests patch `manager.session.get` rather than the module's `requests.get`. """ from unittest.mock import Mock +import requests + from src.base_odds_manager import BaseOddsManager PLUGIN_BUDGET = 30.0 # PluginExecutor(default_timeout=30.0) @@ -25,43 +30,83 @@ def _manager(cache=None): return BaseOddsManager(cache_manager=cache, config_manager=None) +def _timing_out(manager): + """Point the manager's session at a request that always times out.""" + manager.session.get = Mock(side_effect=requests.exceptions.Timeout("x")) + return manager.session.get + + +def _returning(manager, payload): + resp = Mock() + resp.json.return_value = payload + resp.raise_for_status.return_value = None + manager.session.get = Mock(return_value=resp) + return manager.session.get + + class TestRequestTimeout: def test_leaves_room_in_the_operation_budget(self): assert _manager().request_timeout < PLUGIN_BUDGET / 2 def test_the_timeout_is_the_one_actually_used(self): m = _manager() - import src.base_odds_manager as mod - real = mod.requests.get - try: - mod.requests.get = Mock(side_effect=mod.requests.exceptions.Timeout("x")) - m.get_odds("football", "nfl", "401") - assert mod.requests.get.call_args.kwargs["timeout"] == m.request_timeout - finally: - mod.requests.get = real + get = _timing_out(m) + m.get_odds("football", "nfl", "401") + assert get.call_args.kwargs["timeout"] == m.request_timeout + + +class TestIdentifiesItselfToEspn: + """ESPN 403s python-requests' default agent, and bare custom tokens. + + What it accepts is a token carrying a URL that says who is calling. This + path used a bare requests.get and so sent the default -- the one thing + known to be rejected. Everything else in the tree that talks to ESPN + already sends the header below. + """ + + def test_the_user_agent_names_the_project_and_links_to_it(self): + ua = _manager().session.headers["User-Agent"] + assert "python-requests" not in ua + assert "LEDMatrix" in ua + assert "github.com/ChuckBuilds/LEDMatrix" in ua + + def test_it_is_the_same_agent_the_rest_of_the_tree_sends(self): + # Compared against the live value rather than a copied literal, so the + # two cannot drift apart the next time ESPN moves the goalposts. + from src.common.api_helper import APIHelper + assert (_manager().session.headers["User-Agent"] + == APIHelper().session.headers["User-Agent"]) + + def test_the_header_reaches_the_request(self): + m = _manager() + get = _returning(m, {}) + m._extract_espn_data = Mock(return_value=None) + m.get_odds("football", "nfl", "401") + # Sent via the session, so it applies without being passed per-call. + assert get.call_count == 1 + assert "User-Agent" in m.session.headers + + def test_no_retry_adapter_multiplies_the_timeout(self): + # api_helper mounts a retrying adapter; this path must not, or a 5s + # timeout becomes 15s and the budget fix is undone. + m = _manager() + for adapter in m.session.adapters.values(): + retries = getattr(adapter, "max_retries", None) + assert getattr(retries, "total", 0) in (0, None), ( + "odds session mounts a retrying adapter (total=%r); retries " + "multiply request_timeout" % getattr(retries, "total", None)) class TestSlowEspnCannotKillTheUpdate: def test_one_failure_stops_the_rest_of_the_slate_hitting_the_network(self): m = _manager() - import src.base_odds_manager as mod - real = mod.requests.get - calls = {"n": 0} + get = _timing_out(m) + for i in range(16): # a full slate, one game at a time + m.get_odds("football", "nfl", "4018730%02d" % i) - def timeout(*a, **k): - calls["n"] += 1 - raise mod.requests.exceptions.Timeout("timed out") - - try: - mod.requests.get = timeout - for i in range(16): # a full slate, one game at a time - m.get_odds("football", "nfl", "4018730%02d" % i) - finally: - mod.requests.get = real - - assert calls["n"] == 1, ( + assert get.call_count == 1, ( "%d games each paid the timeout; the breaker should have stopped " - "after the first" % calls["n"]) + "after the first" % get.call_count) def test_worst_case_slate_stays_inside_the_budget(self): m = _manager() @@ -70,53 +115,48 @@ class TestSlowEspnCannotKillTheUpdate: def test_recovery_is_automatic(self): m = _manager() import src.base_odds_manager as mod - real_get, real_monotonic = mod.requests.get, mod.time.monotonic + real_monotonic = mod.time.monotonic clock = {"t": 1000.0} try: mod.time.monotonic = lambda: clock["t"] - mod.requests.get = Mock( - side_effect=mod.requests.exceptions.Timeout("timed out")) + get = _timing_out(m) m.get_odds("football", "nfl", "401") assert m._skip_network_until > clock["t"], "breaker did not open" clock["t"] += 1 - before = mod.requests.get.call_count + before = get.call_count m.get_odds("football", "nfl", "402") - assert mod.requests.get.call_count == before, "should not have retried" + assert get.call_count == before, "should not have retried" clock["t"] += m._FAILURE_COOLDOWN m.get_odds("football", "nfl", "403") - assert mod.requests.get.call_count > before, "never retried" + assert get.call_count > before, "never retried" finally: - mod.requests.get, mod.time.monotonic = real_get, real_monotonic + mod.time.monotonic = real_monotonic def test_a_healthy_fetch_clears_the_breaker(self): m = _manager() m._skip_network_until = 0.0 m._extract_espn_data = Mock(return_value=None) - import src.base_odds_manager as mod - real = mod.requests.get - try: - resp = Mock() - resp.json.return_value = {} - resp.raise_for_status.return_value = None - mod.requests.get = Mock(return_value=resp) - m.get_odds("football", "nfl", "401") - finally: - mod.requests.get = real + _returning(m, {}) + m.get_odds("football", "nfl", "401") assert m._skip_network_until == 0.0 + def test_a_403_opens_the_breaker_rather_than_hammering(self): + # raise_for_status raises HTTPError, a RequestException -- so a wrong + # or missing agent backs off instead of 403ing once per game. + m = _manager() + resp = Mock() + resp.raise_for_status.side_effect = requests.exceptions.HTTPError("403") + m.session.get = Mock(return_value=resp) + m.get_odds("football", "nfl", "401") + assert m._skip_network_until > 0.0 + def test_the_stale_cache_fallback_still_works(self): # The failing request must still hand back whatever was cached; only # the *subsequent* games skip the network. cache = Mock() cache.get_with_auto_strategy.side_effect = [None, {"details": "stale"}] m = BaseOddsManager(cache_manager=cache, config_manager=None) - import src.base_odds_manager as mod - real = mod.requests.get - try: - mod.requests.get = Mock( - side_effect=mod.requests.exceptions.Timeout("timed out")) - assert m.get_odds("football", "nfl", "401") == {"details": "stale"} - finally: - mod.requests.get = real + _timing_out(m) + assert m.get_odds("football", "nfl", "401") == {"details": "stale"} From 9fbdd7194136b400d9cc8164e5c29b49a4c1d3db Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:40:07 -0400 Subject: [PATCH 16/53] fix(cache): collect the temp files abandoned writes leave behind (#452) * fix(cache): collect the temp files abandoned writes leave behind DiskCache.set() writes through mkstemp then os.replace, and removes its own temp file in a finally. That covers a write that fails, but not a process that dies between the two -- a SIGKILL, a lost restart race, a power cut, all ordinary on a Pi. Nothing ever collected what was left. The temp names are "..json.", and cleanup_expired_files listed only names ending in .json, so every one of them was invisible to the sweep for as long as the card had been in service. On the dev rig: 76 files, 1,050 MB, 81% of the whole cache directory, the oldest six months old. The startup sweep reported "18/8864 files deleted, 0.01 MB freed" while sitting on top of a gigabyte it could not see. They are removed after an hour. A real write holds its temp file for milliseconds, so that is far outside any in-flight write while still clearing the same day's debris, and it is deliberately not tied to the retention policies: those say how long data stays useful, and a half-written file never was. The predicate is tested harder than the sweep, because a false positive deletes real data. It matches the shape set() creates rather than just a leading dot, so a completed ".json", a stray .gitignore, and a "weather.json.bak" are all left alone -- and one test drives set() itself and asserts the names it produces are matched, so the writer and the predicate cannot drift apart. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 * fix(cache): count swept temp files as scanned files_scanned only counted completed .json files, so a sweep that removed orphans reported more deleted than it had looked at -- the summary line renders "/", which came out as "76/1". Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 --------- Co-authored-by: Claude Opus 5 (1M context) --- src/cache/disk_cache.py | 65 ++++++++- test/test_cache_orphan_temp_sweep.py | 188 +++++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 test/test_cache_orphan_temp_sweep.py diff --git a/src/cache/disk_cache.py b/src/cache/disk_cache.py index e50b3a85..03cfb2f1 100644 --- a/src/cache/disk_cache.py +++ b/src/cache/disk_cache.py @@ -14,6 +14,13 @@ import zlib from typing import Dict, Any, Optional, Protocol from datetime import datetime +# How old an abandoned write's temp file must be before the sweep removes it. +# A real write holds its temp file for milliseconds, so an hour is far beyond +# any in-flight write while still clearing the same day's debris. Deliberately +# not tied to the retention policies: those describe how long data stays +# useful, and a half-written file was never useful. +_ORPHAN_TEMP_MAX_AGE_SECONDS = 3600 + class CacheStrategyProtocol(Protocol): @@ -347,6 +354,23 @@ class DiskCache: """Get the cache directory path.""" return self.cache_dir + @staticmethod + def _is_orphaned_temp(filename: str) -> bool: + """Whether a name is one of set()'s temp files rather than real data. + + Matches only what this class creates: mkstemp with a prefix of + ".." , so ".weather.json.a1b2c3d4". The shape is + checked rather than just the leading dot, because this predicate + deletes things -- a stray dotfile someone left in the cache directory + is not ours to remove, and a completed ".json" never is either. + """ + if not filename.startswith('.') or filename.endswith('.json'): + return False + head, sep, suffix = filename.rpartition('.json.') + # head is the key (non-empty after the leading dot), suffix is + # mkstemp's random component. + return bool(sep) and len(head) > 1 and bool(suffix) + def cleanup_expired_files(self, cache_strategy: CacheStrategyProtocol, retention_policies: Dict[str, int]) -> Dict[str, Any]: """ Clean up expired cache files based on retention policies. @@ -381,11 +405,50 @@ class DiskCache: try: with self._lock: # Get snapshot of files while holding lock briefly - filenames = [f for f in os.listdir(self.cache_dir) if f.endswith('.json')] + entries = os.listdir(self.cache_dir) except OSError as list_error: self.logger.error("Error listing cache directory %s: %s", self.cache_dir, list_error, exc_info=True) stats['errors'] += 1 return stats + + filenames = [f for f in entries if f.endswith('.json')] + + # Sweep temp files abandoned by a write that never finished. set() + # removes its own in a finally, so these are the ones where the + # process died between mkstemp and os.replace -- a SIGKILL, a lost + # restart race, a power cut. Nothing ever collected them: they are + # named "..json.", and the scan above only matches + # names ending in .json, so they accumulated indefinitely. Measured + # on a live rig: 76 files, 1,050 MB, 81% of the whole cache + # directory, the oldest six months old. + stats['orphan_temp_files_deleted'] = 0 + for filename in (f for f in entries if self._is_orphaned_temp(f)): + # Counted as scanned like any other candidate, so files_deleted + # can never exceed files_scanned and the summary line reads + # honestly ("77/8864", not "77/0"). + stats['files_scanned'] += 1 + path = os.path.join(self.cache_dir, filename) + try: + # An in-flight write lives for milliseconds, so anything + # this old is certainly abandoned rather than in progress. + if (current_time - os.path.getmtime(path)) <= _ORPHAN_TEMP_MAX_AGE_SECONDS: + continue + with self._lock: + size = os.path.getsize(path) + os.remove(path) + stats['files_deleted'] += 1 + stats['orphan_temp_files_deleted'] += 1 + stats['space_freed_bytes'] += size + except FileNotFoundError: + continue # another sweep got there first + except OSError as e: + stats['errors'] += 1 + self.logger.warning("Error deleting orphaned temp file %s: %s", filename, e) + + if stats['orphan_temp_files_deleted']: + self.logger.info( + "Removed %d abandoned cache temp file(s)", + stats['orphan_temp_files_deleted']) # Process files outside the lock to avoid blocking get/set operations for filename in filenames: diff --git a/test/test_cache_orphan_temp_sweep.py b/test/test_cache_orphan_temp_sweep.py new file mode 100644 index 00000000..f35579d6 --- /dev/null +++ b/test/test_cache_orphan_temp_sweep.py @@ -0,0 +1,188 @@ +"""Tests that abandoned cache temp files get collected. + +DiskCache.set() writes through tempfile.mkstemp and os.replace, removing its +own temp file in a finally. That covers a failed write, but not a process that +dies between the two -- a SIGKILL, a lost restart race, a power cut, all +ordinary on a Pi. Nothing collected what was left behind: the temp names are +"..json.", and the expiry sweep only listed names ending in +.json, so they accumulated for as long as the card had been in service. + +Measured on a live rig before this fix: 76 orphans totalling 1,050 MB -- 81% +of the entire cache directory -- the oldest six months old. + +The predicate that decides what to delete is tested harder than the sweep +itself, because a false positive here destroys real data. +""" + +import os +import time + +import pytest + +from src.cache.disk_cache import DiskCache, _ORPHAN_TEMP_MAX_AGE_SECONDS + + +class FakeStrategy: + @staticmethod + def get_data_type_from_key(key): + return 'default' + + +POLICIES = {'default': 30} + + +@pytest.fixture +def cache(tmp_path): + return DiskCache(str(tmp_path)) + + +def _age(path, seconds): + old = time.time() - seconds + os.utime(path, (old, old)) + + +def _write(tmp_path, name, body='{}'): + p = tmp_path / name + p.write_text(body, encoding='utf-8') + return p + + +class TestWhatCountsAsAnOrphan: + @pytest.mark.parametrize('name', [ + '.weather.json.a1b2c3d4', + '.odds_espn_football_nfl_401.json.xyz00000', + '.a.json.b', + ]) + def test_our_temp_files_are_orphans(self, name): + assert DiskCache._is_orphaned_temp(name) + + @pytest.mark.parametrize('name', [ + 'weather.json', # real data + '.weather.json', # a dotted key that completed + '.gitignore', # not ours + '.hidden', # not ours + 'weather.json.bak', # no leading dot: someone else's + '.json.abc', # no key between the dot and .json. + '.weather.json.', # no random component + 'notes.txt', + ]) + def test_everything_else_is_left_alone(self, name): + assert not DiskCache._is_orphaned_temp(name) + + def test_the_names_set_actually_creates_are_matched(self, cache, tmp_path): + """Guard against the predicate and the writer drifting apart.""" + created = [] + real = os.replace + + def capture(src, dst): + created.append(os.path.basename(src)) + return real(src, dst) + + import src.cache.disk_cache as mod + mod.os.replace = capture + try: + cache.set('weather', {'v': 1}) + finally: + mod.os.replace = real + + assert created, "set() did not go through the temp-file path" + assert all(DiskCache._is_orphaned_temp(n) for n in created), created + + +class TestTheSweep: + def test_an_old_orphan_is_removed(self, cache, tmp_path): + p = _write(tmp_path, '.weather.json.a1b2c3d4', 'x' * 5000) + _age(p, _ORPHAN_TEMP_MAX_AGE_SECONDS + 60) + + stats = cache.cleanup_expired_files(FakeStrategy(), POLICIES) + + assert not p.exists() + assert stats['orphan_temp_files_deleted'] == 1 + assert stats['space_freed_bytes'] >= 5000 + + def test_an_in_flight_write_is_not_snatched_away(self, cache, tmp_path): + # The whole risk of this sweep: deleting a temp file another thread is + # about to os.replace into place. + p = _write(tmp_path, '.weather.json.inflight') + + cache.cleanup_expired_files(FakeStrategy(), POLICIES) + + assert p.exists() + + def test_real_cache_files_survive(self, cache, tmp_path): + fresh = _write(tmp_path, 'weather.json') + dotted = _write(tmp_path, '.weather.json') + _age(dotted, _ORPHAN_TEMP_MAX_AGE_SECONDS + 60) + + cache.cleanup_expired_files(FakeStrategy(), POLICIES) + + assert fresh.exists() + assert dotted.exists(), "a completed .json was treated as a temp file" + + def test_unrelated_dotfiles_survive(self, cache, tmp_path): + keep = _write(tmp_path, '.gitignore') + _age(keep, 400 * 86400) + + cache.cleanup_expired_files(FakeStrategy(), POLICIES) + + assert keep.exists() + + def test_expiry_still_works_alongside_it(self, cache, tmp_path): + stale = _write(tmp_path, 'old.json') + _age(stale, 40 * 86400) # past the 30-day default + orphan = _write(tmp_path, '.old.json.zz999999') + _age(orphan, _ORPHAN_TEMP_MAX_AGE_SECONDS + 60) + + stats = cache.cleanup_expired_files(FakeStrategy(), POLICIES) + + assert not stale.exists() + assert not orphan.exists() + assert stats['files_deleted'] == 2 + assert stats['orphan_temp_files_deleted'] == 1 + + def test_the_rig_scenario(self, cache, tmp_path): + """76 orphans of assorted ages, none of them reachable before.""" + for i in range(76): + p = _write(tmp_path, '.sched_%d.json.r%06d' % (i, i), 'x' * 1000) + _age(p, (i + 2) * 86400) + keep = _write(tmp_path, 'sched.json') + + stats = cache.cleanup_expired_files(FakeStrategy(), POLICIES) + + assert stats['orphan_temp_files_deleted'] == 76 + assert keep.exists() + assert not list(tmp_path.glob('.sched_*')) + # The summary line is "/", so an orphan that is + # deleted but never counted as scanned renders as "76/1". + assert stats['files_scanned'] == 77 + assert stats['files_deleted'] <= stats['files_scanned'] + + def test_deleted_never_exceeds_scanned(self, cache, tmp_path): + p = _write(tmp_path, '.only.json.a1b2c3d4') + _age(p, _ORPHAN_TEMP_MAX_AGE_SECONDS + 60) + + stats = cache.cleanup_expired_files(FakeStrategy(), POLICIES) + + assert stats['files_deleted'] == 1 + assert stats['files_scanned'] == 1 + + def test_a_missing_file_mid_sweep_is_not_an_error(self, cache, tmp_path): + p = _write(tmp_path, '.weather.json.a1b2c3d4') + _age(p, _ORPHAN_TEMP_MAX_AGE_SECONDS + 60) + + import src.cache.disk_cache as mod + real = mod.os.path.getsize + + def vanish(path): + if path.endswith('.a1b2c3d4'): + os.remove(path) + raise FileNotFoundError(path) + return real(path) + + mod.os.path.getsize = vanish + try: + stats = cache.cleanup_expired_files(FakeStrategy(), POLICIES) + finally: + mod.os.path.getsize = real + + assert stats['errors'] == 0 From 7171e6c0228b3b6855238567cf692d4c831edb1b Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 12 Aug 2026 08:42:56 -0400 Subject: [PATCH 17/53] fix(cache): one cleanup thread per cache directory, not per manager (#453) The display process ran three cleanup threads over one directory: 14:22:59.954 display_controller (the real manager) 14:22:59.973 startup validation, run 1 (discarded) 14:23:01.055 startup validation, run 2 (discarded) Two of those managers existed only to read a directory path. StartupValidator._validate_cache_directory built a whole CacheManager to call get_cache_dir(), and validation runs twice -- once before the plugin manager exists and again after. Each construction also probes writability by writing and deleting .writetest on the card. The discarded ones never went away. cleanup_loop closes over `self`, so the thread keeps its manager alive: two objects that could never be collected, waking every 24 hours to re-scan the same 9,000-file directory. Nothing stopped them either -- stop_cleanup_thread had no callers anywhere in the tree. Two changes. The validator now takes the CacheManager the application actually uses, which is also the more correct thing to validate; when no caller supplies one it still builds its own, but stops the thread afterwards. And CacheManager now tracks which directory it is sweeping, so the second manager over a directory skips starting a thread at all. That is the right granularity regardless of call sites: the sweep lists a directory and deletes from it, so a second thread only duplicates the scan. Ownership is released on stop, so a survivor can take over rather than leaving the directory permanently unclaimed by a dead owner. Measured directly, three managers over one directory: 3 threads before, 1 after. Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5 Co-authored-by: Claude Opus 5 (1M context) --- src/cache_manager.py | 47 ++++++- src/display_controller.py | 6 +- src/startup_validator.py | 29 +++- test/test_cache_cleanup_thread_ownership.py | 146 ++++++++++++++++++++ 4 files changed, 217 insertions(+), 11 deletions(-) create mode 100644 test/test_cache_cleanup_thread_ownership.py diff --git a/src/cache_manager.py b/src/cache_manager.py index 3eaab7fa..bef98784 100644 --- a/src/cache_manager.py +++ b/src/cache_manager.py @@ -46,7 +46,21 @@ from src.cache.disk_cache import DateTimeEncoder # noqa: F401 - deliberate re-e class CacheManager: """Manages caching of API responses to reduce API calls.""" - + + # Which cache directories already have a cleanup thread in this process. + # + # The sweep is directory-scoped work -- it lists a directory and deletes + # from it -- so one per directory is the right number no matter how many + # managers exist. Nothing enforced that before: every instance started its + # own, and because the loop closes over `self`, a discarded manager could + # never be collected and its thread woke to re-scan the same directory + # every 24 hours for the life of the process. Startup validation runs + # twice and built a throwaway manager each time, so a display process + # carried three threads for one cache. + _cleanup_owners: Dict[str, 'CacheManager'] = {} + _cleanup_owners_lock = threading.Lock() + + def __init__(self) -> None: # Initialize logger first self.logger: logging.Logger = get_logger(__name__) @@ -718,11 +732,29 @@ class CacheManager: } def start_cleanup_thread(self) -> None: - """Start background thread for periodic disk cache cleanup.""" + """Start background thread for periodic disk cache cleanup. + + At most one thread per cache directory per process: the sweep is + directory-scoped, so a second one only duplicates the scan. + """ if self._cleanup_thread and self._cleanup_thread.is_alive(): self.logger.debug("Cleanup thread already running") return - + + with CacheManager._cleanup_owners_lock: + owner = CacheManager._cleanup_owners.get(self.cache_dir) + if owner is not None and owner is not self: + thread = owner._cleanup_thread + if thread is not None and thread.is_alive(): + self.logger.debug( + "Cleanup thread for %s already owned by another cache " + "manager in this process; not starting a second", + self.cache_dir) + return + # The owner's thread died or was stopped -- take over. + CacheManager._cleanup_owners[self.cache_dir] = self + + def cleanup_loop(): """Background loop that runs cleanup periodically.""" self.logger.info("Disk cache cleanup thread started (interval: %d hours)", @@ -770,10 +802,17 @@ class CacheManager: Signals the thread to stop and waits for it to finish (with timeout). This allows for clean shutdown during testing or application termination. """ + # Release ownership first and unconditionally, so a manager that never + # started a thread (or whose thread already exited) cannot keep the + # directory claimed and block a live manager from sweeping it. + with CacheManager._cleanup_owners_lock: + if CacheManager._cleanup_owners.get(self.cache_dir) is self: + del CacheManager._cleanup_owners[self.cache_dir] + if not self._cleanup_thread or not self._cleanup_thread.is_alive(): self.logger.debug("Cleanup thread not running") return - + self.logger.info("Stopping disk cache cleanup thread...") self._cleanup_stop_event.set() # Signal thread to stop diff --git a/src/display_controller.py b/src/display_controller.py index 01877279..dada2974 100644 --- a/src/display_controller.py +++ b/src/display_controller.py @@ -90,7 +90,8 @@ class DisplayController: # Validate startup configuration try: from src.startup_validator import StartupValidator - validator = StartupValidator(self.config_manager) + validator = StartupValidator(self.config_manager, + cache_manager=self.cache_manager) is_valid, errors, warnings = validator.validate_all() if warnings: @@ -258,7 +259,8 @@ class DisplayController: # Validate plugins after plugin manager is created try: from src.startup_validator import StartupValidator - validator = StartupValidator(self.config_manager, self.plugin_manager) + validator = StartupValidator(self.config_manager, self.plugin_manager, + cache_manager=self.cache_manager) is_valid, errors, warnings = validator.validate_all() if warnings: diff --git a/src/startup_validator.py b/src/startup_validator.py index 8dd9fce2..9f2e0a51 100644 --- a/src/startup_validator.py +++ b/src/startup_validator.py @@ -15,16 +15,23 @@ from src.logging_config import get_logger class StartupValidator: """Validates system state on startup.""" - def __init__(self, config_manager: Any, plugin_manager: Optional[Any] = None) -> None: + def __init__(self, config_manager: Any, plugin_manager: Optional[Any] = None, + cache_manager: Optional[Any] = None) -> None: """ Initialize the startup validator. - + Args: config_manager: ConfigManager instance plugin_manager: Optional PluginManager instance + cache_manager: The CacheManager the application will actually use. + Pass it. Without one this validator builds its own just to read + a directory path, which reports on a cache the app does not + use and leaves behind a cleanup thread that nothing stops -- + validation runs twice per startup, so that was two of them. """ self.config_manager = config_manager self.plugin_manager = plugin_manager + self.cache_manager = cache_manager self.logger = get_logger(__name__) self.errors: List[str] = [] self.warnings: List[str] = [] @@ -91,9 +98,21 @@ class StartupValidator: def _validate_cache_directory(self) -> None: """Validate cache directory permissions.""" try: - from src.cache_manager import CacheManager - cache_manager = CacheManager() - cache_dir = cache_manager.get_cache_dir() + cache_manager = self.cache_manager + if cache_manager is None: + # No caller supplied one (older embedders, direct use in a + # script). Build one, but do not leave its cleanup thread + # running behind us -- this instance is discarded on the next + # line but the thread is a closure over it, so it would never + # be collected. + from src.cache_manager import CacheManager + cache_manager = CacheManager() + try: + cache_dir = cache_manager.get_cache_dir() + finally: + cache_manager.stop_cleanup_thread() + else: + cache_dir = cache_manager.get_cache_dir() if not cache_dir: self.warnings.append("Cache directory not available - caching will be disabled") diff --git a/test/test_cache_cleanup_thread_ownership.py b/test/test_cache_cleanup_thread_ownership.py new file mode 100644 index 00000000..2efa38c6 --- /dev/null +++ b/test/test_cache_cleanup_thread_ownership.py @@ -0,0 +1,146 @@ +"""Tests that one cache directory gets one cleanup thread per process. + +The sweep lists a directory and deletes from it, so a second thread over the +same directory only duplicates the scan. Nothing enforced that: every +CacheManager started its own, and since the loop closes over `self`, a +discarded manager could never be collected -- its thread stayed alive and +re-scanned the same directory every 24 hours for the life of the process. + +On the dev rig a display process carried three, for one cache directory: + + 14:22:59.954 display_controller (the real one) + 14:22:59.973 startup validation, run 1 (discarded) + 14:23:01.055 startup validation, run 2 (discarded) + +Startup validation runs twice and built a throwaway manager each time, purely +to read a directory path. +""" + +import threading + +import pytest + +from src.cache_manager import CacheManager + + +@pytest.fixture(autouse=True) +def _clean_registry(): + CacheManager._cleanup_owners.clear() + yield + for owner in list(CacheManager._cleanup_owners.values()): + owner.stop_cleanup_thread() + CacheManager._cleanup_owners.clear() + + +def _live_cleanup_threads(): + return [t for t in threading.enumerate() + if t.name == 'DiskCacheCleanup' and t.is_alive()] + + +@pytest.fixture +def manager(tmp_path, monkeypatch): + """A CacheManager pinned to a temp dir, so tests never touch the real one.""" + monkeypatch.setattr(CacheManager, '_get_writable_cache_dir', + lambda self: str(tmp_path)) + return CacheManager + + +class TestOneThreadPerDirectory: + def test_a_single_manager_starts_one(self, manager): + before = len(_live_cleanup_threads()) + m = manager() + try: + assert len(_live_cleanup_threads()) == before + 1 + finally: + m.stop_cleanup_thread() + + def test_three_managers_still_start_one(self, manager): + # Exactly the rig's shape: the real manager plus two throwaways. + before = len(_live_cleanup_threads()) + managers = [manager() for _ in range(3)] + try: + assert len(_live_cleanup_threads()) == before + 1 + finally: + for m in managers: + m.stop_cleanup_thread() + + def test_the_first_one_owns_it(self, manager): + first, second = manager(), manager() + try: + assert CacheManager._cleanup_owners[first.cache_dir] is first + assert second._cleanup_thread is None + finally: + first.stop_cleanup_thread() + second.stop_cleanup_thread() + + def test_the_survivor_can_take_over(self, manager): + first = manager() + first.stop_cleanup_thread() + assert not _live_cleanup_threads() + + second = manager() + try: + # Ownership was released, so the directory is swept again rather + # than being left permanently unclaimed by a dead owner. + assert len(_live_cleanup_threads()) == 1 + assert CacheManager._cleanup_owners[second.cache_dir] is second + finally: + second.stop_cleanup_thread() + + def test_stopping_a_non_owner_does_not_unclaim_the_directory(self, manager): + first, second = manager(), manager() + try: + second.stop_cleanup_thread() # never owned it + assert CacheManager._cleanup_owners[first.cache_dir] is first + assert len(_live_cleanup_threads()) == 1 + finally: + first.stop_cleanup_thread() + + def test_separate_directories_get_separate_threads(self, tmp_path, monkeypatch): + a, b = tmp_path / 'a', tmp_path / 'b' + a.mkdir() + b.mkdir() + dirs = iter([str(a), str(b)]) + monkeypatch.setattr(CacheManager, '_get_writable_cache_dir', + lambda self: next(dirs)) + first, second = CacheManager(), CacheManager() + try: + assert first.cache_dir != second.cache_dir + assert len(_live_cleanup_threads()) == 2 + finally: + first.stop_cleanup_thread() + second.stop_cleanup_thread() + + def test_no_thread_leaks_across_many_constructions(self, manager): + before = len(_live_cleanup_threads()) + made = [manager() for _ in range(12)] + try: + assert len(_live_cleanup_threads()) == before + 1 + finally: + for m in made: + m.stop_cleanup_thread() + assert len(_live_cleanup_threads()) == before + + +class TestValidatorDoesNotBuildItsOwn: + def test_it_uses_the_cache_manager_it_is_given(self, manager): + from src.startup_validator import StartupValidator + + shared = manager() + try: + before = len(_live_cleanup_threads()) + v = StartupValidator(config_manager=object(), cache_manager=shared) + v._validate_cache_directory() + assert len(_live_cleanup_threads()) == before, ( + "validation started another cleanup thread") + finally: + shared.stop_cleanup_thread() + + def test_without_one_it_cleans_up_after_itself(self, manager): + from src.startup_validator import StartupValidator + + before = len(_live_cleanup_threads()) + v = StartupValidator(config_manager=object()) + v._validate_cache_directory() + assert len(_live_cleanup_threads()) == before, ( + "the fallback manager left its cleanup thread running") From fce1fdac57d94f3b17345a1a8a5973b22f31d37c Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:40:48 -0400 Subject: [PATCH 18/53] Add panel orientation setting for upside-down mounting (#455) Adds a display.hardware.orientation config field ("normal" / "180") so panels mounted upside down (e.g. to put the Pi/wiring on a more convenient side) render correctly without custom pixel_mapper_config edits. Composes onto the existing pixel_mapper_config as a trailing "Rotate:180" mapper, so it stays independent of any custom mapper string (e.g. U-mapper chain layouts) already in use. Exposed as a "Panel Orientation" dropdown in the web UI's Display settings, validated server-side, and documented in README and CONFIG_REFERENCE. Claude-Session: https://claude.ai/code/session_01FakipqMDHQLpsFjTuBdSFQ Co-authored-by: Claude --- README.md | 8 ++++ config/config.template.json | 1 + docs/CONFIG_REFERENCE.md | 1 + src/display_manager.py | 22 +++++++++- test/test_display_manager.py | 42 +++++++++++++++++++ web_interface/blueprints/api_v3.py | 9 +++- .../templates/v3/partials/display.html | 8 ++++ 7 files changed, 88 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a23b6720..681176ef 100644 --- a/README.md +++ b/README.md @@ -600,6 +600,14 @@ These settings are typically only needed for non-standard panels or custom confi - Leave empty unless you need custom mapping - See rpi-rgb-led-matrix documentation for full options +- **`orientation`** (string, default: "normal") + - Rotates the rendered image to match how the panel is physically mounted + - Set to `"180"` (or use the "Upside Down" option in the web UI's Display + settings) if the panel is mounted upside down — useful for optimizing + where the Raspberry Pi and wiring sit relative to the mounting location + - Applied independently of `pixel_mapper_config` (appended as a trailing + `Rotate:180` mapper), so custom mapper configs keep working alongside it + - **`row_address_type`** (integer, default: 0) - How rows are addressed on the panel - Most panels use 0 (direct addressing) diff --git a/config/config.template.json b/config/config.template.json index 9edbec18..59bfdb34 100644 --- a/config/config.template.json +++ b/config/config.template.json @@ -112,6 +112,7 @@ "led_rgb_sequence": "RGB", "limit_refresh_rate_hz": 100, "pixel_mapper_config": "", + "orientation": "normal", "row_address_type": 0, "multiplexing": 0, "panel_type": "" diff --git a/docs/CONFIG_REFERENCE.md b/docs/CONFIG_REFERENCE.md index ff623439..ff1202fb 100644 --- a/docs/CONFIG_REFERENCE.md +++ b/docs/CONFIG_REFERENCE.md @@ -66,6 +66,7 @@ in `DisplayManager` (`src/display_manager.py`, ~lines 270–295). | `led_rgb_sequence` | string, `"RGB"` | | `limit_refresh_rate_hz` | int, `100` (code default 90) | | `pixel_mapper_config` | string, `""` — e.g. `"U-mapper"` / `"Rotate:90"` | +| `orientation` | string, `"normal"` — `"180"` rotates the rendered image 180° for panels physically mounted upside down (e.g. to move the Pi/wiring to a more convenient side); composed onto `pixel_mapper_config` as a trailing `Rotate:180` mapper, so it stays independent of any custom `pixel_mapper_config` value | | `row_address_type` | int, `0` — non-standard panel row addressing | | `multiplexing` | int, `0` — panel multiplexing scheme | | `panel_type` | string, `""` — set to `"FM6126A"` or `"FM6127"` for panels needing init | diff --git a/src/display_manager.py b/src/display_manager.py index d96fef35..c33e0861 100644 --- a/src/display_manager.py +++ b/src/display_manager.py @@ -258,6 +258,26 @@ class DisplayManager: # Initialize managers # Calendar manager is now initialized by DisplayController + # Orientation setting -> rpi-rgb-led-matrix "Rotate:" pixel-mapper suffix. + # "normal" needs no suffix since 0 degrees is the identity transform. + _ORIENTATION_ROTATE_DEGREES = {'normal': None, '90': 90, '180': 180, '270': 270} + + def _build_pixel_mapper_config(self, hardware_config: dict) -> str: + """Compose the raw pixel_mapper_config string with the orientation setting. + + `pixel_mapper_config` stays available as a free-form advanced field (e.g. + for "U-mapper" chain layouts); `orientation` is the user-facing dropdown + for physical mounting (e.g. panels mounted upside down) and is appended as + a "Rotate:" mapper rather than overwriting any existing config. + """ + base_mapper = (hardware_config.get('pixel_mapper_config') or '').strip() + orientation = hardware_config.get('orientation', 'normal') + degrees = self._ORIENTATION_ROTATE_DEGREES.get(orientation) + if degrees is None: + return base_mapper + rotate_mapper = f'Rotate:{degrees}' + return f'{base_mapper};{rotate_mapper}' if base_mapper else rotate_mapper + def _setup_matrix(self): """Initialize the RGB matrix with configuration settings.""" _init_error_str = None @@ -283,7 +303,7 @@ class DisplayManager: options.pwm_bits = hardware_config.get('pwm_bits', 10) options.pwm_lsb_nanoseconds = hardware_config.get('pwm_lsb_nanoseconds', 150) options.led_rgb_sequence = hardware_config.get('led_rgb_sequence', 'RGB') - options.pixel_mapper_config = hardware_config.get('pixel_mapper_config', '') + options.pixel_mapper_config = self._build_pixel_mapper_config(hardware_config) options.row_address_type = hardware_config.get('row_address_type', 0) options.multiplexing = hardware_config.get('multiplexing', 0) options.panel_type = hardware_config.get('panel_type', '') diff --git a/test/test_display_manager.py b/test/test_display_manager.py index 9c8b1468..9ccb7773 100644 --- a/test/test_display_manager.py +++ b/test/test_display_manager.py @@ -237,3 +237,45 @@ class TestDisplayManagerDoubleSided: suppress_test_pattern=True) assert dm.set_brightness(70) is True assert mock_rgb_matrix['matrix_instance'].brightness == 70 + + +class TestDisplayManagerOrientation: + """The orientation setting composes onto pixel_mapper_config for panels + mounted upside down, without disturbing a custom pixel_mapper_config.""" + + def _config(self, **hardware_overrides): + config = { + 'display': { + 'hardware': { + 'rows': 32, 'cols': 64, 'chain_length': 2, 'parallel': 1, + 'hardware_mapping': 'adafruit-hat-pwm', 'brightness': 90, + }, + 'runtime': {'gpio_slowdown': 2}, + }, + 'timezone': 'UTC', + 'plugin_system': {'plugins_directory': 'plugins'}, + } + config['display']['hardware'].update(hardware_overrides) + return config + + def test_default_orientation_leaves_pixel_mapper_config_untouched(self, mock_rgb_matrix): + DisplayManager._instance = None + with patch.dict('os.environ', {'EMULATOR': 'false'}): + DisplayManager(self._config(), suppress_test_pattern=True) + options = mock_rgb_matrix['options_class'].return_value + assert options.pixel_mapper_config == '' + + def test_orientation_180_appends_rotate_mapper(self, mock_rgb_matrix): + DisplayManager._instance = None + with patch.dict('os.environ', {'EMULATOR': 'false'}): + DisplayManager(self._config(orientation='180'), suppress_test_pattern=True) + options = mock_rgb_matrix['options_class'].return_value + assert options.pixel_mapper_config == 'Rotate:180' + + def test_orientation_180_composes_with_existing_pixel_mapper_config(self, mock_rgb_matrix): + DisplayManager._instance = None + with patch.dict('os.environ', {'EMULATOR': 'false'}): + DisplayManager(self._config(orientation='180', pixel_mapper_config='U-mapper'), + suppress_test_pattern=True) + options = mock_rgb_matrix['options_class'].return_value + assert options.pixel_mapper_config == 'U-mapper;Rotate:180' diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 6062b85d..06fef117 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -796,7 +796,7 @@ def save_main_config(): 'gpio_slowdown', 'rp1_rio', 'scan_mode', 'disable_hardware_pulsing', 'inverse_colors', 'show_refresh_rate', 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', 'use_short_date_format', 'max_dynamic_duration_seconds', 'led_rgb_sequence', 'multiplexing', 'panel_type', - 'row_address_type', 'pixel_mapper_config'] + 'row_address_type', 'pixel_mapper_config', 'orientation'] if any(k in data for k in display_fields): if 'display' not in current_config: @@ -831,6 +831,11 @@ def save_main_config(): if 'pixel_mapper_config' in data and not isinstance(data['pixel_mapper_config'], str): return jsonify({'status': 'error', 'message': 'pixel_mapper_config must be a string (e.g. "U-mapper;Rotate:90" or empty)'}), 400 + # Validate orientation (physical mounting rotation; composed onto pixel_mapper_config at runtime) + ORIENTATION_ALLOWED = {'normal', '180'} + if 'orientation' in data and data['orientation'] not in ORIENTATION_ALLOWED: + return jsonify({'status': 'error', 'message': f"Invalid orientation '{data['orientation']}'. Allowed values: {', '.join(sorted(ORIENTATION_ALLOWED))}"}), 400 + # Validate row_address_type if 'row_address_type' in data: try: @@ -844,7 +849,7 @@ def save_main_config(): for field in ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'hardware_mapping', 'scan_mode', 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', 'led_rgb_sequence', 'multiplexing', 'panel_type', 'row_address_type', - 'pixel_mapper_config']: + 'pixel_mapper_config', 'orientation']: if field in data: if field in ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'scan_mode', 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', diff --git a/web_interface/templates/v3/partials/display.html b/web_interface/templates/v3/partials/display.html index 1e2c423a..95e64d33 100644 --- a/web_interface/templates/v3/partials/display.html +++ b/web_interface/templates/v3/partials/display.html @@ -117,6 +117,14 @@
+
+ + +
+
. That produced two boxes on the calendar + # page -- the widget's own, and a stray one for the same field -- and + # no way to tell which to paste into. + template = (Path(project_root) + / 'web_interface/templates/v3/partials/plugin_config.html' + ).read_text(encoding='utf-8') + allow_list_line = [ln for ln in template.splitlines() + if "str_widget in [" in ln] + assert allow_list_line, "the string widget allow-list moved" + assert "'google-oauth'" in allow_list_line[0], allow_list_line[0] + + def test_the_widget_script_is_served(self): + base = (Path(project_root) / 'web_interface/templates/v3/base.html' + ).read_text(encoding='utf-8') + assert 'widgets/google-oauth.js' in base + + def test_the_status_line_is_announced(self): + # Every message the widget gives arrives after an async call, so a + # screen reader hears nothing unless the element is a live region. + widget = (Path(project_root) + / 'web_interface/static/v3/js/widgets/google-oauth.js' + ).read_text(encoding='utf-8') + # Both attributes must be on the *status* element. Searching for them + # separately would pass with each on a different node, which announces + # nothing. + assert "status.setAttribute('role', 'status')" in widget, widget[:0] + assert "status.setAttribute('aria-live', 'polite')" in widget + + def test_the_paste_box_has_an_accessible_name(self): + # A visible label is not enough on its own: without the association the + # input's only name is a placeholder, which vanishes on focus -- which + # is exactly when the value is being pasted. + widget = (Path(project_root) + / 'web_interface/static/v3/js/widgets/google-oauth.js' + ).read_text(encoding='utf-8') + # The binding is what matters, not that both lines exist: a `for` and + # an `id` that disagree leave the input just as anonymous. Both must + # go through the same identifier. + import re as _re + for_target = _re.search(r"codeLabel\.setAttribute\('for',\s*(\w+)\)", widget) + id_source = _re.search(r"codeInput\.id\s*=\s*(\w+)", widget) + assert for_target and id_source, (for_target, id_source) + assert for_target.group(1) == id_source.group(1), ( + "label points at %r but the input is %r" + % (for_target.group(1), id_source.group(1))) + + def test_the_failed_page_is_called_out_loudly(self): + # The loopback redirect lands on a browser error page at exactly the + # moment the user has to act. In small grey text it gets missed and the + # flow reads as broken while it is working. + widget = (Path(project_root) + / 'web_interface/static/v3/js/widgets/google-oauth.js' + ).read_text(encoding='utf-8') + assert 'expected' in widget.lower() + assert 'amber' in widget, "the warning is not visually distinguished" + + +class TestItSaysWhatIsWrong: + def test_listing_without_a_token_asks_for_step_2(self, client): + response = client.get('/api/v3/plugins/calendar/list-calendars') + assert response.status_code == 400 + body = response.get_json() + assert body['status'] == 'error' + assert 'step 2' in body['message'].lower(), body['message'] + + def test_authenticating_without_credentials_asks_for_step_1(self, client): + response = client.post('/api/v3/plugins/calendar/authenticate', json={}) + assert response.status_code == 400 + assert 'step 1' in response.get_json()['message'].lower() + + def test_an_uninstalled_plugin_says_so(self, uninstalled): + for response in ( + uninstalled.get('/api/v3/plugins/calendar/list-calendars'), + uninstalled.post('/api/v3/plugins/calendar/authenticate', json={}), + ): + assert response.status_code == 404 + # A 404 here is honest -- but it must name the plugin, not read as + # the generic "Resource not found" that started this. + assert 'not installed' in response.get_json()['message'].lower() + + +class TestTheScriptRunner: + def test_it_returns_the_json_the_script_prints(self, tmp_path): + script = tmp_path / 'calendar_registration.py' + script.write_text( + 'print(\'{"status": "success", "auth_url": "https://x"}\')\n', + encoding='utf-8') + payload, error = mod._run_calendar_registration(tmp_path, '') + assert error is None + assert payload['auth_url'] == 'https://x' + + def test_it_ignores_noise_before_the_json(self, tmp_path): + # An import warning or a library writing to stdout would otherwise + # make the last-line parse fail. + script = tmp_path / 'calendar_registration.py' + script.write_text( + 'print("some library warning")\n' + 'print(\'{"status": "success"}\')\n', encoding='utf-8') + payload, error = mod._run_calendar_registration(tmp_path, '') + assert error is None and payload['status'] == 'success' + + def test_it_passes_stdin_through(self, tmp_path): + script = tmp_path / 'calendar_registration.py' + script.write_text( + 'import sys, json\n' + 'print(json.dumps({"status": "success", "got": sys.stdin.read().strip()}))\n', + encoding='utf-8') + payload, _ = mod._run_calendar_registration(tmp_path, 'http://127.0.0.1/?code=abc') + assert payload['got'] == 'http://127.0.0.1/?code=abc' + + def test_a_missing_script_is_reported(self, tmp_path): + payload, error = mod._run_calendar_registration(tmp_path, '') + assert payload is None + assert 'script not found' in error.lower() + + def test_output_that_is_not_json_is_reported_with_context(self, tmp_path): + script = tmp_path / 'calendar_registration.py' + script.write_text('import sys\nsys.stderr.write("boom\\n")\n', encoding='utf-8') + payload, error = mod._run_calendar_registration(tmp_path, '') + assert payload is None + assert 'no result' in error.lower() + assert 'boom' in error + + +class TestListingShape: + """The picker reads cal.id, cal.summary and cal.primary.""" + + def _authenticate(self, client, monkeypatch, items): + creds = type('C', (), {'expired': False, 'refresh_token': None, 'valid': True})() + (client.plugin_dir / 'token.pickle').write_bytes(pickle.dumps({'x': 1})) + monkeypatch.setattr(mod.pickle if hasattr(mod, 'pickle') else pickle, + 'loads', lambda *a, **k: creds, raising=False) + + import types + fake_pickle = types.SimpleNamespace(load=lambda f: creds, dump=lambda *a: None) + # Callers pass a flat list of calendars; the API returns them wrapped + # in a page. One page is all these cases need -- TestPagination builds + # its own multi-page sequences. + pages = [{'items': items}] + + state = {'i': 0} + + def fake_list(**kwargs): + page = pages[min(state['i'], len(pages) - 1)] + state['i'] += 1 + return types.SimpleNamespace(execute=lambda: page) + + def fake_build(*args, **kwargs): + return types.SimpleNamespace( + calendarList=lambda: types.SimpleNamespace(list=fake_list)) + + real_import = __builtins__['__import__'] if isinstance(__builtins__, dict) \ + else __builtins__.__import__ + + def fake_import(name, *args, **kwargs): + if name == 'pickle': + return fake_pickle + if name == 'google.auth.transport.requests': + return types.SimpleNamespace(Request=object) + if name == 'googleapiclient.discovery': + return types.SimpleNamespace(build=fake_build) + return real_import(name, *args, **kwargs) + + monkeypatch.setattr('builtins.__import__', fake_import) + + def test_it_returns_id_summary_and_primary(self, client, monkeypatch): + self._authenticate(client, monkeypatch, [ + {'id': 'b@x', 'summary': 'Work'}, + {'id': 'a@x', 'summary': 'Personal', 'primary': True}, + ]) + body = client.get('/api/v3/plugins/calendar/list-calendars').get_json() + assert body['status'] == 'success' + assert {c['id'] for c in body['calendars']} == {'a@x', 'b@x'} + assert all(set(c) == {'id', 'summary', 'primary'} for c in body['calendars']) + + def test_the_primary_calendar_comes_first(self, client, monkeypatch): + # Short list, but the one the user wants is almost always their own. + self._authenticate(client, monkeypatch, [ + {'id': 'z@x', 'summary': 'Aardvarks'}, + {'id': 'a@x', 'summary': 'Zebras', 'primary': True}, + ]) + body = client.get('/api/v3/plugins/calendar/list-calendars').get_json() + assert body['calendars'][0]['id'] == 'a@x' + assert body['calendars'][0]['primary'] is True + + def test_a_calendar_without_a_name_still_lists(self, client, monkeypatch): + self._authenticate(client, monkeypatch, [{'id': 'noname@x'}]) + body = client.get('/api/v3/plugins/calendar/list-calendars').get_json() + assert body['calendars'][0]['summary'] == 'noname@x' + + def test_entries_without_an_id_are_dropped(self, client, monkeypatch): + # Nothing could be selected by such a row, and the checkbox value + # would be undefined. + self._authenticate(client, monkeypatch, [{'summary': 'ghost'}, {'id': 'real@x'}]) + body = client.get('/api/v3/plugins/calendar/list-calendars').get_json() + assert [c['id'] for c in body['calendars']] == ['real@x'] + + +class TestPagination: + """calendarList.list pages at 250 and defaults to 100.""" + + def _paged(self, client, monkeypatch, pages): + import types + creds = type('C', (), {'expired': False, 'refresh_token': None, 'valid': True})() + (client.plugin_dir / 'token.pickle').write_bytes(b'x') + state = {'i': 0} + seen = [] + + def fake_list(**kwargs): + seen.append(kwargs) + page = pages[min(state['i'], len(pages) - 1)] + state['i'] += 1 + return types.SimpleNamespace(execute=lambda: page) + + def fake_build(*args, **kwargs): + return types.SimpleNamespace( + calendarList=lambda: types.SimpleNamespace(list=fake_list)) + + real_import = __builtins__['__import__'] if isinstance(__builtins__, dict) \ + else __builtins__.__import__ + + def fake_import(name, *args, **kwargs): + if name == 'pickle': + return types.SimpleNamespace(load=lambda f: creds, dump=lambda *a: None) + if name == 'google.auth.transport.requests': + return types.SimpleNamespace(Request=object) + if name == 'googleapiclient.discovery': + return types.SimpleNamespace(build=fake_build) + return real_import(name, *args, **kwargs) + + monkeypatch.setattr('builtins.__import__', fake_import) + return seen + + def test_every_page_is_collected(self, client, monkeypatch): + # Taking only the first page would hide calendars from the picker with + # nothing to say the list was cut short. + self._paged(client, monkeypatch, [ + {'items': [{'id': 'a@x', 'summary': 'A'}], 'nextPageToken': 't1'}, + {'items': [{'id': 'b@x', 'summary': 'B'}], 'nextPageToken': 't2'}, + {'items': [{'id': 'c@x', 'summary': 'C'}]}, + ]) + body = client.get('/api/v3/plugins/calendar/list-calendars').get_json() + assert [c['id'] for c in body['calendars']] == ['a@x', 'b@x', 'c@x'] + + def test_the_page_token_is_passed_back(self, client, monkeypatch): + seen = self._paged(client, monkeypatch, [ + {'items': [{'id': 'a@x', 'summary': 'A'}], 'nextPageToken': 'tok'}, + {'items': [{'id': 'b@x', 'summary': 'B'}]}, + ]) + client.get('/api/v3/plugins/calendar/list-calendars') + assert seen[0]['pageToken'] is None + assert seen[1]['pageToken'] == 'tok' + assert all(k['maxResults'] == 250 for k in seen) + + def test_a_looping_token_cannot_spin_forever(self, client, monkeypatch): + # Every page claims another follows. + self._paged(client, monkeypatch, [ + {'items': [{'id': 'a@x', 'summary': 'A'}], 'nextPageToken': 'same'}, + ]) + body = client.get('/api/v3/plugins/calendar/list-calendars').get_json() + assert body['status'] == 'success' + assert len(body['calendars']) <= mod._CALENDAR_LIST_MAX_PAGES + + +class TestDiagnosticsAreRedacted: + def test_script_stderr_is_redacted_on_the_way_out(self, tmp_path): + script = tmp_path / 'calendar_registration.py' + script.write_text( + 'import sys\n' + 'sys.stderr.write("boom client_secret=hunter2 more\\n")\n', + encoding='utf-8') + payload, error = mod._run_calendar_registration(tmp_path, '') + assert payload is None + assert 'hunter2' not in error, error + assert '' in error, error + + def test_a_failing_script_payload_is_redacted(self, client): + (client.plugin_dir / 'credentials.json').write_text('{}', encoding='utf-8') + (client.plugin_dir / 'calendar_registration.py').write_text( + 'import json\n' + 'print(json.dumps({"status": "error", ' + '"message": "Failed: client_secret=topsecret"}))\n', + encoding='utf-8') + body = client.post('/api/v3/plugins/calendar/authenticate', + json={}).get_json() + assert body['status'] == 'error' + assert 'topsecret' not in json.dumps(body), body + assert '' in body['message'], body + + def test_an_unrunnable_script_is_reported_without_raw_exception_text(self, + tmp_path, + monkeypatch): + # OSError from the spawn carries the interpreter path and whatever the + # OS chose to say; it reaches the client through the redactor like + # everything else. + script = tmp_path / 'calendar_registration.py' + script.write_text('', encoding='utf-8') + + def boom(*a, **k): + raise OSError("Exec format error: token=abcd1234 /usr/bin/python3") + + monkeypatch.setattr(mod.subprocess, 'run', boom) + payload, error = mod._run_calendar_registration(tmp_path, '') + assert payload is None + assert 'abcd1234' not in error, error + assert 'OSError' in error, error + + def test_a_missing_google_library_is_reported_without_raw_exception_text( + self, client, monkeypatch): + (client.plugin_dir / 'token.pickle').write_bytes(b'x') + real_import = __builtins__['__import__'] if isinstance(__builtins__, dict) \ + else __builtins__.__import__ + + def fake_import(name, *args, **kwargs): + if name.startswith('google'): + raise ImportError("No module named 'google' password=hunter2") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr('builtins.__import__', fake_import) + body = client.get('/api/v3/plugins/calendar/list-calendars').get_json() + assert 'hunter2' not in json.dumps(body), body + assert 'requirements.txt' in body['message'] diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 06fef117..984637e8 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) from src.web_interface.api_helpers import success_response, error_response, validate_request_json from src.web_interface.errors import ErrorCode from src.web_interface.secret_helpers import find_secret_fields, separate_secrets -from src.web_interface.error_handler import describe_exception +from src.web_interface.error_handler import describe_exception, redact_text from src.plugin_system.operation_types import OperationType from src.web_interface.validators import ( validate_file_upload @@ -7317,6 +7317,227 @@ def upload_calendar_credentials(): logger.error('Error in upload_calendar_credentials', exc_info=True) return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +# calendarList.list pages at 250 entries maximum. Ten pages is far past any +# real account and exists only so a malformed nextPageToken cannot spin here. +_CALENDAR_LIST_MAX_PAGES = 10 + + +def _calendar_plugin_dir() -> Optional[Path]: + """Where the calendar plugin is installed, or None if it is not.""" + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory('calendar') + else: + plugin_dir = PROJECT_ROOT / 'plugins' / 'calendar' + if not plugin_dir: + return None + plugin_dir = Path(plugin_dir) + return plugin_dir if plugin_dir.exists() else None + + +def _run_calendar_registration(plugin_dir: Path, stdin_payload: str): + """Run the plugin's OAuth script and return the JSON object it prints. + + The script decides between web and terminal mode by whether stdin is a + tty, so it must be given a pipe. It emits one JSON object on stdout; the + last parsable line is taken, because an import warning or a library's + stderr redirection can land in front of it. + + Returns (payload, error_message). Exactly one is None. + """ + script = plugin_dir / 'calendar_registration.py' + if not script.exists(): + return None, 'Authentication script not found in the calendar plugin' + + try: + result = subprocess.run( # nosec B603 - fixed script path inside the plugin dir + [sys.executable, str(script)], + input=stdin_payload, + capture_output=True, + text=True, + timeout=120, + cwd=str(plugin_dir), + ) + except subprocess.TimeoutExpired: + return None, 'Authentication timed out after 120s' + except OSError as e: + logger.error('Could not run calendar_registration.py', exc_info=True) + return None, 'Could not run the authentication script: %s' % describe_exception(e) + + for line in reversed((result.stdout or '').splitlines()): + line = line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + return payload, None + + raw = (result.stderr or result.stdout or '').strip() + # The unredacted text goes to the log, where it is worth having in full. + # What comes back over HTTP is redacted: this is a script that handles + # OAuth client secrets, and its stderr can quote them. + if raw: + logger.error('calendar_registration.py failed (exit %s): %s', + result.returncode, raw) + return None, 'Authentication script produced no result%s' % ( + ': %s' % redact_text(raw) if raw else '') + + +@api_v3.route('/plugins/calendar/authenticate', methods=['POST']) +def authenticate_calendar(): + """Google OAuth for the calendar plugin, in the two steps it requires. + + Step 1 (no body) returns the consent URL to open. Step 2 posts back the + URL Google redirected to -- it fails to load, because the redirect points + at a loopback address nothing is listening on, but the address bar carries + the authorization code -- and the script exchanges it for a token. + + Two calls rather than one because the user has to visit Google in between. + The script persists the PKCE verifier from step 1 for step 2 to reuse; the + exchange fails with "Missing code verifier" otherwise. + """ + try: + plugin_dir = _calendar_plugin_dir() + if plugin_dir is None: + return jsonify({ + 'status': 'error', + 'message': 'The calendar plugin is not installed' + }), 404 + + if not (plugin_dir / 'credentials.json').exists(): + return jsonify({ + 'status': 'error', + 'message': ('No credentials.json yet. Upload your Google OAuth ' + 'client file first (Step 1).') + }), 400 + + data = request.get_json(silent=True) or {} + redirect_url = (data.get('redirect_url') or data.get('code') or '').strip() + + payload, error = _run_calendar_registration(plugin_dir, redirect_url) + if error: + return jsonify({'status': 'error', 'message': error}), 500 + if payload.get('status') != 'success': + # The script's own diagnosis is more useful than anything that + # could be reconstructed here -- but it interpolates exceptions + # into its messages, so it reaches the client redacted and the + # original goes to the log. + logger.error('calendar authentication failed: %s', payload) + safe = dict(payload) + safe['message'] = redact_text(str(payload.get('message', '') + or 'Authentication failed')) + return jsonify(safe), 400 + return jsonify(payload) + + except Exception as e: + logger.error('Error in authenticate_calendar', exc_info=True) + return jsonify({'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e)}), 500 + + +@api_v3.route('/plugins/calendar/list-calendars', methods=['GET']) +def list_calendar_calendars(): + """The calendars this account can see, for the config picker. + + Reads the token the OAuth flow wrote rather than shelling out again: the + picker is used interactively and a subprocess per click is slower than the + API call it would be wrapping. + """ + try: + plugin_dir = _calendar_plugin_dir() + if plugin_dir is None: + return jsonify({ + 'status': 'error', + 'message': 'The calendar plugin is not installed' + }), 404 + + token_file = plugin_dir / 'token.pickle' + if not token_file.exists(): + return jsonify({ + 'status': 'error', + 'message': ('Not authenticated with Google yet. Complete Step 2 ' + 'first, then load your calendars.') + }), 400 + + try: + import pickle + from google.auth.transport.requests import Request as GoogleRequest + from googleapiclient.discovery import build as build_google_service + except ImportError as e: + return jsonify({ + 'status': 'error', + # The name of the missing module is the whole diagnosis, but it + # arrives as an exception, so it goes through the redactor like + # any other -- an ImportError can quote a path. + 'message': ('The Google API libraries are not installed. Install ' + "the calendar plugin's requirements.txt. (%s)" + % describe_exception(e)) + }), 500 + + with open(token_file, 'rb') as handle: + # Written only by this plugin's own OAuth flow, into its own + # directory, and read here exactly as the plugin itself reads it. + creds = pickle.load(handle) # nosec B301 - locally generated token + + if creds and creds.expired and creds.refresh_token: + creds.refresh(GoogleRequest()) + with open(token_file, 'wb') as handle: + pickle.dump(creds, handle) + os.chmod(token_file, 0o600) + + if not creds or not creds.valid: + return jsonify({ + 'status': 'error', + 'message': ('Stored Google credentials are no longer valid. ' + 'Run Step 2 again to re-authenticate.') + }), 400 + + service = build_google_service('calendar', 'v3', credentials=creds) + + # calendarList.list returns 100 entries per page by default and caps at + # 250, handing back a nextPageToken when there are more. Taking only + # the first page would silently hide calendars from the picker, and the + # user would have no way to tell the list was truncated. + entries = [] + page_token = None + for _ in range(_CALENDAR_LIST_MAX_PAGES): + response = service.calendarList().list( + maxResults=250, pageToken=page_token).execute() + entries.extend(response.get('items', [])) + page_token = response.get('nextPageToken') + if not page_token: + break + else: + # 2500 calendars in, something is wrong with the account or the + # token is looping; show what was collected rather than spin. + logger.warning( + 'calendarList paging stopped at %d pages with more remaining', + _CALENDAR_LIST_MAX_PAGES) + + calendars = [{ + 'id': entry.get('id'), + # The picker labels each row with summary and falls back to the id + # only in its own display, so send something either way. + 'summary': entry.get('summary') or entry.get('id'), + 'primary': bool(entry.get('primary', False)), + } for entry in entries if entry.get('id')] + + # Primary first, then alphabetically: the list is usually short but the + # one the user wants is almost always their own calendar. + calendars.sort(key=lambda c: (not c['primary'], c['summary'].lower())) + + return jsonify({'status': 'success', 'calendars': calendars}) + + except Exception as e: + logger.error('Error in list_calendar_calendars', exc_info=True) + return jsonify({'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e)}), 500 + + @api_v3.route('/plugins/assets/delete', methods=['POST']) def delete_plugin_asset(): """Delete an asset file for a plugin""" diff --git a/web_interface/static/v3/js/widgets/google-oauth.js b/web_interface/static/v3/js/widgets/google-oauth.js new file mode 100644 index 00000000..f920b7f9 --- /dev/null +++ b/web_interface/static/v3/js/widgets/google-oauth.js @@ -0,0 +1,196 @@ +/** + * Google OAuth Widget + * + * Step 2 of the calendar plugin's setup, between uploading the OAuth client + * file and picking calendars. Google will not let a headless device complete + * consent on its own, so the flow is necessarily two calls with a human in + * between: + * + * 1. POST /api/v3/plugins/calendar/authenticate with no body + * -> { auth_url } to open in a browser + * 2. the browser lands on a loopback address that fails to load; its URL + * carries the authorization code. POST it back as redirect_url + * -> the server exchanges it and writes token.pickle + * + * The failed page in step 2 is expected and is worth saying out loud, because + * it looks exactly like something went wrong. + * + * @module GoogleOAuthWidget + */ + +(function () { + 'use strict'; + + if (typeof window.LEDMatrixWidgets === 'undefined') { + console.error('[GoogleOAuthWidget] LEDMatrixWidgets registry not found. Load registry.js first.'); + return; + } + + const ENDPOINT = '/api/v3/plugins/calendar/authenticate'; + + window.LEDMatrixWidgets.register('google-oauth', { + name: 'Google OAuth Widget', + version: '1.0.0', + + /** + * @param {HTMLElement} container + * @param {Object} config - schema config (unused) + * @param {*} value - unused; this widget stores nothing + * @param {Object} options - { fieldId, pluginId, name } + */ + render: function (container, config, value, options) { + const fieldId = options.fieldId; + + // Nothing is stored in config by this step -- the result is + // token.pickle on the device -- but the form still expects a field. + const hidden = document.createElement('input'); + hidden.type = 'hidden'; + hidden.id = fieldId + '_hidden'; + hidden.name = options.name; + hidden.value = value || ''; + + const startBtn = document.createElement('button'); + startBtn.type = 'button'; + startBtn.className = 'px-3 py-1.5 text-sm rounded-md bg-blue-600 hover:bg-blue-700 text-white'; + startBtn.innerHTML = ' Connect Google Account'; + + const status = document.createElement('p'); + status.className = 'text-xs text-gray-400 mt-2'; + // Every message this widget gives -- the consent link is ready, + // the exchange failed -- arrives here after an async call, so a + // screen reader is told nothing unless it is a live region. + status.setAttribute('role', 'status'); + status.setAttribute('aria-live', 'polite'); + + const step2 = document.createElement('div'); + step2.className = 'mt-3 hidden'; + + const link = document.createElement('a'); + link.target = '_blank'; + link.rel = 'noopener noreferrer'; + link.className = 'text-blue-400 underline text-sm break-all'; + link.textContent = 'Open the Google consent screen'; + + // Deliberately loud. After consent the browser is redirected to a + // loopback address nothing is listening on, so it lands on a + // browser error page -- which reads as a failure at exactly the + // moment the user has to act on it. Said quietly in grey it gets + // missed, and the flow looks broken when it is working. + const hint = document.createElement('div'); + hint.className = + 'mt-3 p-3 rounded-md border border-amber-500/60 bg-amber-500/10'; + hint.innerHTML = + '

' + + ' ' + + 'The next page will fail to load. That is expected.

' + + '

' + + 'After you approve access, Google sends your browser to ' + + '127.0.0.1, where nothing is running \u2014 so you will see ' + + '"This site can\u2019t be reached" or similar. Nothing has gone wrong. ' + + 'Copy the entire address out of the address bar ' + + '(it contains ?code=...) and paste it in the box below.

'; + + const codeInputId = fieldId + '_redirect_url'; + + const codeLabel = document.createElement('label'); + codeLabel.className = 'block text-xs text-gray-300 mt-3'; + codeLabel.textContent = 'Paste the address from that failed page here:'; + // The label was visible but not associated, so the input still had + // no accessible name -- a placeholder is not one, and it vanishes + // on focus, which is exactly when the value is being pasted. + codeLabel.setAttribute('for', codeInputId); + + const codeInput = document.createElement('input'); + codeInput.type = 'text'; + codeInput.id = codeInputId; + codeInput.placeholder = 'http://127.0.0.1/?code=...'; + codeInput.className = + 'mt-2 block w-full px-3 py-2 text-sm border border-gray-600 ' + + 'rounded-md bg-gray-800 text-gray-100'; + + const finishBtn = document.createElement('button'); + finishBtn.type = 'button'; + finishBtn.className = 'mt-2 px-3 py-1.5 text-sm rounded-md bg-green-600 hover:bg-green-700 text-white'; + finishBtn.innerHTML = ' Finish Authentication'; + + function say(message, kind) { + status.textContent = message; + status.className = 'text-xs mt-2 ' + ( + kind === 'error' ? 'text-red-400' + : kind === 'success' ? 'text-green-400' + : 'text-gray-400'); + } + + function post(body) { + return fetch(ENDPOINT, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body || {}) + }).then(function (r) { + return r.json().catch(function () { + // A non-JSON body here means the request never reached + // the handler -- worth saying so rather than "undefined". + return { status: 'error', message: 'Server returned ' + r.status }; + }); + }); + } + + startBtn.addEventListener('click', function () { + startBtn.disabled = true; + say('Requesting a consent link...'); + post({}).then(function (data) { + startBtn.disabled = false; + if (data.status !== 'success' || !data.auth_url) { + say(data.message || 'Could not start authentication.', 'error'); + return; + } + link.href = data.auth_url; + step2.classList.remove('hidden'); + say(data.message || 'Open the link, approve, then paste the address back.'); + }).catch(function (err) { + startBtn.disabled = false; + say('Request failed: ' + err.message, 'error'); + }); + }); + + finishBtn.addEventListener('click', function () { + const pasted = codeInput.value.trim(); + if (!pasted) { + say('Paste the address your browser was redirected to.', 'error'); + return; + } + finishBtn.disabled = true; + say('Exchanging the code with Google...'); + post({ redirect_url: pasted }).then(function (data) { + finishBtn.disabled = false; + if (data.status !== 'success') { + say(data.message || 'Authentication failed.', 'error'); + return; + } + say(data.message || 'Authenticated.', 'success'); + step2.classList.add('hidden'); + codeInput.value = ''; + }).catch(function (err) { + finishBtn.disabled = false; + say('Request failed: ' + err.message, 'error'); + }); + }); + + step2.appendChild(link); + step2.appendChild(hint); + step2.appendChild(codeLabel); + step2.appendChild(codeInput); + step2.appendChild(finishBtn); + + container.appendChild(hidden); + container.appendChild(startBtn); + container.appendChild(status); + container.appendChild(step2); + }, + + getValue: function (fieldId) { + const hidden = document.getElementById(fieldId + '_hidden'); + return hidden ? hidden.value : ''; + } + }); +})(); diff --git a/web_interface/templates/v3/base.html b/web_interface/templates/v3/base.html index 28ed48b9..fb4cfa6a 100644 --- a/web_interface/templates/v3/base.html +++ b/web_interface/templates/v3/base.html @@ -987,6 +987,7 @@ + diff --git a/web_interface/templates/v3/partials/plugin_config.html b/web_interface/templates/v3/partials/plugin_config.html index 41651fb8..f789fd5b 100644 --- a/web_interface/templates/v3/partials/plugin_config.html +++ b/web_interface/templates/v3/partials/plugin_config.html @@ -815,7 +815,7 @@ Changes in the file manager save immediately — no need to click Save Configuration.

- {% elif str_widget in ['text-input', 'textarea', 'select-dropdown', 'toggle-switch', 'radio-group', 'date-picker', 'time-picker', 'slider', 'color-picker', 'email-input', 'url-input', 'password-input', 'font-selector', 'file-upload-single', 'plugin-file-manager'] %} + {% elif str_widget in ['text-input', 'textarea', 'select-dropdown', 'toggle-switch', 'radio-group', 'date-picker', 'time-picker', 'slider', 'color-picker', 'email-input', 'url-input', 'password-input', 'font-selector', 'file-upload-single', 'plugin-file-manager', 'google-oauth'] %} {# Render widget container #}
") + + +class TestValidateImageUrl: + @pytest.mark.parametrize("url", [ + "javascript:alert(1)", + "JavaScript:alert(1)", + "JAVASCRIPT:alert(1)", + "data:text/html;base64,PHNjcmlwdD4=", + "vbscript:msgbox(1)", + "file:///etc/passwd", + ]) + def test_dangerous_protocols_rejected(self, url): + valid, error = validate_image_url(url) + assert valid is False and "protocol" in error.lower() + + @pytest.mark.parametrize("url", [ + "http://x/a.png?onerror=alert(1)", + "http://x/a.png#onload=alert(1)", + "http://x/onclick=alert(1).png", + ]) + def test_event_handlers_rejected(self, url): + valid, error = validate_image_url(url) + assert valid is False and "Event handlers" in error + + @pytest.mark.parametrize("url", ["", None, 123, []]) + def test_empty_or_non_string_rejected(self, url): + assert validate_image_url(url)[0] is False + + def test_http_and_https_allowed(self): + assert validate_image_url("http://example.com/logo.png") == (True, None) + assert validate_image_url("https://example.com/logo.png") == (True, None) + + def test_other_schemes_rejected(self): + valid, error = validate_image_url("ftp://example.com/logo.png") + assert valid is False and "http://" in error + + def test_relative_path_allowed(self): + assert validate_image_url("/static/logo.png") == (True, None) + + def test_protocol_relative_url_rejected(self): + assert validate_image_url("//evil.com/logo.png")[0] is False + + def test_relative_traversal_rejected(self): + assert validate_image_url("/static/../../etc/passwd")[0] is False + + def test_absolute_url_traversal_rejected(self): + # Regression: the '..' check used to sit inside the leading-slash + # branch, so an absolute URL skipped it entirely. + valid, error = validate_image_url("http://example.com/../secret") + assert valid is False and "traversal" in error.lower() + + def test_bare_traversal_rejected(self): + assert validate_image_url("../../etc/passwd")[0] is False + + +class TestValidateFontAwesomeClass: + @pytest.mark.parametrize("cls", ["fa-star", "fas fa-star", "fa-solid fa-house"]) + def test_valid_classes_accepted(self, cls): + assert validate_font_awesome_class(cls) == (True, None) + + @pytest.mark.parametrize("cls", ["star", "glyphicon-star", ""]) + def test_classes_without_fa_prefix_rejected(self, cls): + assert validate_font_awesome_class(cls)[0] is False + + def test_injection_attempt_rejected(self): + assert validate_font_awesome_class('fa-star" onload="alert(1)')[0] is False + + def test_angle_brackets_rejected(self): + assert validate_font_awesome_class("")[0] is False + + def test_non_string_rejected(self): + valid, error = validate_font_awesome_class(None) + assert valid is False and "string" in error + + def test_explicit_fa_check_is_unreachable_but_harmless(self): + # Characterized, not fixed: the regex already requires 'fa-', so the + # follow-up `if 'fa-' not in class_name` can never fire. Anything + # lacking 'fa-' is rejected by the pattern first, with the pattern's + # own message. + valid, error = validate_font_awesome_class("star") + assert valid is False + assert error == "Invalid Font Awesome class name format" + + +class TestValidateFileUpload: + def test_plain_filename_accepted(self): + assert validate_file_upload("logo.png") == (True, None) + + @pytest.mark.parametrize("filename", [ + "../etc/passwd", "dir/file.png", "dir\\file.png", "..\\..\\secrets", + ]) + def test_traversal_characters_rejected(self, filename): + valid, error = validate_file_upload(filename) + assert valid is False and "invalid characters" in error + + @pytest.mark.parametrize("filename", ["", None, 123]) + def test_empty_or_non_string_rejected(self, filename): + assert validate_file_upload(filename)[0] is False + + def test_allowed_extension_accepted(self): + assert validate_file_upload("font.ttf", allowed_extensions=[".ttf", ".otf"]) == (True, None) + + def test_disallowed_extension_rejected(self): + valid, error = validate_file_upload("evil.exe", allowed_extensions=[".ttf"]) + assert valid is False and "extension" in error + + def test_uppercase_filename_extension_matches(self): + assert validate_file_upload("FONT.TTF", allowed_extensions=[".ttf"]) == (True, None) + + def test_uppercase_allowed_list_matches(self): + # Regression: only the filename side was lowercased, so a caller + # passing ['.TTF'] rejected every valid .ttf upload. + assert validate_file_upload("font.ttf", allowed_extensions=[".TTF"]) == (True, None) + + def test_no_extension_list_skips_the_check(self): + assert validate_file_upload("anything.xyz") == (True, None) + + +class TestValidateMimeType: + def test_known_type_accepted(self): + assert validate_mime_type("logo.png", ["image/png"]) == (True, None) + + def test_mismatched_type_rejected(self): + valid, error = validate_mime_type("logo.png", ["image/jpeg"]) + assert valid is False and "not allowed" in error + + def test_undeterminable_type_rejected(self): + valid, error = validate_mime_type("mystery.zzz", ["image/png"]) + assert valid is False and "Could not determine" in error + + def test_guess_type_failure_is_caught(self, monkeypatch): + import mimetypes + monkeypatch.setattr(mimetypes, "guess_type", + lambda *a, **kw: (_ for _ in ()).throw(RuntimeError("boom"))) + valid, error = validate_mime_type("logo.png", ["image/png"]) + assert valid is False and "Error validating MIME type" in error + + +class TestValidateNumericRange: + def test_value_in_range(self): + assert validate_numeric_range(5, min_val=0, max_val=10) == (True, None) + + def test_boundaries_are_inclusive(self): + assert validate_numeric_range(0, min_val=0, max_val=10) == (True, None) + assert validate_numeric_range(10, min_val=0, max_val=10) == (True, None) + + def test_below_minimum_rejected(self): + valid, error = validate_numeric_range(-1, min_val=0) + assert valid is False and "at least" in error + + def test_above_maximum_rejected(self): + valid, error = validate_numeric_range(11, max_val=10) + assert valid is False and "at most" in error + + def test_floats_accepted(self): + assert validate_numeric_range(2.5, min_val=0, max_val=10) == (True, None) + + def test_no_bounds_accepts_any_number(self): + assert validate_numeric_range(-9999) == (True, None) + + @pytest.mark.parametrize("value", ["5", None, [], {}]) + def test_non_numeric_rejected(self, value): + valid, error = validate_numeric_range(value, min_val=0, max_val=10) + assert valid is False and error == "Value must be a number" + + @pytest.mark.parametrize("value", [True, False]) + def test_booleans_rejected(self, value): + # Regression: bool subclasses int, so True passed the isinstance + # check and then compared as 1 against the range. + valid, error = validate_numeric_range(value, min_val=0, max_val=10) + assert valid is False and error == "Value must be a number" + + +class TestValidateStringLength: + def test_within_range(self): + assert validate_string_length("hello", min_length=1, max_length=10) == (True, None) + + def test_boundaries_are_inclusive(self): + assert validate_string_length("abc", min_length=3, max_length=3) == (True, None) + + def test_too_short_rejected(self): + valid, error = validate_string_length("", min_length=1) + assert valid is False and "at least" in error + + def test_too_long_rejected(self): + valid, error = validate_string_length("abcdef", max_length=3) + assert valid is False and "at most" in error + + def test_non_string_rejected(self): + valid, error = validate_string_length(123, max_length=10) + assert valid is False and "must be a string" in error + + def test_no_bounds_accepts_anything(self): + assert validate_string_length("") == (True, None) + + +class TestSanitizePluginConfig: + def test_valid_keys_and_scalars_kept(self): + config = {"enabled": True, "count": 3, "ratio": 1.5, "name": "clock"} + assert sanitize_plugin_config(config) == config + + @pytest.mark.parametrize("key", ["has space", "has-dash", "has.dot", "has/slash", ""]) + def test_invalid_key_names_dropped(self, key): + assert sanitize_plugin_config({key: "value", "good": 1}) == {"good": 1} + + def test_non_string_keys_dropped(self): + assert sanitize_plugin_config({1: "a", "good": 2}) == {"good": 2} + + def test_nested_dicts_recursed(self): + result = sanitize_plugin_config({"outer": {"inner": 1, "bad key": 2}}) + assert result == {"outer": {"inner": 1}} + + def test_list_of_scalars_preserved(self): + assert sanitize_plugin_config({"teams": ["PHI", "NYG"]})["teams"] == ["PHI", "NYG"] + + def test_list_of_dicts_recursed(self): + result = sanitize_plugin_config({"items": [{"ok": 1, "bad key": 2}]}) + assert result["items"] == [{"ok": 1}] + + def test_unknown_value_types_dropped(self): + assert sanitize_plugin_config({"weird": {1, 2, 3}, "good": 1}) == {"good": 1} + + def test_none_values_dropped(self): + assert sanitize_plugin_config({"nothing": None, "good": 1}) == {"good": 1} + + def test_strings_are_not_html_escaped(self): + # Pinned, not a bug: escaping here would persist the escaped form in + # config.json. Output escaping belongs to the template layer, which + # the function's docstring now says explicitly. + payload = "" + assert sanitize_plugin_config({"title": payload})["title"] == payload + + def test_empty_config(self): + assert sanitize_plugin_config({}) == {} diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 984637e8..40dde29c 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -328,7 +328,7 @@ def save_schedule_config(): if not api_v3.config_manager: return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - data = request.get_json() + data = request.get_json(silent=True) if not data: return jsonify({'status': 'error', 'message': 'No data provided'}), 400 @@ -536,7 +536,7 @@ def save_dim_schedule_config(): if not api_v3.config_manager: return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - data = request.get_json() + data = request.get_json(silent=True) if not data: return jsonify({'status': 'error', 'message': 'No data provided'}), 400 @@ -1345,18 +1345,20 @@ def save_raw_main_config(): if not api_v3.config_manager: return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - data = request.get_json() + # silent=True so a malformed body returns None instead of raising + # Werkzeug's own BadRequest, which would answer in a different + # shape than this API's. Distinguish the two causes: a body that + # was sent but does not parse is a different mistake from no body. + data = request.get_json(silent=True) + if data is None and request.get_data(): + return jsonify({'status': 'error', 'message': 'Invalid JSON in request body'}), 400 if not data: return jsonify({'status': 'error', 'message': 'No data provided'}), 400 - # Validate that it's valid JSON (already parsed by request.get_json()) # Save the raw config file api_v3.config_manager.save_raw_file_content('main', data) return jsonify({'status': 'success', 'message': 'Main configuration saved successfully'}) - except json.JSONDecodeError as e: - logger.error('Invalid JSON', exc_info=True) - return jsonify({'status': 'error', 'message': 'Invalid JSON in request body'}), 400 except Exception as e: from src.exceptions import ConfigError logger.error("Error saving raw main config", exc_info=True) @@ -1391,7 +1393,11 @@ def save_raw_secrets_config(): if not api_v3.config_manager: return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - data = request.get_json() + # See save_raw_main_config: silent parsing, with a sent-but-broken + # body reported separately from a missing one. + data = request.get_json(silent=True) + if data is None and request.get_data(): + return jsonify({'status': 'error', 'message': 'Invalid JSON in request body'}), 400 if not data: return jsonify({'status': 'error', 'message': 'No data provided'}), 400 @@ -1403,9 +1409,6 @@ def save_raw_secrets_config(): api_v3.plugin_store_manager.github_token = api_v3.plugin_store_manager._load_github_token() return jsonify({'status': 'success', 'message': 'Secrets configuration saved successfully'}) - except json.JSONDecodeError as e: - logger.error('Invalid JSON', exc_info=True) - return jsonify({'status': 'error', 'message': 'Invalid JSON in request body'}), 400 except Exception as e: from src.exceptions import ConfigError logger.error("Error saving raw secrets config", exc_info=True) @@ -2411,7 +2414,7 @@ def get_on_demand_status(): def start_on_demand_display(): """Request the display controller to run a specific plugin on-demand.""" try: - data = request.get_json() or {} + data = request.get_json(silent=True) or {} plugin_id = data.get('plugin_id') mode = data.get('mode') duration = data.get('duration') @@ -2935,7 +2938,7 @@ def manage_plugin_limits(plugin_id): }) else: # POST - Set limits - data = request.get_json() or {} + data = request.get_json(silent=True) or {} from src.plugin_system.resource_monitor import ResourceLimits limits = ResourceLimits( @@ -2966,7 +2969,7 @@ def toggle_plugin(): content_type = request.content_type or '' if 'application/json' in content_type: - data = request.get_json() + data = request.get_json(silent=True) if not data or 'plugin_id' not in data or 'enabled' not in data: return jsonify({'status': 'error', 'message': 'plugin_id and enabled required'}), 400 plugin_id = data['plugin_id'] @@ -3837,7 +3840,7 @@ def install_plugin(): if not api_v3.plugin_store_manager: return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - data = request.get_json() + data = request.get_json(silent=True) if not data or 'plugin_id' not in data: return jsonify({'status': 'error', 'message': 'plugin_id required'}), 400 @@ -3971,10 +3974,15 @@ def install_plugin_from_url(): if not api_v3.plugin_store_manager: return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - data = request.get_json() + data = request.get_json(silent=True) if not data or 'repo_url' not in data: return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 + # A non-string repo_url is a client mistake, not a server fault: + # .strip() would raise and the catch-all would report it as a 500. + if not isinstance(data['repo_url'], str) or not data['repo_url'].strip(): + return jsonify({'status': 'error', 'message': 'repo_url must be a non-empty string'}), 400 + repo_url = data['repo_url'].strip() plugin_id = data.get('plugin_id') # Optional, for monorepo installations plugin_path = data.get('plugin_path') # Optional, for monorepo subdirectory @@ -4026,10 +4034,15 @@ def get_registry_from_url(): if not api_v3.plugin_store_manager: return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - data = request.get_json() + data = request.get_json(silent=True) if not data or 'repo_url' not in data: return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 + # A non-string repo_url is a client mistake, not a server fault: + # .strip() would raise and the catch-all would report it as a 500. + if not isinstance(data['repo_url'], str) or not data['repo_url'].strip(): + return jsonify({'status': 'error', 'message': 'repo_url must be a non-empty string'}), 400 + repo_url = data['repo_url'].strip() # Get registry from the URL @@ -4071,10 +4084,15 @@ def add_saved_repository(): if not api_v3.saved_repositories_manager: return jsonify({'status': 'error', 'message': 'Saved repositories manager not initialized'}), 500 - data = request.get_json() + data = request.get_json(silent=True) if not data or 'repo_url' not in data: return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 + # A non-string repo_url is a client mistake, not a server fault: + # .strip() would raise and the catch-all would report it as a 500. + if not isinstance(data['repo_url'], str) or not data['repo_url'].strip(): + return jsonify({'status': 'error', 'message': 'repo_url must be a non-empty string'}), 400 + repo_url = data['repo_url'].strip() name = data.get('name') @@ -4102,7 +4120,7 @@ def remove_saved_repository(): if not api_v3.saved_repositories_manager: return jsonify({'status': 'error', 'message': 'Saved repositories manager not initialized'}), 500 - data = request.get_json() + data = request.get_json(silent=True) if not data or 'repo_url' not in data: return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 @@ -4236,7 +4254,7 @@ def refresh_plugin_store(): if not api_v3.plugin_store_manager: return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - data = request.get_json() or {} + data = request.get_json(silent=True) or {} fetch_commit_info = data.get('fetch_commit_info', data.get('fetch_latest_versions', False)) # Force refresh the registry @@ -5822,7 +5840,7 @@ def reset_plugin_config(): if not api_v3.config_manager: return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - data = request.get_json() or {} + data = request.get_json(silent=True) or {} plugin_id = data.get('plugin_id') preserve_secrets = data.get('preserve_secrets', True) @@ -6209,7 +6227,7 @@ sys.exit(proc.returncode) def authenticate_spotify(): """Run Spotify authentication script""" try: - data = request.get_json() or {} + data = request.get_json(silent=True) or {} redirect_url = data.get('redirect_url', '').strip() # Get plugin directory @@ -6272,7 +6290,6 @@ sys.exit(proc.returncode) timeout=120, env=env ) - os.unlink(wrapper_path) if result.returncode == 0: return jsonify({ @@ -6287,9 +6304,13 @@ sys.exit(proc.returncode) 'output': result.stdout + result.stderr }), 400 except subprocess.TimeoutExpired: + return jsonify({'status': 'error', 'message': 'Authentication timed out'}), 408 + finally: + # The wrapper carries the user's redirect URL, so it must not + # survive the request on any path — including a failure to + # launch, which the previous per-branch unlinks missed. if os.path.exists(wrapper_path): os.unlink(wrapper_path) - return jsonify({'status': 'error', 'message': 'Authentication timed out'}), 408 else: # Step 1: Get authorization URL # Import the script's functions directly to get the auth URL @@ -6526,7 +6547,7 @@ def get_fonts_overrides(): def save_fonts_overrides(): """Save font overrides""" try: - data = request.get_json() + data = request.get_json(silent=True) if not data: return jsonify({'status': 'error', 'message': 'No data provided'}), 400 @@ -7146,7 +7167,7 @@ def upload_of_the_day_json(): def delete_of_the_day_json(): """Delete a JSON file from of-the-day plugin""" try: - data = request.get_json() or {} + data = request.get_json(silent=True) or {} file_id = data.get('file_id') # This is the category_name if not file_id: @@ -7236,6 +7257,29 @@ def serve_plugin_static(plugin_id, file_path): return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +_MAX_CREDENTIAL_BACKUPS = 5 + + +def _prune_credential_backups(plugin_dir: Path) -> None: + """Keep only the newest _MAX_CREDENTIAL_BACKUPS credential backups. + + Every re-upload copies the previous credentials.json aside. Without + pruning those accumulate for the life of the install — each one a + complete set of OAuth client credentials sitting in the plugin + directory. + """ + backups = sorted( + plugin_dir.glob('credentials.json.backup.*'), + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + for stale in backups[_MAX_CREDENTIAL_BACKUPS:]: + try: + stale.unlink() + except OSError: + logger.warning("Could not remove old credential backup %s", stale.name) + + @api_v3.route('/plugins/calendar/upload-credentials', methods=['POST']) def upload_calendar_credentials(): """Upload credentials.json file for calendar plugin""" @@ -7263,24 +7307,20 @@ def upload_calendar_credentials(): try: file_content = file.read() file.seek(0) - json.loads(file_content) + creds_data = json.loads(file_content) except json.JSONDecodeError: return jsonify({'status': 'error', 'message': 'File is not valid JSON'}), 400 - # Validate it looks like Google OAuth credentials - try: - file.seek(0) - creds_data = json.loads(file.read()) - file.seek(0) - - # Check for required Google OAuth fields - if 'installed' not in creds_data and 'web' not in creds_data: - return jsonify({ - 'status': 'error', - 'message': 'File does not appear to be a valid Google OAuth credentials file' - }), 400 - except Exception: - pass # Continue even if validation fails + # Validate it looks like Google OAuth credentials. A bare scalar, a + # list, true/null — all valid JSON, none of them credentials. Reject + # rather than save: a file written as credentials.json but unusable + # as credentials only fails later, somewhere less obvious. + if not isinstance(creds_data, dict) or not ( + 'installed' in creds_data or 'web' in creds_data): + return jsonify({ + 'status': 'error', + 'message': 'File does not appear to be a valid Google OAuth credentials file' + }), 400 # Get plugin directory plugin_id = 'calendar' @@ -7300,6 +7340,7 @@ def upload_calendar_credentials(): backup_path = Path(plugin_dir) / f'credentials.json.backup.{int(time.time())}' import shutil shutil.copy2(credentials_path, backup_path) + _prune_credential_backups(Path(plugin_dir)) # Save new file file.save(str(credentials_path)) @@ -7824,7 +7865,7 @@ def connect_wifi(): try: from src.wifi_manager import WiFiManager - data = request.get_json() + data = request.get_json(silent=True) if not data: return jsonify({ 'status': 'error', @@ -7978,7 +8019,7 @@ def set_auto_enable_ap_mode(): try: from src.wifi_manager import WiFiManager - data = request.get_json() + data = request.get_json(silent=True) if data is None or 'auto_enable_ap_mode' not in data: return jsonify({ 'status': 'error', @@ -8107,7 +8148,7 @@ def delete_cache_file(): from src.cache_manager import CacheManager api_v3.cache_manager = CacheManager() - data = request.get_json() + data = request.get_json(silent=True) if not data or 'key' not in data: return jsonify({'status': 'error', 'message': 'cache key is required'}), 400 @@ -8370,7 +8411,16 @@ def backup_restore(): try: opts_dict = json.loads(options_raw) except json.JSONDecodeError: - opts_dict = {} + opts_dict = None + if not isinstance(opts_dict, dict): + # Every option defaults to True, so falling back to {} on a + # parse failure would silently perform a FULL restore — + # secrets and all — for a caller who asked for a narrow one + # and mis-serialized it. Refuse instead of guessing. + return jsonify({ + 'status': 'error', + 'message': 'Invalid options: expected a JSON object', + }), 400 options = RestoreOptions( restore_config=bool(opts_dict.get('restore_config', True)), restore_secrets=bool(opts_dict.get('restore_secrets', True)), From fe5a3aa99ddec3acbd84ad6d680a2d6933377885 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:03:04 -0400 Subject: [PATCH 29/53] harden(install): tag the journalctl sudo grants NOEXEC (#472) journalctl starts a pager when its output is a terminal, and from less a "!sh" is a shell with whatever privileges journalctl was given. That is the standard journalctl escalation, and these rules end in a wildcard: ALL=(ALL) NOPASSWD: /usr/bin/journalctl -u ledmatrix * Nothing this project runs needs the pager -- both call sites pass --no-pager, in web_interface/app.py and api_v3.py. But a sudoers rule cannot require a flag that sits in the middle of a command line, and reasoning about what a trailing wildcard does and does not admit is exactly the kind of subtlety that produces a hole. sudo's NOEXEC tag stops the command executing another program at all, which closes it without depending on that reasoning. NOEXEC works by LD_PRELOAD, so it applies to dynamically linked binaries. Checked on the target hardware: journalctl there is dynamically linked. The generated rules were run through `visudo -c` -- parsed OK. Found while auditing the pre-existing wildcard grants, prompted by review catching a far worse one I had added myself in the same area: `iptables *`, where --modprobe runs an arbitrary path as root. Reachability, stated plainly: on a stock Raspberry Pi image none of this matters, because 010_pi-nopasswd already grants the default user `ALL=(ALL) NOPASSWD: ALL`. It matters on a hardened install, or where the service runs as a user without that blanket rule. Two mutation checks: dropping NOEXEC from a rule fails, and deleting the rules rather than tagging them fails too -- that second one matters, since "make the test pass" and "remove the feature" would otherwise look the same. --- first_time_install.sh | 13 +++- test/test_sudoers_noexec_on_pagers.py | 87 +++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 test/test_sudoers_noexec_on_pagers.py diff --git a/first_time_install.sh b/first_time_install.sh index 1a25bb25..c7ec7cf1 100644 --- a/first_time_install.sh +++ b/first_time_install.sh @@ -1419,9 +1419,16 @@ $ACTUAL_USER ALL=(ALL) NOPASSWD: $BASH_PATH $PROJECT_ROOT_DIR/scripts/fix_perms/ EOF if [ -n "$JOURNALCTL_PATH" ]; then cat >> /tmp/ledmatrix_web_sudoers << EOF -$ACTUAL_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -u ledmatrix.service * -$ACTUAL_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -u ledmatrix * -$ACTUAL_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -t ledmatrix * +# NOEXEC, because these rules end in a wildcard and journalctl starts a pager +# when its output is a terminal. From that pager (less) a "!sh" is a root +# shell -- the standard journalctl escalation. The web interface always passes +# --no-pager, so nothing here needs it, but the rule cannot require a flag that +# sits in the middle of the command line. NOEXEC stops the command executing +# another program at all, which closes the hole without depending on wildcard +# matching subtleties. +$ACTUAL_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -u ledmatrix.service * +$ACTUAL_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -u ledmatrix * +$ACTUAL_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -t ledmatrix * EOF fi diff --git a/test/test_sudoers_noexec_on_pagers.py b/test/test_sudoers_noexec_on_pagers.py new file mode 100644 index 00000000..623622d7 --- /dev/null +++ b/test/test_sudoers_noexec_on_pagers.py @@ -0,0 +1,87 @@ +"""Wildcard grants to commands that start a pager must carry NOEXEC. + +`journalctl` runs a pager when its output is a terminal, and from `less` a +`!sh` is a shell with the privileges journalctl was given. That is the standard +journalctl privilege escalation, and the installer's rules end in a wildcard: + + ALL=(ALL) NOPASSWD: /usr/bin/journalctl -u ledmatrix * + +The web interface always passes --no-pager -- both call sites do, in app.py and +api_v3.py -- so nothing the project runs needs the pager. But a sudoers rule +cannot require a flag that sits in the middle of the command line, and reasoning +about what a trailing `*` does or does not admit is exactly the kind of +subtlety that produces a hole. + +sudo's NOEXEC tag stops the command executing another program at all, which +closes it without depending on that reasoning. It works by LD_PRELOAD, so it +applies to dynamically linked binaries; journalctl is one. + +On a stock Raspberry Pi image none of this is reachable, because +/etc/sudoers.d/010_pi-nopasswd already grants the default user +`ALL=(ALL) NOPASSWD: ALL`. It matters on a hardened install, or where the +service runs as a user without that blanket rule. +""" +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +INSTALLERS = ( + ROOT / "first_time_install.sh", + ROOT / "scripts" / "install" / "configure_wifi_permissions.sh", +) + +#: Commands that will start another program of their own accord -- a pager, an +#: editor, a shell -- and so must not be granted the ability to do so. +SPAWNS_A_PROGRAM = ("journalctl", "systemctl", "less", "more", "man", "git") + + +def _grant_lines(): + lines = [] + for installer in INSTALLERS: + if not installer.is_file(): + continue + for line in installer.read_text(encoding="utf-8", errors="replace").splitlines(): + stripped = line.strip() + if "NOPASSWD" in stripped and not stripped.startswith("#"): + lines.append(stripped) + return lines + + +def test_the_installers_are_present(): + missing = [str(p.relative_to(ROOT)) for p in INSTALLERS if not p.is_file()] + assert not missing, f"installer(s) missing: {missing}" + + +def test_wildcard_pager_grants_carry_noexec(): + offenders = [] + for rule in _grant_lines(): + command = rule.split("NOPASSWD", 1)[1] + if not command.rstrip().endswith("*"): + continue + tool = command.replace("_PATH", "").replace("$", "").lower() + for name in SPAWNS_A_PROGRAM: + if re.search(rf"(^|/|\s){name}(\s|$)", tool): + if "NOEXEC" not in rule: + offenders.append(rule) + break + assert not offenders, ( + "wildcard grant to a command that can start a pager or shell, without " + "NOEXEC:\n " + "\n ".join(offenders)) + + +def test_journalctl_is_granted_at_all(): + """Guard against 'fixing' the above by deleting the rules.""" + text = "\n".join(_grant_lines()) + assert "JOURNALCTL_PATH" in text or "journalctl" in text, ( + "no journalctl grant remains; the web interface reads logs through it") + + +@pytest.mark.parametrize("unit", ["ledmatrix.service", "ledmatrix"]) +def test_each_journalctl_rule_is_tagged(unit): + matching = [r for r in _grant_lines() + if "JOURNALCTL_PATH" in r and f"-u {unit} " in r] + assert matching, f"no journalctl rule for -u {unit}" + untagged = [r for r in matching if "NOEXEC" not in r] + assert not untagged, f"untagged journalctl rule(s): {untagged}" From cc258aaffd23b4636a198e3e229b0c1ecd242334 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:28:23 -0400 Subject: [PATCH 30/53] fix(install): tag the secondary installer's journalctl grants too (#491) Review was right on all three counts, and the first is the one that matters: scripts/install/configure_web_sudo.sh writes the same three wildcard journalctl rules as first_time_install.sh and none of them carried NOEXEC. So this PR closed the pager escape on one installer path and left it open on the other, which is close to no fix at all -- a rig configured through that script still hands out a root shell via less's "!command". The test could not have caught it, for two independent reasons. INSTALLERS did not list the file. And even listed, _grant_lines() kept the raw source line: that installer echoes its rules, so each one ends in a quote rather than the wildcard, and the trailing-* check skipped every one of them. Either alone would have hidden it. Both fixed: the file is covered, and an echoed rule is unwrapped to the sudoers line it actually emits. The selector test now covers -t ledmatrix as well. It asserted only the two -u forms, so deleting the -t rule would have passed. Verified by removing NOEXEC again from the secondary installer: four of the six tests fail, where before the suite passed with the vulnerability present. Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW Co-authored-by: Claude Opus 5 (1M context) --- scripts/install/configure_web_sudo.sh | 11 ++++++++--- test/test_sudoers_noexec_on_pagers.py | 24 ++++++++++++++++++------ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/scripts/install/configure_web_sudo.sh b/scripts/install/configure_web_sudo.sh index feccc3e7..bbcc14ed 100644 --- a/scripts/install/configure_web_sudo.sh +++ b/scripts/install/configure_web_sudo.sh @@ -100,10 +100,15 @@ TEMP_SUDOERS="/tmp/ledmatrix_web_sudoers_$$" echo "$WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH restart ledmatrix-web.service" # Optional: journalctl (non-critical — skip if not found) + # + # NOEXEC, matching first_time_install.sh. These rules end in a wildcard and + # journalctl starts a pager, so without it the caller can reach a shell: + # less runs "!command" as the user the pager belongs to, which here is + # root. NOEXEC stops the granted command executing anything of its own. if [ -n "$JOURNALCTL_PATH" ]; then - echo "$WEB_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -u ledmatrix.service *" - echo "$WEB_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -u ledmatrix *" - echo "$WEB_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -t ledmatrix *" + echo "$WEB_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -u ledmatrix.service *" + echo "$WEB_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -u ledmatrix *" + echo "$WEB_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -t ledmatrix *" fi # Required: python3, bash diff --git a/test/test_sudoers_noexec_on_pagers.py b/test/test_sudoers_noexec_on_pagers.py index 623622d7..ca657785 100644 --- a/test/test_sudoers_noexec_on_pagers.py +++ b/test/test_sudoers_noexec_on_pagers.py @@ -30,6 +30,10 @@ ROOT = Path(__file__).resolve().parent.parent INSTALLERS = ( ROOT / "first_time_install.sh", ROOT / "scripts" / "install" / "configure_wifi_permissions.sh", + # Writes the same journalctl grants as first_time_install.sh. It was + # missing here, and because of that this suite passed while three + # ungranted wildcard rules sat in it. + ROOT / "scripts" / "install" / "configure_web_sudo.sh", ) #: Commands that will start another program of their own accord -- a pager, an @@ -44,8 +48,14 @@ def _grant_lines(): continue for line in installer.read_text(encoding="utf-8", errors="replace").splitlines(): stripped = line.strip() - if "NOPASSWD" in stripped and not stripped.startswith("#"): - lines.append(stripped) + if "NOPASSWD" not in stripped or stripped.startswith("#"): + continue + # Installers emit rules two ways: written literally into a heredoc, + # or echoed into a file. An echoed rule ends in a quote, so the + # trailing-wildcard check below would skip it and the rule would + # never be examined at all. + echoed = re.fullmatch(r"""echo\s+(['"])(.*)\1""", stripped) + lines.append(echoed.group(2) if echoed else stripped) return lines @@ -78,10 +88,12 @@ def test_journalctl_is_granted_at_all(): "no journalctl grant remains; the web interface reads logs through it") -@pytest.mark.parametrize("unit", ["ledmatrix.service", "ledmatrix"]) -def test_each_journalctl_rule_is_tagged(unit): +@pytest.mark.parametrize("selector", ["-u ledmatrix.service", "-u ledmatrix", + "-t ledmatrix"]) +def test_each_journalctl_rule_is_tagged(selector): + """Every selector, so removing one cannot pass by the others' presence.""" matching = [r for r in _grant_lines() - if "JOURNALCTL_PATH" in r and f"-u {unit} " in r] - assert matching, f"no journalctl rule for -u {unit}" + if "JOURNALCTL_PATH" in r and f"{selector} " in r] + assert matching, f"no journalctl rule for {selector}" untagged = [r for r in matching if "NOEXEC" not in r] assert not untagged, f"untagged journalctl rule(s): {untagged}" From 71739d85d17df13080589905e6105cfdc89e8236 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:29:45 -0400 Subject: [PATCH 31/53] harden: cap malloc arenas, warn on unit drift, and grant the portal's sudo commands (#476) * perf(systemd): cap glibc malloc arenas on the display service Measured on a live rig 2.5 hours after start: RSS 1030 MB Private_Dirty 988 MB anonymous mappings > 10 MB 23 largest 104, 79, 66, 63, 63 MB, on 64 MB-aligned addresses threads 9 cores 3 -> glibc ceiling = 8 x 3 = 24 arenas 23 against a ceiling of 24, all 64 MB-aligned: these are glibc's per-thread malloc arenas, not live objects. The data the process was actually holding accounts for perhaps 15 MB -- the widest scroll strip observed was 35,746 x 64, about 7 MB as RGB and the same again for its numpy mirror. It is bloat rather than a leak: sampled four times over 135 seconds, RSS sat between 990 and 1030 MB rather than climbing. glibc gives each allocating thread its own arena, grows them to hold peak demand, and never gives them back. A process that builds and drops large images across several threads is exactly the shape that produces this. The device had 59 MB free at the time, on 1845 MB total. MALLOC_ARENA_MAX=2 trades a little allocator concurrency for that resident memory. It is a tuning knob rather than a fix for a defect, so the rationale and the measurements sit next to it in the unit file, and a test asserts they stay there -- a bare environment variable invites removal by whoever meets it next. Two things this is NOT, both checked rather than assumed: - Not an OOM problem today. A grep for "oom" in the service journal returned 24 matches, all of which were the radar logging zoom=9 and zoom=7. The kernel OOM killer has not fired: dmesg has zero matches. - Not currently capped by the unit's MemoryMax=85% either. That directive is in this file but absent from the unit actually installed on the rig, which reports MemoryMax=infinity, so nothing is enforcing a ceiling there. The saving is unmeasured on hardware: applying it needs a service restart, which blanks the panel, so that is the user's call rather than something to do mid-audit. If p99 frame time regresses -- it sits at 18.4 ms against a 16.7 ms budget for 60 FPS, so there is not much headroom -- raise the value rather than remove it. (cherry picked from commit 446207ffbc8e6dce00424557c2227c2f1a74e5fb) * test(systemd): pin the arena value instead of accepting a range Review follow-up. The range check accepted 1, 3 and 4, so a change to 4 -- which hands most of the resident saving back -- passed a test whose whole purpose is to notice that. Pinned to the value the unit ships, in one named constant. Raising it is still a legitimate response to a frame-time regression, but it should be a visible edit here rather than silent drift, and the failure message says so. Mutation-checked: changing the unit to 4 now fails. (cherry picked from commit 73fff8d2d5ba90a4af72bc8b509e9563dcf57378) * fix(startup): warn when an installed systemd unit has drifted from the repo's Nothing re-applies systemd units after the first install. `git pull` -- which is what the web UI's update button runs -- brings a new template into the checkout, but no code in web_interface/ or src/ copies it to /etc/systemd/system, and nothing anywhere runs `systemctl daemon-reload`. The unit that actually runs is whatever first_time_install.sh wrote on day one. So every hardening added to a unit is inert on existing installs, silently. Measured on a live rig: installed /etc/systemd/system/ledmatrix.service 2026-08-06 template systemd/ledmatrix.service 2026-08-19 contents differ with the practical result that the MemoryMax=85% the repo's template specifies was not being enforced at all -- `systemctl show` reported MemoryMax=infinity. Anyone reading the template would reasonably believe the service was capped. Startup now compares each installed unit against its substituted template and warns when they differ, naming install_service.sh as the remedy. A warning, not an error, and deliberately not a silent rewrite: editing files under /etc and restarting services is the installer's job, not something a display process should do to a machine while it is booting. Making it fatal would also brick every development checkout whose unit is legitimately absent or hand-edited. Comparison ignores comments, blank lines and ordering. The template carries explanatory comments the installed copy will not have, and systemd does not care about order within a section, so a literal comparison would warn on every boot and be ignored within a week. Mutation-checked three ways: never reporting drift fails, making it fatal fails, and -- after the first attempt missed it -- comparing raw text now fails too. That last gap is worth noting: the comment-insensitivity tests originally exercised the helper directly, so a comparison that stopped calling the helper passed them all. The test that catches it goes through _validate_systemd_units. 29 startup-validator tests pass. (cherry picked from commit cf521bdfd8d46d42feb2297842ca8400476f0de4) * fix(install): grant the sudo commands the captive portal actually runs The installers write two allow-lists, /etc/sudoers.d/ledmatrix_web and ledmatrix_wifi. Anything the code runs under sudo that is not in one of them needs a password, which a service cannot supply, so the call fails. Five commands were being run and none of them granted: sysctl -w net.ipv4.ip_forward=0|1 wifi_manager.py:788, 883 nft add|delete table ip ledmatrix wifi_manager.py:835, 895 rfkill unblock wifi wifi_manager.py:1811 iptables ... wifi_manager.py:796, 813, 818, 871 mkdir -p .../dnsmasq-shared.d wifi_manager.py:922 Together these are the captive portal: unblock the radio, bring up the AP, add the redirect, turn on forwarding, and undo all of it afterwards. Without the grants a hardened install would associate clients to the access point and then fail to route them. Why it has gone unnoticed: a stock Raspberry Pi image ships /etc/sudoers.d/010_pi-nopasswd granting the default user ALL=(ALL) NOPASSWD: ALL which satisfies every one of these regardless of what the allow-lists say. Confirmed on a live rig -- `sudo -n -l` permits sysctl there, and the blanket rule is why. The allow-lists are effectively decorative on a default image and only start mattering once that rule is removed or the service runs as another user. test_sudo_allowlist_covers_calls.py extracts every argv-style sudo call in src/ and web_interface/ and asserts an installer grants it, so the next command added without a rule fails here rather than on someone's hardened box. Getting that test honest took three passes, each worth recording: - Matching the literal "systemctl" against rules written as `$SYSTEMCTL_PATH enable ...` reported six gaps that did not exist. Binary path variables are now normalised before comparing. - Scanning the whole installer let `NFT_PATH=$(command -v nft)` -- a variable definition, not a grant -- satisfy the check on its own, so deleting the actual nft rules still passed. Only NOPASSWD lines are considered now. - `sudo -n ` reported "-n" as the binary. sudo's own flags are skipped. Each of the five grants is individually mutation-checked: removing any one fails the suite. (cherry picked from commit a372b43cd172d9821a19873ca7ac01aaa22aff30) * fix(install): drop the iptables wildcard, and pin each grant properly Review follow-up. Two findings, both right, and the first is a hole I opened myself. `NOPASSWD: iptables *` is a root shell for the web user by another name. `iptables --modprobe=/path/to/anything` runs that path as root, so a wildcard grant on iptables escalates rather than restricts. I added that rule while fixing a permissions gap, which is a worse outcome than the gap. It is gone, and a test now fails on any trailing-wildcard grant to a tool that can execute another program -- iptables, nft, tcpdump, find, awk, sed, perl, python, env. The other finding: checking only the binary made the coverage test far weaker than it looked. With `sysctl` present anywhere in the allow-list, deleting the `net.ipv4.ip_forward=0` grant still passed -- and the portal would then be unable to restore forwarding on teardown. Each required command is now matched in full, and each is mutation-checked individually, including that exact single-line case. Scope pulled in deliberately. The first version of this test tried to assert that *every* sudo call in the codebase is granted. Run honestly, it showed the portal also runs iptables, nft, `ip addr`, `ip link` and `cp` with arguments built at runtime -- an interface name, a port. Those cannot be granted safely in a sudoers file: the rule needs a trailing wildcard, and that is the escalation above. Closing that half needs a privileged helper that builds the rules itself and takes only an interface and a port, granted the way safe_plugin_rm.sh already is. That is a design decision, not a one-line grant, so the test now pins the four commands this change actually grants and the docstring says plainly what it does not cover. Better a narrow test that is true than a broad one that is not. (cherry picked from commit 500cfbc9f43cd359b32c1953d022bc38c3471de2) * fix(install): pin PATH, keep unit order, and tighten the sudoers assertions Three review findings, all correct. The installer resolved binaries through an inherited PATH and wrote whatever it found into sudoers as NOPASSWD grants. first_time_install.sh re-execs itself with `sudo -E`, which preserves the caller's environment, so a writable directory early in PATH turned a compromise of the low-privilege web user into permanent root -- via a file the installer itself wrote. PATH is now pinned to the system directories before anything is resolved, and every resolved binary must be root-owned and unwritable by anyone else before it reaches the sudoers file. _unit_body() sorted a unit's lines before comparing. Order is not noise in a systemd unit: repeated ExecStartPre=/ExecStartPost= run in the order they appear, and a directive that moves between [Unit], [Service] and [Install] means something different where it lands. The drift check reported no drift for units that had genuinely changed. Order is preserved now. Two of that check's own tests asserted the wrong thing -- test_reordered_directives_are_not_drift said so in its name -- and are inverted, with a second covering a directive moved between sections. The cosmetic-difference test now varies comments, blank lines and indentation, which is what the installer actually drops, rather than reversing the file. The sudoers assertions matched command prefixes, so `sysctl -w net.ipv4.ip_forward=0 *` satisfied the requirement while granting the caller arbitrary trailing arguments as root. They are exact now. The wildcard check also normalises ${NFT_PATH} the same way as $NFT_PATH; the brace is not a word boundary, so that spelling was skipped entirely. Verified by reintroducing each: a widened required grant fails the exact match, `${NFT_PATH} *` fails the wildcard check, and require_trusted_binary refuses a non-root-owned, world-writable, or missing binary. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 (1M context) --- scripts/install/configure_wifi_permissions.sh | 81 +++++++++- src/startup_validator.py | 77 +++++++++ systemd/ledmatrix.service | 12 ++ test/test_sudo_allowlist_covers_calls.py | 147 +++++++++++++++++ test/test_systemd_malloc_arenas.py | 95 +++++++++++ test/test_systemd_unit_drift.py | 152 ++++++++++++++++++ 6 files changed, 562 insertions(+), 2 deletions(-) create mode 100644 test/test_sudo_allowlist_covers_calls.py create mode 100644 test/test_systemd_malloc_arenas.py create mode 100644 test/test_systemd_unit_drift.py diff --git a/scripts/install/configure_wifi_permissions.sh b/scripts/install/configure_wifi_permissions.sh index cfba4bc4..37e2c7db 100755 --- a/scripts/install/configure_wifi_permissions.sh +++ b/scripts/install/configure_wifi_permissions.sh @@ -25,9 +25,44 @@ if [ "$EUID" -eq 0 ]; then exit 1 fi +# Resolve command paths against a fixed PATH, and check what we resolved. +# +# Every path found here is written into a sudoers file as a NOPASSWD grant, so +# whoever controls the binary at that path controls root. first_time_install.sh +# re-execs itself with `sudo -E`, which preserves the invoking user's +# environment -- PATH included -- so without pinning it, `which nmcli` can +# resolve to anything on that PATH: a writable directory early in it turns a +# compromise of the low-privilege web user into permanent root. +PATH=/usr/sbin:/usr/bin:/sbin:/bin +export PATH + +# A binary named in a sudoers rule must be root-owned and writable by nobody +# else, or the grant hands root to whoever can rewrite it. +require_trusted_binary() { + local label="$1" path="$2" + if [ ! -x "$path" ]; then + echo "✗ $label: $path is not an executable file" + exit 1 + fi + local owner perms + owner=$(stat -c '%u' "$path") || exit 1 + perms=$(stat -c '%a' "$path") || exit 1 + if [ "$owner" != "0" ]; then + echo "✗ $label: $path is not owned by root (uid $owner); refusing to" + echo " grant it NOPASSWD sudo." + exit 1 + fi + # Group- or world-writable means someone other than root can replace it. + case "$perms" in + *[2367]) echo "✗ $label: $path is writable by group or other ($perms);" + echo " refusing to grant it NOPASSWD sudo." + exit 1 ;; + esac +} + # Get the full paths to commands -NMCLI_PATH=$(which nmcli || echo "/usr/bin/nmcli") -SYSTEMCTL_PATH=$(which systemctl) +NMCLI_PATH=$(command -v nmcli || echo "/usr/bin/nmcli") +SYSTEMCTL_PATH=$(command -v systemctl) echo "Command paths:" echo " nmcli: $NMCLI_PATH" @@ -37,6 +72,18 @@ echo " systemctl: $SYSTEMCTL_PATH" echo "" echo "Step 1: Configuring sudo permissions for nmcli..." SUDOERS_FILE="/etc/sudoers.d/ledmatrix_wifi" +SYSCTL_PATH=$(command -v sysctl || echo /usr/sbin/sysctl) +NFT_PATH=$(command -v nft || echo /usr/sbin/nft) +RFKILL_PATH=$(command -v rfkill || echo /usr/sbin/rfkill) +MKDIR_PATH=$(command -v mkdir || echo /usr/bin/mkdir) + +# Checked before any of them reaches the sudoers file. +require_trusted_binary "nmcli" "$NMCLI_PATH" +require_trusted_binary "systemctl" "$SYSTEMCTL_PATH" +require_trusted_binary "sysctl" "$SYSCTL_PATH" +require_trusted_binary "nft" "$NFT_PATH" +require_trusted_binary "rfkill" "$RFKILL_PATH" +require_trusted_binary "mkdir" "$MKDIR_PATH" # Create a temporary sudoers file using mktemp (handles permissions better) TEMP_SUDOERS=$(mktemp) || { @@ -62,6 +109,36 @@ $WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH start dnsmasq $WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH stop dnsmasq $WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH restart dnsmasq $WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH restart NetworkManager +# The captive portal turns IP forwarding on while the access point is up and +# restores the previous value when it comes down (wifi_manager._setup_iptables_ +# redirect / _teardown_iptables_redirect). Without this rule that sudo call +# needs a password, so forwarding stays off and clients associate to the AP but +# cannot route. It goes unnoticed on a stock Raspberry Pi image, where +# /etc/sudoers.d/010_pi-nopasswd grants the default user blanket NOPASSWD and +# masks every gap in this file -- it only bites once that blanket rule is +# removed. +$WEB_USER ALL=(ALL) NOPASSWD: $SYSCTL_PATH -w net.ipv4.ip_forward=0 +$WEB_USER ALL=(ALL) NOPASSWD: $SYSCTL_PATH -w net.ipv4.ip_forward=1 +# The portal's redirect lives in its own nftables table, created when the AP +# comes up and deleted when it goes down, and the radio has to be unblocked +# before the AP can start at all. Same story as the sysctl rules above: called +# with sudo, never granted here, and invisible on a stock Pi image. +$WEB_USER ALL=(ALL) NOPASSWD: $NFT_PATH add table ip ledmatrix +$WEB_USER ALL=(ALL) NOPASSWD: $NFT_PATH delete table ip ledmatrix +$WEB_USER ALL=(ALL) NOPASSWD: $RFKILL_PATH unblock wifi +# NetworkManager's dnsmasq drop-in directory, exact path. +$WEB_USER ALL=(ALL) NOPASSWD: $MKDIR_PATH -p /etc/NetworkManager/dnsmasq-shared.d +# +# iptables is deliberately NOT granted here. Its rules are built from the live +# interface name and port, so a rule covering them needs a trailing wildcard -- +# and `iptables --modprobe=/path/to/anything` runs that path as root, so +# `NOPASSWD: iptables *` is a root shell for the web user by another name. That +# is a worse outcome than the gap it would close, which today is masked anyway +# by the blanket NOPASSWD rule on stock Pi images. +# +# Closing it safely means a wrapper script that builds the rules itself and +# takes only an interface and a port, granted the way safe_plugin_rm.sh already +# is. That belongs in its own change rather than being smuggled into this one. # Allow copying hostapd and dnsmasq config files into place $WEB_USER ALL=(ALL) NOPASSWD: /usr/bin/cp /tmp/hostapd.conf /etc/hostapd/hostapd.conf diff --git a/src/startup_validator.py b/src/startup_validator.py index 9f2e0a51..79275dc2 100644 --- a/src/startup_validator.py +++ b/src/startup_validator.py @@ -62,6 +62,9 @@ class StartupValidator: # Validate plugins if plugin manager is available if self.plugin_manager: self._validate_plugins() + + # Warn when the running systemd unit no longer matches the repo's + self._validate_systemd_units() is_valid = len(self.errors) == 0 @@ -74,6 +77,80 @@ class StartupValidator: return (is_valid, self.errors.copy(), self.warnings.copy()) + #: Units this project installs, and where each is installed to. + _UNITS = ( + ("systemd/ledmatrix.service", "/etc/systemd/system/ledmatrix.service"), + ("systemd/ledmatrix-web.service", "/etc/systemd/system/ledmatrix-web.service"), + ) + + def _validate_systemd_units(self) -> None: + """Warn when an installed unit has drifted from the repo's template. + + Nothing re-applies these after the first install. `git pull` -- which is + what the web UI's update button runs -- brings a new template into the + checkout, but nothing copies it to /etc/systemd/system and nothing runs + `systemctl daemon-reload`, so the unit that actually runs is whatever + first_time_install.sh wrote on day one. + + That makes every hardening added to a unit inert on existing installs. + Measured on one rig: the installed unit was thirteen days older than the + repo's and differed in content, so a MemoryMax the repo had specified + was not being enforced at all -- `systemctl show` reported + MemoryMax=infinity. + + A warning rather than an error, and certainly not a silent rewrite: + editing files under /etc and restarting services is the installer's job, + not something a display process should do to a machine while it boots. + The remedy is to re-run scripts/install/install_service.sh. + """ + try: + project_root = Path(__file__).resolve().parent.parent + for template_rel, installed_path in self._UNITS: + template = project_root / template_rel + installed = Path(installed_path) + if not template.is_file() or not installed.is_file(): + continue + + # The template carries placeholders the installer substitutes, + # so compare the substituted form rather than the raw file. + expected = template.read_text(encoding="utf-8") + expected = expected.replace("__PROJECT_ROOT_DIR__", str(project_root)) + expected = expected.replace("__USER__", "root") + + try: + actual = installed.read_text(encoding="utf-8") + except PermissionError: + continue + + if self._unit_body(expected) != self._unit_body(actual): + self.warnings.append( + f"{installed.name} differs from {template_rel}; the " + "installed unit is not refreshed by an update, so " + "settings added to the template are not in effect. " + "Re-run scripts/install/install_service.sh to apply them." + ) + except OSError as e: + self.logger.debug("Could not compare systemd units: %s", e) + + @staticmethod + def _unit_body(text: str) -> str: + """A unit's meaningful lines, in order: no comments, no blanks. + + Order is preserved deliberately. This used to sort, which made the + comparison insensitive to two changes that matter in a systemd unit: + repeated directives such as ExecStartPre= and ExecStartPost= run in + the order they appear, and a directive that moves between [Unit], + [Service] and [Install] means something different -- or nothing -- + where it lands. A drift check that normalises those away reports no + drift for a unit that has genuinely changed. + """ + lines = [] + for line in text.splitlines(): + line = line.strip() + if line and not line.startswith("#"): + lines.append(line) + return "\n".join(lines) + def _validate_config(self) -> None: """Validate configuration files.""" try: diff --git a/systemd/ledmatrix.service b/systemd/ledmatrix.service index d5f064b6..ab5ddf4d 100644 --- a/systemd/ledmatrix.service +++ b/systemd/ledmatrix.service @@ -8,6 +8,18 @@ Type=simple User=root WorkingDirectory=__PROJECT_ROOT_DIR__ Environment=PYTHONDONTWRITEBYTECODE=1 +# glibc gives each allocating thread its own malloc arena, up to 8 x CPU count, +# and an arena that has grown is never handed back to the OS. This process runs +# 9 threads on a 3-core Pi, so the ceiling is 24 arenas -- and a rig measured at +# 1030 MB resident held 23 large anonymous mappings on 64 MB-aligned addresses, +# 920 MB of them, while the live data it was actually holding (widest scroll +# strip seen: 35,746 x 64) accounts for roughly 15 MB. That gap is arena bloat, +# not leaked objects: RSS was flat across repeated sampling, not climbing. +# +# Capping the arenas trades a little allocator concurrency for a large amount of +# resident memory on a device that has neither to spare. 2 is the usual value; +# raise it if frame times regress. +Environment=MALLOC_ARENA_MAX=2 ExecStart=/usr/bin/python3 __PROJECT_ROOT_DIR__/run.py # Restart=always, not on-failure: run.py exiting 0 (a clean shutdown path taken # for a reason that no longer applies, e.g. a config reload) would otherwise leave diff --git a/test/test_sudo_allowlist_covers_calls.py b/test/test_sudo_allowlist_covers_calls.py new file mode 100644 index 00000000..bec2486b --- /dev/null +++ b/test/test_sudo_allowlist_covers_calls.py @@ -0,0 +1,147 @@ +"""The captive portal's fixed-argument sudo calls must be granted. + +The installers write two allow-lists, /etc/sudoers.d/ledmatrix_web and +ledmatrix_wifi. A sudo call absent from both needs a password, which a service +cannot supply, so it fails. + +Four such calls were ungranted, all of them captive-portal teardown/setup: + + sysctl -w net.ipv4.ip_forward=0|1 wifi_manager.py:788, 883 + nft add|delete table ip ledmatrix wifi_manager.py:835, 895 + rfkill unblock wifi wifi_manager.py:1811 + mkdir -p .../dnsmasq-shared.d wifi_manager.py:922 + +It goes unnoticed because a stock Raspberry Pi image ships +/etc/sudoers.d/010_pi-nopasswd granting the default user +`ALL=(ALL) NOPASSWD: ALL`, which satisfies every gap in both files. It only +bites once that blanket rule is removed or the service runs as another user. + +Scope, deliberately narrow: this pins the four commands above, each of which +can be written out literally. The portal makes further sudo calls whose +arguments are built at runtime -- iptables and nft rules carrying an interface +name and a port, `ip addr`, `ip link` -- and those cannot be granted safely +here. A rule covering them needs a trailing wildcard, and +`iptables --modprobe=/path/to/anything` runs that path as root, so +`NOPASSWD: iptables *` is a root shell for the web user by another name. +Closing that half needs a privileged helper that builds the rules itself and +takes only an interface and a port, granted the way safe_plugin_rm.sh already +is. That is a design decision, not a one-line grant, and belongs in its own +change. +""" +import re +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +INSTALLERS = ( + ROOT / "first_time_install.sh", + ROOT / "scripts" / "install" / "configure_wifi_permissions.sh", +) + +#: Commands this change grants, each fully literal in the source. +REQUIRED = ( + ("sysctl", "-w", "net.ipv4.ip_forward=0"), + ("sysctl", "-w", "net.ipv4.ip_forward=1"), + ("nft", "add", "table", "ip", "ledmatrix"), + ("nft", "delete", "table", "ip", "ledmatrix"), + ("rfkill", "unblock", "wifi"), + ("mkdir", "-p", "/etc/NetworkManager/dnsmasq-shared.d"), +) + +#: Tools with an option that executes a program of the caller's choosing. +#: A trailing wildcard on any of these is a privilege escalation. +EXEC_CAPABLE = ("iptables", "ip6tables", "nft", "tcpdump", "find", "awk", + "sed", "perl", "python", "python3", "env") + + +def _grant_lines(): + lines = [] + for installer in INSTALLERS: + if not installer.is_file(): + continue + for line in installer.read_text(encoding="utf-8", errors="replace").splitlines(): + if "NOPASSWD:" in line: + lines.append(line.split("NOPASSWD:", 1)[1]) + return lines + + +def _normalise(rule): + """One rule with binary-path variables reduced to bare tool names. + + Rules are written as `$SYSCTL_PATH -w ...` or `${NFT_PATH} ...`, so + matching the literal "sysctl" finds nothing and every rule looks absent -- + which is exactly how an earlier version of this test reported six gaps + that did not exist. Both spellings are handled: shell expands them + identically, and a check that understood only one silently skipped the + other. + """ + rule = re.sub(r"\$\{?([A-Z][A-Z0-9_]*)_PATH\}?", + lambda m: m.group(1).lower(), rule) + return re.sub(r"/usr/(?:s?bin)/", "", rule) + + +def _granted_commands(): + """The command each NOPASSWD rule actually grants, normalised. + + _grant_lines() already returns everything after "NOPASSWD:", so what + arrives here is the command, possibly preceded by the NOEXEC tag and + possibly still carrying the closing quote of an `echo "..."` that wrote + it. Both are stripped so the result is comparable to a plain command. + """ + commands = [] + for rule in _grant_lines(): + command = _normalise(rule).strip() + command = re.sub(r"^NOEXEC:\s*", "", command) + command = command.rstrip('"').rstrip("'").strip() + if command: + commands.append(" ".join(command.split())) + return commands + + +def test_the_installers_are_present(): + missing = [str(p.relative_to(ROOT)) for p in INSTALLERS if not p.is_file()] + assert not missing, f"installer(s) missing: {missing}" + + +@pytest.mark.parametrize("command", REQUIRED, ids=lambda c: " ".join(c)) +def test_the_command_is_granted(command): + """Whole command, not just the binary. + + Checking only the binary made this far weaker than it looked: with + `sysctl` present anywhere, deleting the ip_forward=0 grant still passed, + and the portal would then be unable to restore forwarding on teardown. + """ + wanted = " ".join(command) + granted = _granted_commands() + # Exact match, not a prefix. A substring search was satisfied by + # `sysctl -w net.ipv4.ip_forward=0 *`, and that trailing wildcard lets the + # caller append whatever they like to a command running as root -- a far + # wider grant than the one this test is meant to be confirming. + assert wanted in granted, ( + f"no installer grants exactly `{wanted}`; closest matches: " + + str([g for g in granted if g.startswith(command[0])])[:200]) + + +def test_no_wildcard_on_a_tool_that_can_exec(): + """`NOPASSWD: iptables *` hands the web user root. + + iptables --modprobe=/path runs that path as root. This caught a grant added + in this very change, which is why it is here. + """ + offenders = [] + for rule in _grant_lines(): + rule = rule.strip() + if not rule.endswith("*"): + continue + # Normalised the same way as everything else: `${NFT_PATH} *` left a + # brace before the tool name, and the word-boundary check below does + # not treat "{" as a boundary, so that spelling slipped through. + haystack = _normalise(rule).lower() + for tool in EXEC_CAPABLE: + if re.search(rf"(^|/|\s|\$){tool}(\s|$)", haystack): + offenders.append(rule) + break + assert not offenders, ( + "wildcard grant on a tool that can execute another program:\n " + + "\n ".join(offenders)) diff --git a/test/test_systemd_malloc_arenas.py b/test/test_systemd_malloc_arenas.py new file mode 100644 index 00000000..bb4bc8f0 --- /dev/null +++ b/test/test_systemd_malloc_arenas.py @@ -0,0 +1,95 @@ +"""The display unit must cap glibc's malloc arenas. + +glibc hands each allocating thread its own malloc arena, up to 8 x CPU count, +and an arena that has grown is never returned to the OS. This process runs +threads for the render loop, the update workers and the background fetchers, so +on a 3-core Pi the ceiling is 24 arenas. + +Measured on a live rig, 2.5 hours in: + + RSS 1030 MB + Private_Dirty 988 MB + anonymous mappings > 10 MB 23 (ceiling is 8 x 3 = 24) + largest few 104, 79, 66, 63, 63 MB, on 64 MB-aligned addresses + +against live data that accounts for perhaps 15 MB -- the widest scroll strip +observed was 35,746 x 64, about 7 MB as RGB and the same again for its numpy +mirror. Repeated sampling showed RSS flat between 990 and 1030 MB rather than +climbing, so this is arena bloat rather than a leak: memory Python has freed +but glibc is holding per-arena. + +The device had 59 MB free at the time. + +Capping the arena count trades a little allocator concurrency for that resident +memory. The render loop is latency-sensitive, so if p99 frame time regresses the +right response is to raise this rather than remove it. +""" +import re +from pathlib import Path + +import pytest + +UNIT = (Path(__file__).resolve().parent.parent / "systemd" / "ledmatrix.service") + +#: The value the unit is expected to carry. 2 is the usual choice for a +#: threaded Python process; 1-4 all keep some of the saving, but only one of +#: them is what this project ships. +EXPECTED_ARENA_MAX = 2 + + +def _environment(unit_text): + return dict( + line.split("=", 2)[1:3] if line.count("=") >= 2 else (line.split("=", 1)[1], "") + for line in unit_text.splitlines() + if line.startswith("Environment=") + ) + + +def test_the_unit_exists(): + assert UNIT.is_file(), f"{UNIT} is missing" + + +def test_malloc_arena_max_is_capped(): + env = _environment(UNIT.read_text(encoding="utf-8")) + assert "MALLOC_ARENA_MAX" in env, ( + "the display unit does not cap glibc arenas; on a 3-core Pi the default " + "ceiling is 24 and a measured rig held 23 of them, 920 MB" + ) + value = int(env["MALLOC_ARENA_MAX"]) + # Pinned, not a range. A range let a change to 4 -- which hands most of the + # saving back -- pass unnoticed, which was the point of the finding that + # prompted this. Raising it is a legitimate response to a frame-time + # regression, but it should be a visible edit here rather than a silent + # drift, so the number lives in one place and changing it shows up in + # review. + assert value == EXPECTED_ARENA_MAX, ( + f"MALLOC_ARENA_MAX={value}, expected {EXPECTED_ARENA_MAX}. If this was " + "raised deliberately because frame times regressed, update " + "EXPECTED_ARENA_MAX here and say so in the commit." + ) + + +def test_the_reason_is_recorded_next_to_it(): + """A bare tuning knob invites removal by whoever meets it next.""" + text = UNIT.read_text(encoding="utf-8") + index = text.index("Environment=MALLOC_ARENA_MAX") + preamble = text[:index].splitlines()[-12:] + comment = "\n".join(line for line in preamble if line.startswith("#")) + assert "arena" in comment.lower(), "no explanation precedes the setting" + assert re.search(r"\d", comment), ( + "the explanation cites no measurement, so a reader cannot tell whether " + "it still applies to their hardware" + ) + + +@pytest.mark.parametrize("unit", ["ledmatrix.service"]) +def test_the_unit_still_parses_as_ini(unit): + """systemd will refuse a malformed unit, and the panel stays dark.""" + import configparser + + path = UNIT.parent / unit + parser = configparser.ConfigParser(strict=False) + # systemd allows repeated keys; ConfigParser needs them merged, not rejected. + parser.read_string(path.read_text(encoding="utf-8")) + assert parser.has_section("Service") + assert parser.has_option("Service", "ExecStart") diff --git a/test/test_systemd_unit_drift.py b/test/test_systemd_unit_drift.py new file mode 100644 index 00000000..21cf56e0 --- /dev/null +++ b/test/test_systemd_unit_drift.py @@ -0,0 +1,152 @@ +"""An installed unit that no longer matches the repo's must be reported. + +Nothing re-applies systemd units after the first install. `git pull` -- what +the web UI's update button runs -- brings a new template into the checkout, but +no code in web_interface/ or src/ copies it to /etc/systemd/system or runs +`systemctl daemon-reload`. The unit that actually runs is whatever +first_time_install.sh wrote on day one. + +So every hardening added to a unit is inert on existing installs. Measured on a +live rig: the installed unit was dated 2026-08-06 and the repo's 2026-08-19, +and they differed -- with the result that a MemoryMax=85% present in the repo's +template was not being enforced at all. `systemctl show` reported +MemoryMax=infinity. + +This is a warning, not an error, and deliberately not a silent rewrite: +editing files under /etc and restarting services is the installer's job, not +something a display process should do to a machine while it boots. +""" +import logging +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from src.startup_validator import StartupValidator + + +@pytest.fixture +def validator(): + v = StartupValidator(config_manager=MagicMock()) + v.logger = logging.getLogger("test") + v.warnings = [] + v.errors = [] + return v + + +def test_a_matching_unit_produces_no_warning(validator, tmp_path): + """The installed unit, substituted exactly as the installer would.""" + project_root = Path("src/startup_validator.py").resolve().parent.parent + template_rel = "systemd/ledmatrix.service" + template = project_root / template_rel + if not template.is_file(): + pytest.skip("repo unit template not present") + + installed = tmp_path / "ledmatrix.service" + installed.write_text( + template.read_text(encoding="utf-8") + .replace("__PROJECT_ROOT_DIR__", str(project_root)) + .replace("__USER__", "root"), + encoding="utf-8") + + validator._UNITS = ((template_rel, str(installed)),) + validator._validate_systemd_units() + assert not validator.warnings, f"a matching unit warned: {validator.warnings}" + assert not validator.errors + + +def test_comments_and_blank_lines_are_not_drift(): + """Otherwise every comment the repo adds would look like a changed unit.""" + a = "[Service]\n# explains a setting\nExecStart=/x\nRestart=always\n" + b = "[Service]\nExecStart=/x\n\nRestart=always\n" + assert StartupValidator._unit_body(a) == StartupValidator._unit_body(b) + + +def test_a_changed_directive_is_drift(): + a = "[Service]\nExecStart=/x\nMemoryMax=85%\n" + b = "[Service]\nExecStart=/x\n" + assert StartupValidator._unit_body(a) != StartupValidator._unit_body(b) + + +def test_reordered_directives_are_drift(): + """Order is not noise in a systemd unit. + + Repeated directives -- ExecStartPre=, ExecStartPost= -- run in the order + they appear, and a directive that moves between [Unit], [Service] and + [Install] means something different, or nothing, where it lands. This + check used to sort the lines before comparing, which reported no drift for + a unit that had genuinely changed. + """ + a = "[Service]\nExecStartPre=/first\nExecStartPre=/second\n" + b = "[Service]\nExecStartPre=/second\nExecStartPre=/first\n" + assert StartupValidator._unit_body(a) != StartupValidator._unit_body(b), ( + "swapping two ExecStartPre= lines changes what runs first, and was " + "being normalised away") + + +def test_a_directive_moved_between_sections_is_drift(): + a = "[Unit]\nDescription=x\n[Service]\nExecStart=/x\n" + b = "[Unit]\nDescription=x\nExecStart=/x\n[Service]\n" + assert StartupValidator._unit_body(a) != StartupValidator._unit_body(b), ( + "ExecStart= in [Unit] is not the same unit, and sorting hid it") + + +def test_cosmetic_differences_do_not_warn(validator, tmp_path): + """Through the real comparison, not the helper. + + The repo's template carries explanatory comments the installed copy may not + have, and the installer does not preserve ordering or blank lines. If those + counted as drift, every boot would warn and the warning would be ignored. + Asserting this on _unit_body alone would not catch a comparison that stopped + calling it -- which is exactly what a careless edit does. + """ + project_root = Path("src/startup_validator.py").resolve().parent.parent + template_rel = "systemd/ledmatrix.service" + template = project_root / template_rel + if not template.is_file(): + pytest.skip("repo unit template not present") + + substituted = (template.read_text(encoding="utf-8") + .replace("__PROJECT_ROOT_DIR__", str(project_root)) + .replace("__USER__", "root")) + # Cosmetic means comments, blank lines and stray indentation -- the things + # the installer really does drop. Not reordering: that changes the unit, + # and is asserted as drift above. + directives = [line.strip() for line in substituted.splitlines() + if line.strip() and not line.strip().startswith("#")] + installed = tmp_path / "ledmatrix.service" + installed.write_text( + "\n\n".join(" " + d for d in directives) + "\n", encoding="utf-8") + + validator._UNITS = ((template_rel, str(installed)),) + validator._validate_systemd_units() + assert not validator.warnings, ( + f"cosmetic-only difference reported as drift: {validator.warnings}") + + +def test_drift_is_reported_as_a_warning(validator, tmp_path): + """The whole point: a real difference must surface, and only as a warning.""" + installed = tmp_path / "ledmatrix.service" + installed.write_text("[Service]\nExecStart=/usr/bin/python3 /x/run.py\n") + + project_root = Path("src/startup_validator.py").resolve().parent.parent + template_rel = "systemd/ledmatrix.service" + template = project_root / template_rel + if not template.is_file(): + pytest.skip("repo unit template not present") + + validator._UNITS = ((template_rel, str(installed)),) + validator._validate_systemd_units() + + assert validator.warnings, "a differing unit produced no warning" + assert "install_service.sh" in validator.warnings[0], ( + "the warning does not tell the user how to fix it") + assert not validator.errors, "drift must not be fatal at startup" + + +def test_a_missing_installed_unit_is_silent(validator, tmp_path): + """Development checkouts have no /etc/systemd unit; that is not drift.""" + validator._UNITS = (("systemd/ledmatrix.service", str(tmp_path / "absent.service")),) + validator._validate_systemd_units() + assert not validator.warnings + assert not validator.errors From 9c0c0dc8519c54b155305318de1cd989d6f44256 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:30:29 -0400 Subject: [PATCH 32/53] consolidate(web): credential exposure, secret loss, and the update path (#485) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(web): stop /config/main handing out every credential it holds The endpoint returned the raw config to anyone who could reach the port, and this web interface has no authentication of any kind. An unauthenticated request against a live rig returned: github.api_token 40 chars incoming-packages.ha_token 183 chars jellyfin-now-playing.api_key 32 chars ledmatrix-weather.api_key 32 chars on-air.mqtt_password 8 chars youtube.api_key 20 chars youtube-stats.api_key 39 chars A GitHub token and a Home Assistant long-lived token among them. Anything on that LAN could read them. The x-secret masking the plugin config endpoints use does not reach here: this route never consults a schema, and core keys such as github.api_token have no schema to carry the marker. Several of the fields above *are* tagged x-secret in their plugin's schema and were still returned in full, which is what rules out the schema route as the fix for this endpoint. Credential-named fields are now blanked. Matching on the name is blunt, and for a whole-config dump that is the right default: anything named like a credential should not leave the process, and a new plugin adding a differently-shaped secret is covered without anyone remembering to tag it. Blanked rather than removed, and safe to blank: POST /config/main merges into the freshly loaded config and writes only the keys it was given, so a client that round-trips this response cannot erase a secret it never saw. The web API suites confirm it -- 81 passing, unchanged. On the test that matters: the first version of this suite exercised the two helpers and nothing else, and reverting the single line that wires the redactor into the route passed all thirty of them. A property asserted on a helper is not a property asserted on the endpoint, and it is the endpoint that is exposed to the network. The added test goes through the view function, and it does fail on that revert. This also corrects an earlier claim of mine. I reported that GET /api/v3/config did not expose these values; that path 404s, so the check proved nothing. The real route is /config/main and it exposed all of them. * fix(web): stop an unrelated config edit from erasing a plugin's secret Saving any field on a plugin's config form destroyed that plugin's stored credential. On a rig with a weather API key, changing the city silently emptied the key, and the plugin stopped working at the next fetch with no indication why. The path had no guard at any step. The config partial masks secrets before rendering (pages_v3.py:740), so the browser posts them back blank; _parse_value deliberately preserves "" for optional string fields; separate_secrets routes that "" into secrets_config, which is a truthy dict; deep_merge writes it over the stored value; save_raw_file_content persists it. The blank does not even need the round-trip. merge_with_defaults injects the schema's api_key default ("") into every save, so a client that never sends the field at all still erases it. test_secret_count_message_counts_top_level_keys was counting exactly that injected blank as a saved secret field -- the visible edge of the bug, pinned as expected behaviour. remove_empty_secrets() already existed for this, with seven unit tests and a docstring describing this precise scenario ("clients will send those empty strings back ... so that existing stored secrets are not overwritten with blanks"). It was never wired into a call site. This wires it into both save paths that merge into the secrets file. A blank now means "unchanged" rather than "delete", which is the same contract the helper's tests already describe. The cost is that a secret can no longer be cleared by emptying the field; clearing needs its own affordance, since a control that erases credentials as a side effect of ordinary edits is not one. Verified by reverting the guard: the new round-trip test then fails with the stored key read back as ''. 262 web tests pass with it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(web): stop dumping the config and request headers to the journal save_main_config logged its entire POST body and the full request headers at ERROR on every save. The body is the configuration itself, and the headers carry the session cookie, so a routine settings change wrote both to the journal -- at a level that guarantees they survive any sane log filter. The lines are leftover debug output: they say "DEBUG:" in the message while calling logging.error, and they went through the root logger rather than the module logger, bypassing the level configured for this blueprint. Replaced with a debug-level line recording the shape of the request, which is the part with diagnostic value. The local `import logging` went with them; it shadowed a module-level import that was already there. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(web): stop /config/secrets handing out every credential it holds GET /api/v3/config/secrets returned config_secrets.json in full to anyone who could reach the port, and this interface has no authentication. Probed against a real rig it produced six populated credential fields: a 40-character GitHub token, a 183-character Home Assistant token, and Jellyfin and weather API keys. This is the second door onto the same credentials; #477 closes the first. Masking the response alone would have been worse than the leak. The only client fetches every secret, edits one field and posts all of them back, and save_raw_file_content replaces the file wholesale -- so a masked GET followed by the client's own save would write the mask over every credential the user had not touched. That is why this was left open when the leak was found; it needs both halves. Read side: mask_all_secret_values(), which already existed for exactly this endpoint -- its docstring names it -- and had never been wired to a call site. It leaves empty values and YOUR_* placeholders alone, so a client can still tell "set" from "not set" without being told the secret. Write side: strip the echoed mask and blanks from the submission, then merge onto what is stored, so "unchanged" means unchanged. The cost is that a secret can no longer be cleared by blanking it; that wants its own affordance, since a control that erases credentials as a side effect of saving an unrelated one is not one. Browser side: the token field is now left empty rather than filled from the response. Filling it with the mask would have stored eight bullet characters as the token the next time the user pressed Save, and filling it with the real value is the thing being fixed. It reports whether a token is saved instead. Verified end to end through the Flask endpoints, not the helpers. Reverting the masking fails the leak tests; reverting the merge fails the preservation tests; both halves are independently guarded. 278 web tests pass. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(web): stop reporting "no update" when the update check could not run check-update returned update_available=False whenever git failed. The banner is the only route to the update button, so a checkout git refuses to touch looked exactly like a current one -- permanently, with nothing on screen to act on and only a log line recording why. The common cause is an install performed as root. scripts/install/one-shot-install.sh clones into ${HOME}/LEDMatrix, never consults SUDO_USER, and contains no chown at all, while its own error text suggests running the whole thing under sudo. The result is a root-owned checkout, and on a rig this is what every git command in it does: fatal: detected dubious ownership in repository at '...' including the fetch this endpoint runs. Verified on real hardware rather than assumed. A failed check now reports check_failed with a message the user can act on -- for dubious ownership, the chown that fixes it. The banner shows that message instead of hiding itself, with the update button suppressed since updating cannot work until the cause is fixed. The success path is untouched. This does not fix the installer, which is the real cause; it stops the symptom being invisible. The installer needs SUDO_USER handling and a chown, and its suggestion to run as root should go. Reverting the endpoint change fails four of the five new tests; the fifth guards the success path and correctly does not move. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(web): stop the installer chmod stripping exec bits on every update git tracks five scripts as mode 644 that first_time_install.sh then chmods to 755 (start_display.sh, stop_display.sh, the two install_*_service.sh, and one-shot-install.sh does the same to first_time_install.sh). With core.fileMode true, the default on Linux, git reports all five as modified from then on, in files the user never touched. The update button stashes local changes before pulling, so it is not blocked by this. But it never pops that stash -- stash pop and stash apply appear nowhere in the update flow -- so the mode change is stashed away and left there, and the files revert: === file modes after the update button's stash === 664 first_time_install.sh <- installer had made these 755 664 start_display.sh 664 stop_display.sh 664 scripts/install/install_service.sh So every web-UI update silently strips the executable bit from the installer's own scripts, and leaves a stash entry holding the difference. start_display.sh and stop_display.sh stop working from the shell afterwards. A manual `git pull --rebase` over SSH fails outright, since nothing stashes for it: "cannot pull with rebase: You have unstaged changes". That is the likely source of the reports, since plenty of people update that way. Tracking the five as 755 -- what they should always have been, as the installer chmodding them attests -- removes the spurious mode change entirely: nothing to stash, nothing stripped, no stash entry, and manual pulls work. The pull also passes --autostash, for the case the code explicitly tolerates: when the stash fails it logs a warning and pulls anyway, and that pull is what then fails. Autostash also pops what it stashes, which the manual stash does not. Note that `git add -A` after `git update-index --chmod=+x` silently reverts the index to the on-disk mode, so the modes here were set by chmodding the files themselves. Regression test asserts the five stay tracked executable; reverting any one of them fails it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(web): ask for the restart that makes an update take effect The update button pulls new code and restarts nothing. There is no systemctl, restart, reload or reboot anywhere in the 172-line git_pull handler -- it stashes, pulls, installs changed requirements, re-removes plugins the user had uninstalled, and returns "Code updated successfully." Meanwhile both services go on running the code they loaded at boot. So the display keeps rendering the old build, the web interface keeps serving the old build, and the user is told the update worked. Nothing on screen suggests otherwise, and the next reboot is what actually applies it -- whenever that is. The affordance for this already exists: the restart-pending banner, raised after main-config saves, with a Restart Now button wired to the display service. A code update is a stronger reason to show it than a config save is. The response now reports restart_required, and applyUpdate raises the banner with wording for a code update rather than a config save. The banner's message became a parameter and is persisted next to the flag, since it outlives the page that raised it. restart_required is only true when the pull actually moved HEAD. "Already up to date" is a success too, and prompting after a no-op would train users to dismiss the prompt unread. This covers the display service, which is what the Restart Now button drives and what users notice. The web interface still picks up its own new code on its next restart; restarting it from inside a request it is serving is a larger change than this one. Reverting the flag fails the test that a pull which moved HEAD asks for a restart. 290 tests pass. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * Mask list-shaped secrets element-wise, close two vacuous tests mask_all_secret_values treated any non-empty list as a scalar, so a secrets file holding "accounts": [{"name": "a", "token": "tok-a"}, {...}] came back as a single "••••••••". The caller could not see how many entries existed, and the raw editor was handed a string where the file holds an array. Recurse into lists in both _mask_value and _contains_mask. Lists merge by replacement, not key-wise, so strip_masked_values now drops a list outright if any element still carries the mask -- storing a half-masked list would discard the untouched entries. Two tests could pass without exercising what they claim to check: - test_git_pull_resolution asserted modes only for paths git ls-files returned. A renamed or deleted installer target is simply absent from that output, so its mode was never checked. Assert every CHMODDED path is tracked first. - test_config_secrets_masking never checked the POST status. A 500 leaves the old file in place, which satisfies every assertion that follows. Assert 200 before reading the file back. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 (1M context) --- assets/sports/ncaa_logos/COR.png | Bin 0 -> 467 bytes first_time_install.sh | 0 scripts/install/install_service.sh | 0 scripts/install/install_web_service.sh | 0 src/web_interface/secret_helpers.py | 83 ++++++++-- start_display.sh | 0 stop_display.sh | 0 test/test_config_main_redacts_secrets.py | 143 ++++++++++++++++ test/test_git_pull_resolution.py | 47 +++++- test/test_update_check_reports_failure.py | 85 ++++++++++ test/test_update_prompts_restart.py | 84 ++++++++++ .../test_api_v3_secret_roundtrip.py | 41 ++++- .../test_config_secrets_masking.py | 123 ++++++++++++++ web_interface/blueprints/api_v3.py | 152 ++++++++++++++++-- web_interface/static/v3/app.js | 20 ++- web_interface/static/v3/plugins_manager.js | 14 +- web_interface/templates/v3/base.html | 28 +++- 17 files changed, 774 insertions(+), 46 deletions(-) create mode 100644 assets/sports/ncaa_logos/COR.png mode change 100644 => 100755 first_time_install.sh mode change 100644 => 100755 scripts/install/install_service.sh mode change 100644 => 100755 scripts/install/install_web_service.sh mode change 100644 => 100755 start_display.sh mode change 100644 => 100755 stop_display.sh create mode 100644 test/test_config_main_redacts_secrets.py create mode 100644 test/test_update_check_reports_failure.py create mode 100644 test/test_update_prompts_restart.py create mode 100644 test/web_interface/test_config_secrets_masking.py diff --git a/assets/sports/ncaa_logos/COR.png b/assets/sports/ncaa_logos/COR.png new file mode 100644 index 0000000000000000000000000000000000000000..d39027fabec5bf8b8f20c6457c51ab79677ac05e GIT binary patch literal 467 zcmeAS@N?(olHy`uVBq!ia0vp^4j|0I1|(Ny7T#lEV4UUY;uumf=k2wFzK0D&*dExg zU{`Tt=plM2Td_4i_7_blh45g z2^>AtseG|%*FuA55r1Z1zS5!D+{P9gO)Ty?#@ZL+@hiv2d*nLSI{p8U_qd0^eZ%P`~Stut{+ANdYssT?ty_nl!Lcihe+ zJjMSeX&P)fw=Orp%#7jrE#|+C=Vh{*8`otT7!=lqz3VFM z=gv(^`{`6Ur^@tQmi(DkfuQC%#yv|z?RVR!$@ec0{y2Nt%eO4&cCK7mu=^oiV+Am>7(8A5T-G@yGywp) CdCl1X literal 0 HcmV?d00001 diff --git a/first_time_install.sh b/first_time_install.sh old mode 100644 new mode 100755 diff --git a/scripts/install/install_service.sh b/scripts/install/install_service.sh old mode 100644 new mode 100755 diff --git a/scripts/install/install_web_service.sh b/scripts/install/install_web_service.sh old mode 100644 new mode 100755 diff --git a/src/web_interface/secret_helpers.py b/src/web_interface/secret_helpers.py index 310d9848..6a00cb94 100644 --- a/src/web_interface/secret_helpers.py +++ b/src/web_interface/secret_helpers.py @@ -143,6 +143,12 @@ def mask_secret_fields(config: Dict[str, Any], schema_properties: Dict[str, Any] return result +#: What a masked secret looks like on the wire. Named because the write path +#: has to recognise it coming back: a client that renders the mask and posts +#: it unchanged must not store the mask as if it were the secret. +SECRET_MASK = '\u2022' * 8 + + def mask_all_secret_values(config: Dict[str, Any]) -> Dict[str, Any]: """Blanket-mask every non-empty value in a secrets config dict. @@ -156,15 +162,25 @@ def mask_all_secret_values(config: Dict[str, Any]) -> Dict[str, Any]: Returns: A copy with all real values replaced by ``'••••••••'``. """ - masked: Dict[str, Any] = {} - for k, v in config.items(): - if isinstance(v, dict): - masked[k] = mask_all_secret_values(v) - elif v not in (None, '') and not (isinstance(v, str) and v.startswith('YOUR_')): - masked[k] = '••••••••' - else: - masked[k] = v - return masked + return {k: _mask_value(v) for k, v in config.items()} + + +def _mask_value(value: Any) -> Any: + """Mask one value, recursing through dicts and lists. + + A list used to be masked as though it were a scalar, so + ``accounts: [{"name": "a", "token": "..."}]`` came back as a single + ``'••••••••'``. Nothing leaked, but the caller could no longer see how + many entries there were or any of their non-secret fields, and the raw + editor was shown a string where the file holds an array. + """ + if isinstance(value, dict): + return {k: _mask_value(v) for k, v in value.items()} + if isinstance(value, list): + return [_mask_value(item) for item in value] + if value in (None, '') or (isinstance(value, str) and value.startswith('YOUR_')): + return value + return SECRET_MASK def remove_empty_secrets(secrets: Dict[str, Any]) -> Dict[str, Any]: @@ -189,3 +205,52 @@ def remove_empty_secrets(secrets: Dict[str, Any]) -> Dict[str, Any]: elif v is not None and not (isinstance(v, str) and v.strip() == ''): result[k] = v return result + + +def strip_masked_values(secrets: Dict[str, Any]) -> Dict[str, Any]: + """Remove values a client echoed back rather than changed. + + The counterpart to :func:`mask_all_secret_values`. A client that GETs the + masked secrets, edits one field and POSTs the whole object back is sending + ``SECRET_MASK`` for every field it did not touch. Storing those would + replace each untouched credential with eight bullet characters. + + Drops the mask and, like :func:`remove_empty_secrets`, blank values -- so + the caller can merge the result onto what is already stored and have + "unchanged" mean unchanged. Empty nested dicts are pruned. + """ + result: Dict[str, Any] = {} + for k, v in secrets.items(): + if isinstance(v, dict): + nested = strip_masked_values(v) + if nested: + result[k] = nested + elif isinstance(v, list): + # A list is merged by replacement, not element by element -- there + # is no identity to match entries on -- so a list that still holds + # a mask cannot be merged safely: keeping it would store bullets, + # and keeping the submitted entries alone would drop whichever the + # client did not send back. Dropping the key leaves the stored + # list untouched, which is what an untouched list should do. + # + # The consequence, deliberately: editing one secret inside a list + # through this endpoint requires sending real values for all of + # them. Sending some masks leaves the whole list as it was. + if not _contains_mask(v): + result[k] = v + elif v is None: + continue + elif isinstance(v, str) and (v.strip() == '' or v == SECRET_MASK): + continue + else: + result[k] = v + return result + + +def _contains_mask(value: Any) -> bool: + """True when a mask sentinel survives anywhere inside ``value``.""" + if isinstance(value, dict): + return any(_contains_mask(v) for v in value.values()) + if isinstance(value, list): + return any(_contains_mask(item) for item in value) + return value == SECRET_MASK diff --git a/start_display.sh b/start_display.sh old mode 100644 new mode 100755 diff --git a/stop_display.sh b/stop_display.sh old mode 100644 new mode 100755 diff --git a/test/test_config_main_redacts_secrets.py b/test/test_config_main_redacts_secrets.py new file mode 100644 index 00000000..15172df2 --- /dev/null +++ b/test/test_config_main_redacts_secrets.py @@ -0,0 +1,143 @@ +"""GET /config/main must not hand out credentials. + +The endpoint returned the raw config to anyone who could reach the port, and +this web interface has no authentication of any kind. Measured against a live +rig, an unauthenticated request returned: + + github.api_token 40 chars + incoming-packages.ha_token 183 chars + jellyfin-now-playing.api_key 32 chars + ledmatrix-weather.api_key 32 chars + on-air.mqtt_password 8 chars + youtube.api_key 20 chars + youtube-stats.api_key 39 chars + +A GitHub token and a Home Assistant long-lived token among them. + +The x-secret masking the plugin config endpoints use does not apply here: this +endpoint never consults a schema, and core keys such as github.api_token have +no schema to carry the marker. Several of those fields *are* tagged x-secret in +their plugin's schema and were still returned in full, which is what makes the +schema route the wrong one to rely on for this endpoint. + +Matching on field name is blunt. For a whole-config dump it is the right +default: anything named like a credential should not leave the process, and a +new plugin that adds a differently-shaped secret is covered without anyone +remembering to tag it. +""" +import pytest + +from web_interface.blueprints.api_v3 import ( + _looks_like_a_credential, + _redact_credentials, +) + + +@pytest.mark.parametrize("name", [ + "password", "mqtt_password", "opensky_password", "passwd", + "api_key", "apikey", "API_KEY", "flightaware_api_key", + "token", "ha_token", "api_token", "access_token", + "secret", "client_secret", "spotify_client_secret", + "access_key", "private_key", +]) +def test_credential_names_are_recognised(name): + assert _looks_like_a_credential(name) + + +@pytest.mark.parametrize("name", [ + "timezone", "city", "brightness", "enabled", "update_interval", + "favorite_teams", "display_duration", "keyword", +]) +def test_ordinary_names_are_left_alone(name): + assert not _looks_like_a_credential(name) + + +def test_the_measured_leak_is_closed(): + """The exact shape taken off the rig.""" + config = { + "github": {"api_token": "ghp_" + "x" * 36}, + "incoming-packages": {"ha_token": "y" * 183, "enabled": True}, + "jellyfin-now-playing": {"api_key": "z" * 32}, + "on-air": {"mqtt_password": "hunter22"}, + "youtube": {"api_key": "k" * 20}, + "timezone": "America/New_York", + } + out = _redact_credentials(config) + assert out["github"]["api_token"] == "" + assert out["incoming-packages"]["ha_token"] == "" + assert out["jellyfin-now-playing"]["api_key"] == "" + assert out["on-air"]["mqtt_password"] == "" + assert out["youtube"]["api_key"] == "" + # Everything else survives, or the config editor breaks. + assert out["timezone"] == "America/New_York" + assert out["incoming-packages"]["enabled"] is True + + +def test_nested_and_listed_credentials_are_reached(): + config = {"a": {"b": {"c": {"password": "p"}}}, + "feeds": [{"name": "x", "api_key": "k"}, {"name": "y"}]} + out = _redact_credentials(config) + assert out["a"]["b"]["c"]["password"] == "" + assert out["feeds"][0]["api_key"] == "" + assert out["feeds"][0]["name"] == "x" + + +def test_the_original_is_not_mutated(): + """The caller holds the live config; redaction must not edit it in place.""" + config = {"github": {"api_token": "keepme"}} + _redact_credentials(config) + assert config["github"]["api_token"] == "keepme" + + +def test_a_credential_shaped_container_is_still_walked(): + """`secrets: {...}` is a section name, not a value to blank.""" + config = {"secrets": {"api_key": "k", "note": "keep"}} + out = _redact_credentials(config) + assert out["secrets"]["api_key"] == "" + assert out["secrets"]["note"] == "keep" + + +def test_non_dict_input_passes_through(): + assert _redact_credentials("plain") == "plain" + assert _redact_credentials(7) == 7 + assert _redact_credentials(None) is None + + +def test_the_endpoint_itself_redacts(): + """Through the view function, not the helper. + + The helper tests above all passed with the route still returning + `config` -- reverting the one line that calls the redactor changed + nothing, because nothing exercised the route. A property asserted on a + helper is not a property asserted on the endpoint, and it is the endpoint + that is exposed to the network. + """ + import json as _json + from unittest.mock import MagicMock + + import flask + + from web_interface.blueprints import api_v3 as mod + + raw = {"github": {"api_token": "ghp_secret_value"}, + "timezone": "America/New_York"} + + manager = MagicMock() + manager.load_config.return_value = raw + previous = getattr(mod.api_v3, "config_manager", None) + mod.api_v3.config_manager = manager + + app = flask.Flask(__name__) + try: + with app.test_request_context("/config/main"): + response = mod.get_main_config() + payload = response.get_json() if hasattr(response, "get_json") else _json.loads(response[0].data) + finally: + mod.api_v3.config_manager = previous + + data = payload["data"] + assert data["github"]["api_token"] == "", ( + "the endpoint returned the token; the redactor is not wired in") + assert data["timezone"] == "America/New_York" + # And the config the manager handed over is untouched. + assert raw["github"]["api_token"] == "ghp_secret_value" diff --git a/test/test_git_pull_resolution.py b/test/test_git_pull_resolution.py index d5668142..f7426d1f 100644 --- a/test/test_git_pull_resolution.py +++ b/test/test_git_pull_resolution.py @@ -58,7 +58,7 @@ def repos(tmp_path): def test_branch_with_upstream_uses_a_plain_pull(repos): args, note, error = resolve_pull_command(str(repos)) assert error is None - assert args == ['git', 'pull', '--rebase'] + assert args == ['git', 'pull', '--rebase', '--autostash'] assert note == '' @@ -73,7 +73,7 @@ def test_branch_without_upstream_falls_back_to_origin_branch(repos): args, note, error = resolve_pull_command(str(repos)) assert error is None - assert args == ['git', 'pull', '--rebase', 'origin', 'audit'] + assert args == ['git', 'pull', '--rebase', '--autostash', 'origin', 'audit'] assert 'audit' in note @@ -155,7 +155,7 @@ def test_switching_attaches_tracking_so_pull_needs_no_fallback(repos): args, note, error = resolve_pull_command(str(repos)) assert error is None - assert args == ['git', 'pull', '--rebase'] + assert args == ['git', 'pull', '--rebase', '--autostash'] assert note == '' @@ -200,3 +200,44 @@ def test_stash_option_lets_the_switch_through_and_keeps_the_work(repos): assert _git('branch', '--show-current', cwd=repos).stdout.strip() == 'other' # The edit is not lost — it is on the stash. assert 'switch to other' in _git('stash', 'list', cwd=repos).stdout + + +class TestInstallerDoesNotBlockTheUpdateButton: + """first_time_install.sh chmods scripts that git tracked as 644. + + With core.fileMode true -- the default on Linux -- that leaves five + permanently modified tracked files on every machine that ran the + installer, and `git pull --rebase` refuses to start: + + error: cannot pull with rebase: You have unstaged changes. + + Tracking them as executable makes the installer's chmod a no-op. + """ + + CHMODDED = [ + 'first_time_install.sh', + 'start_display.sh', + 'stop_display.sh', + 'scripts/install/install_service.sh', + 'scripts/install/install_web_service.sh', + ] + + def test_scripts_the_installer_chmods_are_tracked_executable(self): + import subprocess + from pathlib import Path + root = Path(__file__).resolve().parent.parent + out = subprocess.run(['git', 'ls-files', '-s', *self.CHMODDED], + capture_output=True, text=True, cwd=str(root)).stdout + modes = {line.split()[3]: line.split()[0] for line in out.strip().split('\n') if line} + # git ls-files says nothing about a path it does not track, so a + # renamed or deleted script would simply be absent here and the mode + # check below would pass over it silently. + untracked = sorted(set(self.CHMODDED) - set(modes)) + assert not untracked, ( + f"{untracked} are chmodded by the installer but not tracked by " + "git, so their mode cannot be asserted at all") + non_exec = sorted(f for f, m in modes.items() if m != '100755') + assert not non_exec, ( + f"{non_exec} are chmodded by the installer but tracked non-executable, " + "so every install leaves the working tree dirty and the update " + "button cannot pull") diff --git a/test/test_update_check_reports_failure.py b/test/test_update_check_reports_failure.py new file mode 100644 index 00000000..f638bb9d --- /dev/null +++ b/test/test_update_check_reports_failure.py @@ -0,0 +1,85 @@ +"""A check that could not run must not be reported as "up to date". + +check-update returned update_available=False whenever git failed. The banner +is the only route to the update button, so a checkout git refuses to touch +looked exactly like a current one -- permanently, and with nothing for the +user to act on. The usual cause is an install performed as root, after which +every git command fails with "detected dubious ownership". +""" +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest +from flask import Flask + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from web_interface.blueprints import api_v3 as mod # noqa: E402 +from web_interface.blueprints.api_v3 import api_v3 # noqa: E402 + +DUBIOUS = ("fatal: detected dubious ownership in repository at " + "'/home/pi/LEDMatrix'\nTo add an exception for this directory, call:\n" + "\tgit config --global --add safe.directory /home/pi/LEDMatrix\n") + + +@pytest.fixture +def client(): + app = Flask(__name__) + app.config['TESTING'] = True + app.register_blueprint(api_v3, url_prefix='/api/v3') + mod._update_check_cache['result'] = None + mod._update_check_cache['ts'] = 0 + return app.test_client() + + +def _fetch_fails(stderr: bytes): + def fake_run(args, **kwargs): + if args[:2] == ['git', 'fetch']: + return subprocess.CompletedProcess(args, 1, stdout=b'', stderr=stderr) + return subprocess.CompletedProcess(args, 0, stdout='', stderr='') + return fake_run + + +class TestFailedCheckIsNotSilence: + def test_dubious_ownership_is_reported_not_swallowed(self, client): + with patch.object(mod.subprocess, 'run', _fetch_fails(DUBIOUS.encode())): + data = client.get('/api/v3/system/check-update').get_json() + assert data['check_failed'] is True, ( + "a git failure was reported as a successful 'no update' check") + assert data['update_available'] is False + + def test_the_message_tells_the_user_what_to_do(self, client): + with patch.object(mod.subprocess, 'run', _fetch_fails(DUBIOUS.encode())): + data = client.get('/api/v3/system/check-update').get_json() + assert 'chown' in data['error'], ( + "dubious ownership is unactionable without the fix command") + assert 'root' in data['error'] + + def test_an_ordinary_git_failure_still_surfaces(self, client): + with patch.object(mod.subprocess, 'run', + _fetch_fails(b'fatal: some other git problem\n')): + data = client.get('/api/v3/system/check-update').get_json() + assert data['check_failed'] is True + assert 'some other git problem' in data['error'] + + def test_offline_reads_as_offline(self, client): + with patch.object(mod.subprocess, 'run', + _fetch_fails(b'fatal: could not resolve host: github.com\n')): + data = client.get('/api/v3/system/check-update').get_json() + assert 'Could not reach GitHub' in data['error'] + + +class TestSuccessPathUnchanged: + def test_up_to_date_carries_no_failure_flag(self, client): + def fake_run(args, **kwargs): + if args[:2] == ['git', 'fetch']: + return subprocess.CompletedProcess(args, 0, stdout=b'', stderr=b'') + if args[:2] == ['git', 'rev-parse']: + return subprocess.CompletedProcess(args, 0, stdout='abc123\n', stderr='') + return subprocess.CompletedProcess(args, 0, stdout='0\n', stderr='') + with patch.object(mod.subprocess, 'run', fake_run): + data = client.get('/api/v3/system/check-update').get_json() + assert data['update_available'] is False + assert not data.get('check_failed'), "a healthy check must not look like a failure" diff --git a/test/test_update_prompts_restart.py b/test/test_update_prompts_restart.py new file mode 100644 index 00000000..d7871e85 --- /dev/null +++ b/test/test_update_prompts_restart.py @@ -0,0 +1,84 @@ +"""A pull that changed nothing on the running system is not an applied update. + +git_pull replaces files on disk and restarts nothing -- there is no systemctl +call anywhere in the handler. The display and web services keep running the +code they loaded at boot, so the user is told "Code updated successfully" and +sees no change until they happen to reboot. The response now says whether a +restart is owed, and the UI raises the existing restart-pending banner. +""" +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest +from flask import Flask + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from web_interface.blueprints import api_v3 as mod # noqa: E402 +from web_interface.blueprints.api_v3 import api_v3 # noqa: E402 + + +@pytest.fixture +def client(): + app = Flask(__name__) + app.config['TESTING'] = True + app.register_blueprint(api_v3, url_prefix='/api/v3') + # The handler consults these after a successful pull; None is the + # "not wired up" case it already guards for. + api_v3.plugin_store_manager = None + api_v3.config_manager = None + return app.test_client() + + +def _git(heads, pull_rc=0, pull_out='Updating a1b2c3..d4e5f6\n'): + """Fake git. `heads` are the successive answers to rev-parse HEAD.""" + seq = list(heads) + + def run(args, **kwargs): + def ok(stdout='', rc=0, b=False): + return subprocess.CompletedProcess( + args, rc, stdout=(stdout.encode() if b else stdout), + stderr=(b'' if b else '')) + if args[:2] == ['git', 'rev-parse'] and args[-1] == 'HEAD': + return ok(seq.pop(0) + '\n' if seq else 'deadbeef\n') + if 'symbolic-full-name' in args or '@{u}' in args: + return ok('origin/main\n') + if args[:2] == ['git', 'status']: + return ok('') + if args[:2] == ['git', 'diff']: + return ok('') + if args[:2] == ['git', 'pull']: + return ok(pull_out, pull_rc) + return ok('') + return run + + +def _pull(client): + return client.post('/api/v3/system/action', + json={'action': 'git_pull'}).get_json() + + +class TestRestartIsRequestedWhenCodeChanged: + def test_a_pull_that_moved_head_asks_for_a_restart(self, client): + with patch.object(mod.subprocess, 'run', _git(['aaa111', 'bbb222'])): + data = _pull(client) + assert data['status'] == 'success' + assert data['restart_required'] is True, ( + "new code on disk, services still running the old code, and " + "nothing told the user to restart") + + def test_already_up_to_date_does_not(self, client): + with patch.object(mod.subprocess, 'run', + _git(['aaa111', 'aaa111'], pull_out='Already up to date.\n')): + data = _pull(client) + assert data['status'] == 'success' + assert data['restart_required'] is False, ( + "prompting after a no-op update trains users to ignore the prompt") + + def test_a_failed_pull_does_not(self, client): + with patch.object(mod.subprocess, 'run', _git(['aaa111'], pull_rc=1)): + data = _pull(client) + assert data['status'] == 'error' + assert data['restart_required'] is False diff --git a/test/web_interface/test_api_v3_secret_roundtrip.py b/test/web_interface/test_api_v3_secret_roundtrip.py index a4648824..1d70f009 100644 --- a/test/web_interface/test_api_v3_secret_roundtrip.py +++ b/test/web_interface/test_api_v3_secret_roundtrip.py @@ -194,15 +194,46 @@ class TestSavePluginConfig: def test_secret_count_message_counts_top_level_keys(self, env): # Pinned: the "(N secret field(s))" message counts TOP-LEVEL keys of - # the separated secrets dict. Here that is 2: the posted accounts - # array (all its item tokens count as ONE key) plus the schema's - # api_key default ("") that merge_with_defaults adds before - # separation. + # the separated secrets dict. Here that is 1: the posted accounts + # array, whose item tokens all count as ONE key. + # + # It was 2 before blank secrets were dropped, the second being the + # schema's api_key default (""), which merge_with_defaults adds to + # every save. Counting it was the visible edge of a real bug: that + # injected blank was merged over the stored api_key, so saving any + # unrelated field destroyed the credential. See + # test_an_unrelated_edit_does_not_erase_a_stored_secret. resp = self._save(env, { "accounts": [{"name": "a", "token": "t"}], }) message = resp.get_json()["message"] - assert "(2 secret field(s) saved to config_secrets.json)" in message + assert "(1 secret field(s) saved to config_secrets.json)" in message + + def test_an_unrelated_edit_does_not_erase_a_stored_secret(self, env): + """Editing one field must not wipe the plugin's API key. + + The config form renders secrets masked, so the browser posts them + back blank; merge_with_defaults injects a blank api_key even when + the client omits it entirely. Either way a "" reached the secrets + file and deep_merge wrote it over the stored credential. + """ + assert self._save(env, {"api_key": "REAL-KEY-0123456789", + "city": "Austin"}).status_code == 200 + assert _on_disk(env.secrets_file)[PLUGIN_ID]["api_key"] == \ + "REAL-KEY-0123456789" + + # the user changes the city; the masked api_key rides along blank + assert self._save(env, {"api_key": "", "city": "Dallas"}).status_code == 200 + + assert _on_disk(env.secrets_file)[PLUGIN_ID]["api_key"] == \ + "REAL-KEY-0123456789", "an unrelated edit destroyed the API key" + assert env.fresh_load()[PLUGIN_ID]["city"] == "Dallas" + + def test_a_secret_can_still_be_changed(self, env): + """Dropping blanks must not stop a real new value from being saved.""" + self._save(env, {"api_key": "first-key"}) + self._save(env, {"api_key": "second-key"}) + assert _on_disk(env.secrets_file)[PLUGIN_ID]["api_key"] == "second-key" def test_resave_replaces_stored_secrets_list_wholesale(self, env): # Characterized: api_v3's deep_merge intentionally replaces lists, diff --git a/test/web_interface/test_config_secrets_masking.py b/test/web_interface/test_config_secrets_masking.py new file mode 100644 index 00000000..421c8800 --- /dev/null +++ b/test/web_interface/test_config_secrets_masking.py @@ -0,0 +1,123 @@ +"""GET /config/secrets must not hand out credentials, and the client's +read-modify-write cycle must not destroy them. + +This interface has no authentication. The endpoint returned the whole +config_secrets.json to anyone who could reach the port; on one rig that was a +40-character GitHub token, a 183-character Home Assistant token and three API +keys. Masking it alone is not enough: the only client fetches every secret, +edits one field and posts all of them back, so the write path has to treat an +echoed mask as "unchanged". +""" +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from test_api_v3_secret_roundtrip import env, _on_disk # noqa: F401,E402 +from src.web_interface.secret_helpers import SECRET_MASK # noqa: E402 + +STORED = { + "github": {"api_token": "ghp_" + "x" * 36}, + "ledmatrix-weather": {"api_key": "w" * 32}, + "incoming-packages": {"ha_token": "h" * 183}, + "unset-plugin": {"api_key": ""}, + "placeholder-plugin": {"api_key": "YOUR_API_KEY_HERE"}, +} + + +def _seed(env): + env.secrets_file.write_text(json.dumps(STORED)) + + +def _get(env): + r = env.client.get("/api/v3/config/secrets") + assert r.status_code == 200, r.get_data(as_text=True)[:200] + return r.get_json()["data"] + + +def test_no_credential_leaves_the_process(env): + _seed(env) + body = json.dumps(_get(env)) + for secret in ("ghp_" + "x" * 36, "w" * 32, "h" * 183): + assert secret not in body, "endpoint returned a stored credential" + + +def test_set_and_unset_remain_distinguishable(env): + _seed(env) + data = _get(env) + assert data["github"]["api_token"] == SECRET_MASK + assert data["unset-plugin"]["api_key"] == "" + assert data["placeholder-plugin"]["api_key"] == "YOUR_API_KEY_HERE" + + +def test_the_clients_read_modify_write_preserves_every_other_secret(env): + """What the GitHub-token save button actually does.""" + _seed(env) + secrets = _get(env) # everything arrives masked + secrets["github"]["api_token"] = "ghp_" + "n" * 36 # user changes one + r = env.client.post("/api/v3/config/raw/secrets", json=secrets) + assert r.status_code == 200, r.get_data(as_text=True)[:200] + + on_disk = _on_disk(env.secrets_file) + assert on_disk["github"]["api_token"] == "ghp_" + "n" * 36, "new token not saved" + assert on_disk["ledmatrix-weather"]["api_key"] == "w" * 32 + assert on_disk["incoming-packages"]["ha_token"] == "h" * 183 + + +def test_a_mask_echoed_back_is_never_stored(env): + _seed(env) + # Assert the write succeeded. A 500 leaves the old file in place, so the + # assertions below would hold without the write path running at all. + resp = env.client.post("/api/v3/config/raw/secrets", json=_get(env)) + assert resp.status_code == 200, resp.get_data(as_text=True)[:200] + on_disk = _on_disk(env.secrets_file) + assert SECRET_MASK not in json.dumps(on_disk), "the mask was stored as a secret" + assert on_disk["github"]["api_token"] == "ghp_" + "x" * 36 + + +def test_a_brand_new_secret_can_still_be_added(env): + _seed(env) + env.client.post("/api/v3/config/raw/secrets", + json={"new-plugin": {"api_key": "brand-new"}}) + on_disk = _on_disk(env.secrets_file) + assert on_disk["new-plugin"]["api_key"] == "brand-new" + assert on_disk["github"]["api_token"] == "ghp_" + "x" * 36 + + +def test_a_list_of_secrets_keeps_its_shape(env): + """A list must not be masked as though it were one scalar. + + accounts: [{...}, {...}] came back as a single '••••••••', so a caller + could not see how many entries existed, and the raw editor was shown a + string where the file holds an array. + """ + env.secrets_file.write_text(json.dumps({ + "myplugin": {"accounts": [{"name": "a", "token": "tok-a"}, + {"name": "b", "token": "tok-b"}]}})) + accounts = _get(env)["myplugin"]["accounts"] + assert isinstance(accounts, list), "the list was flattened to a scalar" + assert len(accounts) == 2, "entries were lost" + assert all(isinstance(a, dict) for a in accounts), "entry shape was lost" + assert "tok-a" not in json.dumps(accounts), "a token survived masking" + + +def test_a_list_posted_back_unchanged_is_left_alone(env): + """Lists merge by replacement, so a half-masked list must not be stored.""" + original = {"myplugin": {"accounts": [{"name": "a", "token": "tok-a"}, + {"name": "b", "token": "tok-b"}]}} + env.secrets_file.write_text(json.dumps(original)) + resp = env.client.post("/api/v3/config/raw/secrets", json=_get(env)) + assert resp.status_code == 200, resp.get_data(as_text=True)[:200] + assert _on_disk(env.secrets_file)["myplugin"]["accounts"] == \ + original["myplugin"]["accounts"], "round-tripping the mask damaged the list" + + +def test_a_fully_supplied_list_still_saves(env): + env.secrets_file.write_text(json.dumps( + {"myplugin": {"accounts": [{"name": "a", "token": "old"}]}})) + body = _get(env) + body["myplugin"]["accounts"] = [{"name": "a", "token": "new"}] + resp = env.client.post("/api/v3/config/raw/secrets", json=body) + assert resp.status_code == 200, resp.get_data(as_text=True)[:200] + assert _on_disk(env.secrets_file)["myplugin"]["accounts"][0]["token"] == "new" diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 40dde29c..37df1141 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -21,7 +21,9 @@ logger = logging.getLogger(__name__) # Import new infrastructure from src.web_interface.api_helpers import success_response, error_response, validate_request_json from src.web_interface.errors import ErrorCode -from src.web_interface.secret_helpers import find_secret_fields, separate_secrets +from src.web_interface.secret_helpers import (find_secret_fields, mask_all_secret_values, + remove_empty_secrets, separate_secrets, + strip_masked_values) from src.web_interface.error_handler import describe_exception, redact_text from src.plugin_system.operation_types import OperationType from src.web_interface.validators import ( @@ -262,15 +264,54 @@ def _stop_display_service(): result['status'] = status return result +#: Field names whose value is a credential. Matched by name because this +#: endpoint returns the whole config, core keys included, and core config has +#: no schema to carry x-secret markers. +_CREDENTIAL_NAME_PARTS = ("password", "passwd", "secret", "token", "api_key", + "apikey", "access_key", "private_key", "client_secret") + + +def _looks_like_a_credential(name: str) -> bool: + lowered = name.lower() + return any(part in lowered for part in _CREDENTIAL_NAME_PARTS) + + +def _redact_credentials(value): + """A copy of `value` with credential-named fields blanked. + + /config/main returned the raw config to anyone who could reach the port, + and this interface has no authentication. On one rig that meant a 40-char + GitHub token, a 183-char Home Assistant token and five API keys were + readable by anything on the LAN. + + The x-secret masking used by the plugin config endpoints does not help + here: this endpoint never consults a schema, and core keys such as + github.api_token have no schema to mark. Matching on the field name is + blunt, but for a whole-config dump the right default is that anything + named like a credential does not leave the process. + + Blanked rather than removed, and safe to blank: POST /config/main merges + into the loaded config and only writes the keys it was given, so a client + that round-trips this response cannot erase a secret it never saw. + """ + if isinstance(value, dict): + return {k: ("" if _looks_like_a_credential(k) and not isinstance(v, (dict, list)) + else _redact_credentials(v)) + for k, v in value.items()} + if isinstance(value, list): + return [_redact_credentials(item) for item in value] + return value + + @api_v3.route('/config/main', methods=['GET']) def get_main_config(): - """Get main configuration""" + """Get main configuration, with credentials redacted.""" try: if not api_v3.config_manager: return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 config = api_v3.config_manager.load_config() - return jsonify({'status': 'success', 'data': config}) + return jsonify({'status': 'success', 'data': _redact_credentials(config)}) except Exception as e: logger.error('Unhandled exception', exc_info=True) return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @@ -715,10 +756,12 @@ def save_main_config(): if not data: return jsonify({'status': 'error', 'message': 'No data provided'}), 400 - import logging - logging.error(f"DEBUG: save_main_config received data: {data}") - logging.error(f"DEBUG: Content-Type header: {request.content_type}") - logging.error(f"DEBUG: Headers: {dict(request.headers)}") + # What arrives here is the config itself, and the headers carry the + # session cookie -- neither belongs in the journal, least of all at + # ERROR on every save. The shape of the request is the part with + # diagnostic value, so log that, at the level it deserves. + logger.debug("save_main_config: %s, %d top-level key(s)", + request.content_type or 'no content-type', len(data)) # Merge with existing config (similar to original implementation) current_config = api_v3.config_manager.load_config() @@ -1216,6 +1259,11 @@ def save_main_config(): # Separate secrets from regular config (same logic as save_plugin_config) regular_config, secrets_config = separate_secrets(plugin_config, secret_fields) + # The config form renders secrets masked, so every save posts + # them back blank. Without this the blank is merged over the + # stored value and the credential is destroyed by the act of + # changing an unrelated setting. A blank means "unchanged". + secrets_config = remove_empty_secrets(secrets_config) # PRE-PROCESSING: Preserve 'enabled' state if not in regular_config # This prevents overwriting the enabled state when saving config from a form that doesn't include the toggle @@ -1333,7 +1381,12 @@ def get_secrets_config(): return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 config = api_v3.config_manager.get_raw_file_content('secrets') - return jsonify({'status': 'success', 'data': config}) + # This interface has no authentication, and this file is nothing but + # credentials. It was handing all of them to anyone who could reach + # the port. Values are masked; empty and YOUR_* placeholders are left + # alone so a client can still tell "set" from "not set". + return jsonify({'status': 'success', + 'data': mask_all_secret_values(config)}) except Exception as e: logger.error('Unhandled exception', exc_info=True) return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @@ -1401,8 +1454,19 @@ def save_raw_secrets_config(): if not data: return jsonify({'status': 'error', 'message': 'No data provided'}), 400 - # Save the secrets config - api_v3.config_manager.save_raw_file_content('secrets', data) + # The GET above masks what it returns, and this endpoint's only client + # reads the whole file, edits one field and posts all of it back. So + # most of what arrives here is the mask, echoed rather than changed -- + # storing it verbatim would replace every untouched credential with + # eight bullets. Strip those, then merge onto what is already stored, + # which makes "unchanged" mean unchanged. + # + # The cost is that a secret can no longer be cleared by blanking it. + # That needs its own affordance; a control that erases credentials as + # a side effect of saving an unrelated one is not it. + current = api_v3.config_manager.get_raw_file_content('secrets') or {} + merged = deep_merge(current, strip_masked_values(data)) + api_v3.config_manager.save_raw_file_content('secrets', merged) # Reload GitHub token in plugin store manager if it exists if api_v3.plugin_store_manager: @@ -1660,13 +1724,22 @@ def resolve_pull_command(project_dir): backup, or following an install guide that names one. The update button then reports a failure the user cannot act on. + ``--autostash`` is passed for the same reason. Rebase refuses to start + when any tracked file is modified, and on these installs something always + is: first_time_install.sh chmods five scripts that git tracked as 644, so + every machine that ran the installer carries five permanent mode changes + and the update button reports "cannot pull with rebase: You have unstaged + changes". Those modes are corrected in this commit, but a user cannot pull + the correction while the pull is what is blocked, and any other local edit + would reproduce it anyway. Autostash reapplies the changes afterwards. + Returns ``(args, note, error)``. When ``origin/`` exists the pull is made explicit against it, so the update proceeds and the branch is given tracking information afterwards. """ upstream = _git_upstream(project_dir) if upstream: - return ['git', 'pull', '--rebase'], '', None + return ['git', 'pull', '--rebase', '--autostash'], '', None branch = _git_current_branch(project_dir) if not branch: @@ -1676,7 +1749,7 @@ def resolve_pull_command(project_dir): ) if _git_remote_branch_exists(project_dir, branch): return ( - ['git', 'pull', '--rebase', 'origin', branch], + ['git', 'pull', '--rebase', '--autostash', 'origin', branch], f"Branch '{branch}' had no upstream; pulled from origin/{branch} and set it as the upstream.", None, ) @@ -1824,6 +1897,33 @@ def get_system_version(): _update_check_cache: Dict[str, Any] = {'result': None, 'ts': 0.0} _UPDATE_CHECK_TTL = 300 # 5 minutes — avoids a git fetch on every page load +def _update_check_failed(detail: str) -> Dict[str, Any]: + """A check that could not run is not the same as being up to date. + + Reporting update_available=False on a git failure hides the banner, and + the banner is the only route to the update button -- so a checkout git + refuses to touch looks exactly like a current one, permanently. The most + common cause is an install performed as root: git then reports "dubious + ownership" and every command fails, including the fetch here. + """ + return {'update_available': False, 'remote_sha': 'unknown', + 'commits_behind': 0, 'check_failed': True, 'error': detail} + + +def _describe_git_failure(stderr: str) -> str: + """Turn git's stderr into something the user can act on.""" + text = (stderr or '').strip() + if 'dubious ownership' in text or 'detected dubious ownership' in text: + return ("This checkout is owned by a different user than the one " + "running the web interface, so git refuses to use it. It is " + "usually the result of installing as root. Fix the ownership " + "and the update will work: sudo chown -R $USER:$USER " + + str(PROJECT_ROOT)) + if 'could not resolve host' in text.lower() or 'network is unreachable' in text.lower(): + return "Could not reach GitHub to check for updates." + return "Could not check for updates: " + (text.splitlines()[0] if text else "git failed") + + @api_v3.route('/system/check-update', methods=['GET']) def check_for_update(): """Check whether a newer LEDMatrix commit is available on origin/main.""" @@ -1839,12 +1939,13 @@ def check_for_update(): capture_output=True, timeout=10, cwd=cwd, ) if fetch_result.returncode != 0: + stderr = fetch_result.stderr.decode(errors='replace').strip() logger.warning("check-update: git fetch failed (rc=%d): %s", - fetch_result.returncode, - fetch_result.stderr.decode(errors='replace').strip()) - _update_check_cache['result'] = _safe + fetch_result.returncode, stderr) + failed = _update_check_failed(_describe_git_failure(stderr)) + _update_check_cache['result'] = failed _update_check_cache['ts'] = now - return jsonify(_safe) + return jsonify(failed) local = subprocess.run( ['git', 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=5, cwd=cwd, @@ -1872,7 +1973,8 @@ def check_for_update(): return jsonify(result) except Exception as e: logger.warning("check-update failed: %s", e) - return jsonify(_safe) + return jsonify(_update_check_failed( + "Could not check for updates; see logs for details.")) @api_v3.route('/system/action', methods=['POST']) def execute_system_action(): @@ -1999,6 +2101,11 @@ def execute_system_action(): except subprocess.TimeoutExpired: logger.warning("git rev-parse timed out before pull") + # Whether the pull actually brought new code in. "Already up to + # date" is a success too, and prompting for a restart then would + # train users to ignore the prompt. + code_changed = False + # Perform the git pull. Branches without an upstream were given # an explicit "origin " above so the update still works. result = subprocess.run( @@ -2042,6 +2149,7 @@ def execute_system_action(): capture_output=True, text=True, timeout=10, cwd=project_dir) new_head = _post.stdout.strip() if _post.returncode == 0 else None if old_head and new_head and old_head != new_head: + code_changed = True diff = subprocess.run( ['git', 'diff', '--name-only', f'{old_head}..{new_head}'], capture_output=True, text=True, timeout=15, cwd=project_dir) @@ -2101,9 +2209,14 @@ def execute_system_action(): if ln.strip()), '') pull_message = f"Update failed: {detail}" if detail else "Update failed; check logs for details" + # Nothing here restarts anything: the pull replaces files on + # disk while the display and web services keep running the code + # they loaded at boot. Without this the user is told the update + # succeeded and sees no change until they happen to reboot. return jsonify({ 'status': 'success' if result.returncode == 0 else 'error', 'message': pull_message, + 'restart_required': bool(result.returncode == 0 and code_changed), }) elif action == 'checkout_branch': # Switch branches from the Tools tab. Needed because a checkout @@ -5617,6 +5730,11 @@ def save_plugin_config(): # Separate secrets from regular config (handles nested configs and # array-item secrets — see src/web_interface/secret_helpers.py) regular_config, secrets_config = separate_secrets(plugin_config, secret_fields) + # The config form renders secrets masked, so every save posts + # them back blank. Without this the blank is merged over the + # stored value and the credential is destroyed by the act of + # changing an unrelated setting. A blank means "unchanged". + secrets_config = remove_empty_secrets(secrets_config) # Get current configs current_config = api_v3.config_manager.load_config() diff --git a/web_interface/static/v3/app.js b/web_interface/static/v3/app.js index 82b2a9e5..581c66f2 100644 --- a/web_interface/static/v3/app.js +++ b/web_interface/static/v3/app.js @@ -116,14 +116,25 @@ document.body.addEventListener('htmx:afterRequest', function(event) { // ===== Restart-pending banner ===== // Shown after restart-requiring saves; persists across tab switches (and // reloads, via sessionStorage) until the display restarts or it's dismissed. -window.showRestartPending = function() { - try { sessionStorage.setItem('ledmatrix-restart-pending', '1'); } catch { /* private browsing */ } +window.showRestartPending = function(message) { + try { + sessionStorage.setItem('ledmatrix-restart-pending', '1'); + // Persisted alongside the flag: a code update and a config save want + // different wording, and the banner outlives the page that raised it. + if (message) sessionStorage.setItem('ledmatrix-restart-pending-text', message); + else sessionStorage.removeItem('ledmatrix-restart-pending-text'); + } catch { /* private browsing */ } const banner = document.getElementById('restart-pending-banner'); + const text = document.getElementById('restart-pending-text'); + if (text && message) text.textContent = message; if (banner) banner.style.display = 'block'; }; window.dismissRestartPending = function() { - try { sessionStorage.removeItem('ledmatrix-restart-pending'); } catch { /* no-op */ } + try { + sessionStorage.removeItem('ledmatrix-restart-pending'); + sessionStorage.removeItem('ledmatrix-restart-pending-text'); + } catch { /* no-op */ } const banner = document.getElementById('restart-pending-banner'); if (banner) banner.style.display = 'none'; }; @@ -151,6 +162,9 @@ document.addEventListener('DOMContentLoaded', function() { try { if (sessionStorage.getItem('ledmatrix-restart-pending') === '1') { const banner = document.getElementById('restart-pending-banner'); + const saved = sessionStorage.getItem('ledmatrix-restart-pending-text'); + const text = document.getElementById('restart-pending-text'); + if (text && saved) text.textContent = saved; if (banner) banner.style.display = 'block'; } } catch { /* no-op */ } diff --git a/web_interface/static/v3/plugins_manager.js b/web_interface/static/v3/plugins_manager.js index 8bca1821..ea020998 100644 --- a/web_interface/static/v3/plugins_manager.js +++ b/web_interface/static/v3/plugins_manager.js @@ -4622,15 +4622,17 @@ window.loadGithubToken = function() { // Handle empty data (secrets file doesn't exist) - API returns {} in this case const secrets = data.data || {}; const token = secrets.github?.api_token || ''; + const configured = token && token !== 'YOUR_GITHUB_PERSONAL_ACCESS_TOKEN'; if (input) { - if (token && token !== 'YOUR_GITHUB_PERSONAL_ACCESS_TOKEN') { - // Token exists and is valid - input.value = token; - showNotification('GitHub token loaded successfully', 'success'); + // The endpoint masks what it returns, so this never holds + // the real token -- and the field is deliberately left + // empty rather than filled with the mask, which would be + // saved verbatim the next time the user pressed Save. + input.value = ''; + if (configured) { + showNotification('A GitHub token is saved. Enter a new one to replace it.', 'success'); } else { - // No token configured or placeholder value - input.value = ''; showNotification('No GitHub token configured. Enter a new token to save.', 'info'); } } diff --git a/web_interface/templates/v3/base.html b/web_interface/templates/v3/base.html index fb4cfa6a..e739dd8c 100644 --- a/web_interface/templates/v3/base.html +++ b/web_interface/templates/v3/base.html @@ -413,7 +413,8 @@
- + Configuration saved — restart the display to apply the changes
@@ -1107,15 +1108,29 @@ fetch('/api/v3/system/check-update') .then(function(r) { return r.json(); }) .then(function(data) { + var banner = document.getElementById('update-banner'); + var btn = document.getElementById('update-banner-btn'); + if (data.check_failed) { + // A check that could not run is not the same as being up + // to date. Hiding the banner here made a checkout git + // refuses to touch look permanently current, with no + // route to the update button and nothing to act on. + document.getElementById('update-banner-text').textContent = + data.error || 'Could not check for updates.'; + if (btn) btn.style.display = 'none'; + banner.style.display = ''; + return; + } + if (btn) btn.style.display = ''; if (data.update_available && getDismissedSha() !== data.remote_sha) { var n = data.commits_behind || 0; var msg = 'A new LEDMatrix update is available'; if (n > 0) msg += ' (' + n + ' commit' + (n > 1 ? 's' : '') + ')'; document.getElementById('update-banner-text').textContent = msg; - document.getElementById('update-banner').style.display = ''; + banner.style.display = ''; try { sessionStorage.setItem('update-sha', data.remote_sha); } catch(e) {} } else { - document.getElementById('update-banner').style.display = 'none'; + banner.style.display = 'none'; } }) .catch(function() {}); @@ -1146,6 +1161,13 @@ if (data.status === 'success') { document.getElementById('update-banner').style.display = 'none'; try { sessionStorage.removeItem('update-sha-dismissed'); } catch(e) {} + // The pull replaced files on disk; the running services still + // hold the code they loaded at boot. Ask for the restart that + // makes the update actually take effect. + if (data.restart_required && typeof window.showRestartPending === 'function') { + window.showRestartPending( + 'Update installed \u2014 restart the display to run the new code'); + } } if (typeof showNotification === 'function') { showNotification(data.message || 'Update complete', data.status || 'success'); From 863e4a1ecd5f6eb0ce80a1b72c400594afa0651e Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:50:55 -0400 Subject: [PATCH 33/53] consolidate(perf): cut SD writes, log volume, and metrics churn (#486) * fix(plugins): one bad metrics cache entry should not stop every plugin Caught live on a rig: every plugin failing, once each, continuously. ERROR - src.plugin_system.plugin_manager - plugin geochron operation failed: ResourceMetrics.__init__() got an unexpected keyword argument 'consecutive_failures' ERROR - ... plugin text-display operation failed: ... ERROR - ... plugin news operation failed: ... ERROR - ... plugin odds-ticker operation failed: ... with /api/v3/health reporting plugin_system: not_initialized while the display process itself kept running and updating the panel. `consecutive_failures` is a plugin_health field, not a metrics one. get_metrics() does ResourceMetrics(**cached), which raises TypeError on a single unrecognised key, and that exception escapes into plugin_manager and is reported per plugin. One malformed cache entry takes the whole plugin system down. How a health-shaped record came to sit under a plugin_metrics key on that machine is not established, and I could not finish the diagnosis: the rig went back into its EIO failure mode partway through -- SSH resetting pre-banner, systemctl unexecutable -- while the web API kept answering from RAM. Checked before that: the cache files on disk are correctly shaped and separate, and CacheManager.get() returns the right record for each key, so it is not a live key collision. A restored backup mixing two machines' caches is the likeliest explanation, and that rig had one restored onto it. Either way the loader should not be brittle enough for the answer to matter. plugin_health already repairs its records field by field rather than trusting what is on disk; this does the same. Known fields are kept, unknown ones are dropped and named once in the log so a genuine schema change stays visible rather than being silently discarded, and a non-mapping entry no longer raises. Keeping the known fields matters: discarding the record wholesale would throw away real call counts and timings because of an unrelated stray key. Mutation-checked: restoring ResourceMetrics(**cached) fails 6 checks, dropping the whole record fails the field-preservation check, and dropping unknown fields silently fails the logging check. 28 tests pass across the resource monitor and plugin health suites. * perf(health): stop rewriting a health record on every healthy cycle Every successful plugin update called record_success(), which persisted the record unconditionally. In steady state the only fields that had changed were total_successes and last_success_time -- a counter and a timestamp that health_monitor surfaces for display and that nothing reads back after a restart. Nothing alerts on the age of last_successful_update; it is carried in the metrics dataclass and shown. Measured on a rig running 24 plugins, all steady-state (0 consecutive failures, circuit closed): a five-minute sample caught 22 health-file rewrites, about 4.4 a minute or 6,300 a day. Each write is ~400 bytes through cache_manager.set(), which writes a file per call, so each one costs a filesystem block plus an ext4 journal write. That lands on an SD card, where the unit of cost is an erase-block cycle rather than the bytes involved, and where wear is what eventually kills the card. Two cards have already failed on the other rig with the same signature -- unreadable block device, EIO on exec, sshd unable to read its host keys. The circuit breaker still has to survive a restart, so the write is kept for exactly the fields it is rebuilt from: consecutive_failures, circuit_state, circuit_opened_time, half_open_start_time. A failure, a circuit opening and a recovery are all still written the moment they happen. In-memory state is updated every time either way, so the health API and web UI show what they always did. Tested: 100 healthy cycles now perform zero writes after the first, the counters remain accurate in memory, and a failure, a recovery and a half-open-to-closed transition each still reach disk. One test kills and rebuilds the tracker from the cache to prove the breaker's state genuinely survives what is no longer written. Mutation-checked both ways: persisting unconditionally again fails the steady-state test, and widening _DURABLE_FIELDS to include last_success_time fails it too. The 46 existing health tests pass. (cherry picked from commit 14abea2d24df454f70ecff1044f6b79f93be63bc) (cherry picked from commit 0f77bd2345427a994a5e2d736304b2dd44c9e264) * perf(vegas): trace the content path at DEBUG instead of INFO plugin_adapter narrates every step of acquiring content from every plugin -- "Has get_vegas_content", "Native: calling get_vegas_content()", "Native content returned None", "Has scroll_helper", per-item sizes -- once per plugin per cycle, all at INFO. Measured on a live rig: 13,408 log lines an hour, of which 13,366 were INFO and 35 were WARNING. Roughly 223 lines a minute of string formatting on a Pi that is also driving the panel, written through journald to the SD card, with the 35 lines that actually indicate a problem buried among them. Top repeated messages in that hour: 717 Scroll progress: elapsed=... total_scrolled=.../... px 399 [plugin] --> INCLUDED in Vegas scroll 323 [plugin] content_type=static, display_mode=fixed 195 [plugin] Has get_vegas_content: True 195 [plugin] Native: calling get_vegas_content() 168 [plugin] Native: get_vegas_content() returned None 168 [plugin] Native content returned None <- the same fact, twice 54 logger.info calls in plugin_adapter become logger.debug, along with the per-frame scroll-progress line in scroll_helper. Together those are 3,174 of the 13,408 lines an hour, a 23% cut, and the ~3,600 odds-manager lines are addressed separately by ledmatrix-plugins#300. Nothing is lost: the 19 warning/error/exception calls in the module are untouched, so real failures still surface at their own level. This is a logging-level change only -- no control flow, no behaviour. One INFO call is deliberate and stays. The padding-strip message picks its level at runtime (`logger.warning if (left and right) else logger.info`) and test_vegas_plugin_adapter.py pins that choice; it survives because it is not a direct logger.info call site. That test still passes. Mutation-checked both ways: reintroducing a single INFO trace fails the guard, and demoting the warning/error calls along with the trace fails a second guard written for exactly that mistake. 537 vegas and scroll tests pass. (cherry picked from commit e496d95dfe7709e5637b6118be5695cfbab025fb) (cherry picked from commit 8d1e43c15aaf68ae6de32f2be7a9ff30e39871d9) * fix(logging): give the journal the real severity of each line Everything this process writes to stdout reaches the journal as PRIORITY=6, whatever the Python level was, because journald has nothing else to go on. Measured on a live rig over 24 hours: lines containing " - ERROR - " 55 lines containing " - WARNING - " 13 journald PRIORITY recorded 6, for every one of them So `journalctl -p err -u ledmatrix` returns nothing while errors are being logged, and `-p warning` likewise. Triage falls back to grepping message text, which is slower and unreliable: during this audit a search for "oom" matched the radar logging "zoom=9" twenty-four times and briefly looked like the OOM killer had been firing. systemd reads a leading "" on each stdout line and takes it as the priority (sd-daemon(3)), so a formatter that prefixes one costs no dependency. Every line of a multi-line record is tagged, not just the first -- the journal splits them, and an untagged continuation reverts to the default, which would leave the body of a traceback filed as informational while its first line was an error. Applied only when JOURNAL_STREAM is set, which systemd sets for services whose output it captures. Run from a terminal, in the emulator or under pytest the prefixes would be literal noise, and the file handler keeps the plain formatter for the same reason. Mutation-checked three ways: prefixing unconditionally fails the outside-systemd test, prefixing only the first line fails the multi-line test, and mapping ERROR to 6 fails the level mapping. 39 tests pass across the logging suites. (cherry picked from commit 780fca6365052a7767fbcd458da6baacb0456229) * fix(logging): let callers see through the journald formatter wrapper CI caught what local testing could not: two existing tests in test_logging_config.py assert that setup_logging() selected a StructuredFormatter or a ContextualFormatter, by checking the console handler's formatter directly. Wrapping that formatter to tag each line with its syslog priority makes those assertions false. They passed locally and failed on the runner because the wrapper is applied only when JOURNAL_STREAM is set -- absent in a terminal, present in CI. An environment-dependent break, which is the kind that gets shipped. The wrapper now exposes the formatter it delegates to, and those two tests look through it. They are about which formatter format_type selects, and that behaviour is unchanged; only the object they have to reach for moved. Verified both ways this time: 39 tests pass with JOURNAL_STREAM set and with it unset. * perf(plugins): stop rewriting a plugin's metrics file on every call Plugin metrics were persisted to the cache inside monitor_call, so every call by every plugin rewrote a small JSON file. Measured on a running rig: one plugin's plugin_metrics file changed nine times a minute, with fourteen such files active. Each is around 350 bytes, which on ext4 costs a 4KB block plus a journal entry, so the cost is dominated by the write itself rather than the payload. Cache writes accounted for essentially all of that device's 2.4 MB/min of SD traffic, on a card that wears out and has already failed twice on the other rig. Metrics cannot be de-duplicated the way health state can, because call_count changes on every call and the timings usually do too. So they are rate-limited instead: at most one write per plugin per 30 seconds. The in-memory copy stays authoritative and exact -- a plugin's call_count is still precise the instant after it runs. Only the cross-process snapshot the web UI reads is delayed, and telemetry up to half a minute old is still a fair description of a long-running plugin. reset_metrics clears the throttle timestamp, so a reset is not left showing a deleted key for the rest of the interval. Extrapolating the sampled rate, this takes metric writes from roughly 126 a minute to 28. Health persistence, the other half of the churn, is handled separately in #475. Verified by reverting the throttle: the churn test then reports 50 writes for 50 calls. 88 tests pass across resource monitor, plugin system and web API. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix: use a monotonic clock and only mark metrics persisted once written Two review findings on the throttle, both right. The interval compared wall-clock timestamps. These devices have no RTC, so the clock jumps by however far off boot-time was the moment NTP first syncs -- a forward jump would allow an early write, a backward one would stall the snapshot well past the interval. time.monotonic() is not subject to either. The timestamp was also recorded before cache_manager.set(). A set() that raised would buy the next interval's silence without leaving a snapshot behind, which is the one case where skipping the write is least affordable. Recorded after the write lands instead, so a failure is retried on the next call. Verified by restoring the original ordering: the new test then reports one write where two are expected. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * Address all six review findings on the perf consolidation CodeRabbit reported six; this is all six, checked against its own "Actionable comments posted: 6" rather than against what I happened to scroll past. Three are real defects in the code: 1. _under_systemd() trusted the presence of JOURNAL_STREAM. systemd publishes JOURNAL_STREAM as "dev:ino", and every child process inherits it -- including one whose stdout has been redirected to a pipe or a file. The variable outlives the descriptor it describes, so a subprocess would decide it was talking to the journal and emit the "" priority prefixes as literal noise into that captured output. That is exactly the noise the function exists to prevent. It now parses the pair and fstats stdout, per systemd's own guidance, and returns False for missing, malformed, mismatched, or unusable descriptors. 2. Cached metrics were not type-checked. A dataclass does not enforce its annotations, so ResourceMetrics(call_count="not a number") builds happily and only fails later, deep inside monitor_call: TypeError: can only concatenate str (not "int") to str Values are now coerced to their declared type at load, where there is still a cache key to name in the warning, and a value that cannot be coerced starts the plugin fresh instead of arming a delayed failure. A numeric string is accepted rather than discarded -- a JSON round-trip can widen an int, and that is recoverable. 3. The first metrics snapshot was skipped for the first 30s of uptime. _persist_metrics used 0.0 as the "never written" default. monotonic() is time since boot on Linux and systemd starts this service at boot, so `now - 0.0 < 30` was true for the first half-minute of every run: the throttle swallowed the very first write, the one that matters most after a restart. The sentinel is now None and the interval is only applied when a previous write exists. Three are tests that could pass without testing anything: 4. test_health_write_churn's fake cache stored by reference, so the tracker kept mutating the object already in the store -- a record could look persisted when no write had happened, which is precisely what test_durable_state_survives_a_restart exists to detect. Both directions now deep-copy, like a cache that serialises to a file. Verified: disabling the one real cache write now fails three tests. 5. test_values_of_the_wrong_type_do_not_raise asserted only that a dataclass had been constructed, which was true with the bad value still in it. It now asserts the loaded metrics are usable -- the field is numeric, and arithmetic on it does not raise -- across four kinds of bad value. 6. test_vegas_log_volume counted "logger.error(" in the source text, which also matches comments, docstrings and string literals -- including that module's own docstring, which names those levels. A real error call could be demoted with the tally unmoved. It now walks the AST, reusing the helper already in the file. Verified: demoting all 18 warning/error/exception calls now fails the test. Verified: every fix mutation-checked by reverting it and confirming the matching test fails. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 (1M context) --- src/common/scroll_helper.py | 2 +- src/logging_config.py | 85 +++++++++++++- src/plugin_system/plugin_health.py | 29 ++++- src/plugin_system/resource_monitor.py | 137 +++++++++++++++++++--- src/vegas_mode/plugin_adapter.py | 108 ++++++++--------- test/test_health_write_churn.py | 119 +++++++++++++++++++ test/test_journald_log_priority.py | 137 ++++++++++++++++++++++ test/test_logging_config.py | 20 +++- test/test_metrics_cache_unknown_fields.py | 128 ++++++++++++++++++++ test/test_resource_monitor.py | 86 +++++++++++++- test/test_vegas_log_volume.py | 73 ++++++++++++ 11 files changed, 845 insertions(+), 79 deletions(-) create mode 100644 test/test_health_write_churn.py create mode 100644 test/test_journald_log_priority.py create mode 100644 test/test_metrics_cache_unknown_fields.py create mode 100644 test/test_vegas_log_volume.py diff --git a/src/common/scroll_helper.py b/src/common/scroll_helper.py index 4f2e215e..88c6d498 100644 --- a/src/common/scroll_helper.py +++ b/src/common/scroll_helper.py @@ -328,7 +328,7 @@ class ScrollHelper: elapsed_time = current_time - (self.scroll_start_time or current_time) # The image already includes display_width padding, so we only need total_scroll_width required_total_distance = self.total_scroll_width - self.logger.info( + self.logger.debug( "Scroll progress: elapsed=%.2fs, target=%.2fs, total_scrolled=%.0f/%d px (%.1f%%)", elapsed_time, self.calculated_duration, diff --git a/src/logging_config.py b/src/logging_config.py index 17d831bb..843257eb 100644 --- a/src/logging_config.py +++ b/src/logging_config.py @@ -130,7 +130,12 @@ def setup_logging( # Console handler (always add) console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(level) - console_handler.setFormatter(formatter) + # Under systemd, tag each line so the journal records the real severity + # rather than filing everything as informational. The file handler below + # keeps the plain formatter: the prefix is meaningful to journald and noise + # anywhere else. + console_handler.setFormatter( + JournalPriorityFormatter(formatter) if _under_systemd() else formatter) root_logger.addHandler(console_handler) # File handler (if specified) @@ -145,6 +150,84 @@ def setup_logging( sys.stderr.write(f"Warning: Could not set up file logging to {log_file}: {e}\n") +#: syslog priorities, which is what systemd parses from a "" prefix on +#: stdout. Mapped from Python's levels. +_SYSLOG_PRIORITY = { + logging.CRITICAL: 2, # LOG_CRIT + logging.ERROR: 3, # LOG_ERR + logging.WARNING: 4, # LOG_WARNING + logging.INFO: 6, # LOG_INFO + logging.DEBUG: 7, # LOG_DEBUG +} + + +class JournalPriorityFormatter(logging.Formatter): + """Wraps a formatter, prefixing each line with its syslog priority. + + Under systemd everything this process writes to stdout lands in the journal + as PRIORITY=6, whatever the Python level was. Measured on a live rig: 55 + ERROR lines and 13 WARNING lines in a day, every one of them recorded as + informational, so `journalctl -p err -u ledmatrix` returned nothing at all + while errors were being logged. Anyone triaging has to grep the message + text instead, which is both slower and wrong -- a search for "oom" matches + the radar logging "zoom=9". + + systemd reads a leading "" on each line and uses it as the priority + (sd-daemon(3)), so this needs no extra dependency. Multi-line records get + the prefix on every line, since the journal splits them and an unprefixed + continuation would fall back to the default. + """ + + def __init__(self, inner: logging.Formatter): + super().__init__() + self._inner = inner + + @property + def inner(self) -> logging.Formatter: + """The formatter doing the actual work. + + Whether journald tagging is applied depends on JOURNAL_STREAM, so it is + on under systemd and off in a terminal -- and anything asserting which + formatter setup_logging() selected would otherwise get a different + answer in CI than on a developer's machine. Exposing the inner one lets + those checks stay about format_type, which is what they mean. + """ + return self._inner + + def format(self, record: logging.LogRecord) -> str: + text = self._inner.format(record) + prefix = f"<{_SYSLOG_PRIORITY.get(record.levelno, 6)}>" + return "\n".join(prefix + line for line in text.split("\n")) + + +def _under_systemd() -> bool: + """True when stdout really is the journal. + + systemd sets JOURNAL_STREAM to "dev:ino" for services whose output it + captures. Presence alone is not enough to act on: the variable is + inherited by child processes and survives redirection, so a subprocess + whose stdout is a pipe or a file still sees it and would emit the "" + priority prefixes as literal noise into that output. systemd's own + guidance is to fstat the descriptor and compare st_dev/st_ino, which is + what distinguishes "the journal is somewhere in my ancestry" from "my + stdout is the journal". + """ + declared = os.environ.get("JOURNAL_STREAM") + if not declared: + return False + try: + dev_text, ino_text = declared.split(":", 1) + declared_ids = (int(dev_text), int(ino_text)) + except (ValueError, AttributeError): + return False + try: + stat_result = os.fstat(sys.stdout.fileno()) + except (OSError, ValueError, AttributeError): + # No usable stdout: captured by pytest, detached, or already closed. + return False + return (stat_result.st_dev, stat_result.st_ino) == declared_ids + + class PluginLoggerAdapter(logging.LoggerAdapter): """LoggerAdapter that stamps every record with its plugin_id. diff --git a/src/plugin_system/plugin_health.py b/src/plugin_system/plugin_health.py index d9b4f2c5..3908d9ca 100644 --- a/src/plugin_system/plugin_health.py +++ b/src/plugin_system/plugin_health.py @@ -178,11 +178,21 @@ class PluginHealthTracker: ) return self._health_state[plugin_id] + # Fields the circuit breaker is rebuilt from after a restart. Everything + # else in a health record is reporting, read only for display. + _DURABLE_FIELDS = ('consecutive_failures', 'circuit_state', + 'circuit_opened_time', 'half_open_start_time') + + def _durable(self, state: Dict[str, Any]) -> tuple: + """The part of a health record whose loss would change behaviour.""" + return tuple(state.get(field) for field in self._DURABLE_FIELDS) + def record_success(self, plugin_id: str) -> None: """Record a successful plugin execution.""" state = self.get_health_state(plugin_id) current_time = time.time() - + durable_before = self._durable(state) + # Reset consecutive failures state['consecutive_failures'] = 0 state['total_successes'] = state.get('total_successes', 0) + 1 @@ -198,9 +208,20 @@ class PluginHealthTracker: # Shouldn't happen, but handle it state['circuit_state'] = CircuitState.CLOSED.value state['circuit_opened_time'] = None - - self._save_health_state(plugin_id, state) - + + # A healthy plugin reports success every cycle, and in that steady state + # the only fields changed above are a counter and a timestamp that + # nothing reads back after a restart. Persisting them anyway rewrites a + # small file per plugin per cycle: on a rig running 24 plugins, a + # five-minute sample measured 22 rewrites, about 4.4 a minute or 6,300 a + # day. Those land on an SD card, where the cost is an erase-block cycle + # rather than the 400 bytes involved, and where wear is what eventually + # kills the card. + # In-memory state is still updated every time, so the health API and web + # UI show exactly what they did before; only the write is skipped. + if self._durable(state) != durable_before: + self._save_health_state(plugin_id, state) + def record_failure(self, plugin_id: str, error: Optional[Exception] = None) -> None: """Record a failed plugin execution.""" state = self.get_health_state(plugin_id) diff --git a/src/plugin_system/resource_monitor.py b/src/plugin_system/resource_monitor.py index 42ded4ec..1b63d74e 100644 --- a/src/plugin_system/resource_monitor.py +++ b/src/plugin_system/resource_monitor.py @@ -9,7 +9,7 @@ import time import logging import threading from typing import Dict, Optional, Any, Callable -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields try: import psutil @@ -49,6 +49,20 @@ class ResourceMetrics: self.total_execution_time = self.total_execution_time / self.call_count +#: How often a plugin's metrics are written to the cache, in seconds. +#: +#: Persisting on every call meant a small file rewritten roughly nine times a +#: minute per plugin. On a rig with fourteen active plugins that was ~126 +#: writes a minute for metrics alone, and since each ~350-byte file costs a +#: 4KB block plus an ext4 journal entry, it dominated the device's write +#: volume -- on an SD card, which wears out. +#: +#: The in-memory copy stays authoritative and exact; only the cross-process +#: snapshot the web UI reads is delayed, and telemetry up to half a minute old +#: is still a fair description of a long-running plugin. +_METRICS_PERSIST_INTERVAL = 30.0 + + class PluginResourceMonitor: """ Monitors resource usage for plugins. @@ -75,6 +89,10 @@ class PluginResourceMonitor: # Resource metrics per plugin self._metrics: Dict[str, ResourceMetrics] = {} self._limits: Dict[str, ResourceLimits] = {} + # When each plugin's metrics last reached the cache. Metrics change on + # every call, so they cannot be de-duplicated the way health state can; + # they are rate-limited instead. See _METRICS_PERSIST_INTERVAL. + self._metrics_persisted_at: Dict[str, float] = {} # Thread-local storage for execution tracking self._local = threading.local() @@ -102,6 +120,66 @@ class PluginResourceMonitor: "psutil not available - resource monitoring will be limited to execution time only" ) + def _metrics_from_cache(self, plugin_id: str, cached: Any) -> "ResourceMetrics": + """Build metrics from a cached record, ignoring anything unrecognised. + + ResourceMetrics(**cached) raises TypeError on a single unexpected key, + and that exception escapes into plugin_manager, which reports it as + "plugin operation failed". Every plugin fails, and the plugin + system never finishes initialising. + + Seen on a live rig: every plugin failing with + + ResourceMetrics.__init__() got an unexpected keyword argument + 'consecutive_failures' + + which is a plugin_health field, not a metrics one. How a health-shaped + record came to sit under a plugin_metrics key on that machine is not + established -- a restored backup that mixed two machines' caches is the + likeliest explanation -- but the loader should not be brittle enough for + it to matter. plugin_health already repairs its records field by field + rather than trusting whatever is on disk; this does the same. + + Unknown keys are dropped and named once, so a genuine schema change is + visible in the log instead of silently discarded. + """ + if not isinstance(cached, dict): + self.logger.warning( + "Ignoring cached metrics for %s: expected a mapping, got %s", + plugin_id, type(cached).__name__) + return ResourceMetrics() + + known = {f.name for f in fields(ResourceMetrics)} + unknown = sorted(set(cached) - known) + if unknown: + self.logger.warning( + "Dropping unrecognised field(s) from cached metrics for %s: %s", + plugin_id, ", ".join(unknown)) + # A dataclass does not enforce its annotations, so + # ResourceMetrics(call_count="not a number") builds happily and only + # blows up later, deep inside monitor_call ("can only concatenate str + # (not \"int\") to str"). Coerce here, where there is still a cache + # key to name in the warning. + declared = {f.name: f.type for f in fields(ResourceMetrics)} + usable = {} + for key, value in cached.items(): + if key not in known: + continue + try: + usable[key] = int(value) if declared[key] in ('int', int) else float(value) + except (TypeError, ValueError): + self.logger.warning( + "Cached metrics for %s have a bad %s (%r); starting fresh", + plugin_id, key, value) + return ResourceMetrics() + try: + return ResourceMetrics(**usable) + except (TypeError, ValueError) as e: + self.logger.warning( + "Cached metrics for %s unusable (%s); starting fresh", + plugin_id, e) + return ResourceMetrics() + def _get_metrics_key(self, plugin_id: str) -> str: """Get cache key for plugin metrics.""" return f"plugin_metrics:{plugin_id}" @@ -126,7 +204,7 @@ class PluginResourceMonitor: cache_key, max_age=None, memory_ttl=0 if force_reload else None ) if cached: - metrics = ResourceMetrics(**cached) + metrics = self._metrics_from_cache(plugin_id, cached) else: metrics = ResourceMetrics() self._metrics[plugin_id] = metrics @@ -232,18 +310,8 @@ class PluginResourceMonitor: # CPU is harder to measure per-call, so we track it separately metrics.cpu_percent = self._get_process_cpu_percent() - # Persist metrics - cache_key = self._get_metrics_key(plugin_id) - self.cache_manager.set(cache_key, { - 'memory_mb': metrics.memory_mb, - 'cpu_percent': metrics.cpu_percent, - 'execution_time': metrics.execution_time, - 'call_count': metrics.call_count, - 'total_execution_time': metrics.total_execution_time, - 'max_execution_time': metrics.max_execution_time, - 'min_execution_time': metrics.min_execution_time if metrics.min_execution_time != float('inf') else 0.0, - 'last_update_time': metrics.last_update_time - }) + # Persist metrics, at most once per interval per plugin. + self._persist_metrics(plugin_id, metrics) # Check limits if limits: @@ -363,6 +431,44 @@ class PluginResourceMonitor: summaries[plugin_id] = self.get_metrics_summary(plugin_id) return summaries + def _persist_metrics(self, plugin_id: str, metrics: ResourceMetrics, + force: bool = False) -> None: + """Write a plugin's metrics to the cache, at most once per interval. + + Caller must hold ``self._lock``. + """ + # Monotonic, not wall clock: these devices have no RTC, so the clock + # jumps by however far off boot-time was the moment NTP first syncs. + # A forward jump would allow an early write, a backward one would + # stall the snapshot well past the interval. + # + # The sentinel for "never written" is None, not 0.0. monotonic() is + # time since boot on Linux, and systemd starts this service *at* boot, + # so `now - 0.0 < 30` was true for the first half-minute of every + # single run -- the throttle swallowed the very first snapshot, which + # is the one that matters most after a restart. + now = time.monotonic() + last_written = self._metrics_persisted_at.get(plugin_id) + if (not force and last_written is not None + and now - last_written < _METRICS_PERSIST_INTERVAL): + return + cache_key = self._get_metrics_key(plugin_id) + self.cache_manager.set(cache_key, { + 'memory_mb': metrics.memory_mb, + 'cpu_percent': metrics.cpu_percent, + 'execution_time': metrics.execution_time, + 'call_count': metrics.call_count, + 'total_execution_time': metrics.total_execution_time, + 'max_execution_time': metrics.max_execution_time, + 'min_execution_time': (metrics.min_execution_time + if metrics.min_execution_time != float('inf') + else 0.0), + 'last_update_time': metrics.last_update_time, + }) + # Only after the write lands. Marking it first would mean a failed + # set() bought the next interval's silence without leaving a snapshot. + self._metrics_persisted_at[plugin_id] = now + def reset_metrics(self, plugin_id: str) -> None: """Reset metrics for a plugin.""" with self._lock: @@ -370,4 +476,7 @@ class PluginResourceMonitor: self._metrics[plugin_id] = ResourceMetrics() cache_key = self._get_metrics_key(plugin_id) self.cache_manager.delete(cache_key) + # Let the next call persist immediately rather than leaving the + # deleted key absent for the rest of the interval. + self._metrics_persisted_at.pop(plugin_id, None) diff --git a/src/vegas_mode/plugin_adapter.py b/src/vegas_mode/plugin_adapter.py index 0ea71571..01ef8cca 100644 --- a/src/vegas_mode/plugin_adapter.py +++ b/src/vegas_mode/plugin_adapter.py @@ -83,7 +83,7 @@ class PluginAdapter: # into unrelated headlines once the strip refreshed to 9,505px. self._offset_shapes: dict = {} - logger.info( + logger.debug( "PluginAdapter initialized: display=%dx%d", self.display_width, self.display_height ) @@ -109,7 +109,7 @@ class PluginAdapter: Returns: List of PIL Images representing plugin content, or None if no content """ - logger.info( + logger.debug( "[%s] Getting content (class=%s)", plugin_id, plugin.__class__.__name__ ) @@ -118,7 +118,7 @@ class PluginAdapter: cached = self._get_cached(plugin_id) if cached is not None: total_width = sum(img.width for img in cached) - logger.info( + logger.debug( "[%s] Using cached content: %d images, %dpx total", plugin_id, len(cached), total_width ) @@ -126,46 +126,46 @@ class PluginAdapter: # Try native Vegas content method first has_native = hasattr(plugin, 'get_vegas_content') - logger.info("[%s] Has get_vegas_content: %s", plugin_id, has_native) + logger.debug("[%s] Has get_vegas_content: %s", plugin_id, has_native) if has_native: content = self._get_native_content(plugin, plugin_id, offscreen_only) if content: total_width = sum(img.width for img in content) - logger.info( + logger.debug( "[%s] Native content SUCCESS: %d images, %dpx total", plugin_id, len(content), total_width ) return self._finalize(content, plugin_id, 'native', plugin) - logger.info("[%s] Native content returned None", plugin_id) + logger.debug("[%s] Native content returned None", plugin_id) # Try to get scroll_helper's cached image (for scrolling plugins like stocks/odds) has_scroll_helper = hasattr(plugin, 'scroll_helper') - logger.info("[%s] Has scroll_helper: %s", plugin_id, has_scroll_helper) + logger.debug("[%s] Has scroll_helper: %s", plugin_id, has_scroll_helper) content = self._get_scroll_helper_content(plugin, plugin_id, offscreen_only) if content: total_width = sum(img.width for img in content) - logger.info( + logger.debug( "[%s] ScrollHelper content SUCCESS: %d images, %dpx total", plugin_id, len(content), total_width ) return self._finalize(content, plugin_id, 'scroll_helper', plugin) if has_scroll_helper: - logger.info("[%s] ScrollHelper content returned None", plugin_id) + logger.debug("[%s] ScrollHelper content returned None", plugin_id) if offscreen_only: # Display capture needs the shared canvas; leave it to the caller. - logger.info( + logger.debug( "[%s] Needs display capture, deferring to the render thread", plugin_id ) return None # Fall back to display capture - logger.info("[%s] Trying fallback display capture...", plugin_id) + logger.debug("[%s] Trying fallback display capture...", plugin_id) content = self._capture_display_content(plugin, plugin_id) if content: total_width = sum(img.width for img in content) - logger.info( + logger.debug( "[%s] Fallback capture SUCCESS: %d images, %dpx total", plugin_id, len(content), total_width ) @@ -226,7 +226,7 @@ class PluginAdapter: kept.append(result.image) if not kept: - logger.info( + logger.debug( "[%s] All %d image(s) from %s were blank — contributing nothing", plugin_id, len(images), source ) @@ -235,14 +235,14 @@ class PluginAdapter: trimmed_width = sum(img.width for img in kept) if trimmed_width < self.config.min_plugin_width: - logger.info( + logger.debug( "[%s] Trimmed content %dpx is below min_plugin_width %dpx — skipping", plugin_id, trimmed_width, self.config.min_plugin_width ) return None if trimmed_width != original_width or dropped_blank: - logger.info( + logger.debug( "[%s] Trimmed %s content: %dpx -> %dpx (%.0f%% reclaimed), " "%d image(s) kept, %d blank dropped", plugin_id, source, original_width, trimmed_width, @@ -431,7 +431,7 @@ class PluginAdapter: """ if self._offset_shapes.get(plugin_id) != shape: if plugin_id in self._item_offsets: - logger.info( + logger.debug( "[%s] Content is %s now, was %s — restarting the rotation " "rather than resuming at a position that no longer means " "anything", plugin_id, shape, @@ -579,7 +579,7 @@ class PluginAdapter: consumed += 1 if mode == 'truncate': - logger.info( + logger.debug( "[%s] Width budget %dpx: showing the first %d of %d row(s) " "(%dpx incl. gaps); the rest are not shown (overflow=truncate)", plugin_id, budget, len(selected), len(images), used @@ -587,7 +587,7 @@ class PluginAdapter: else: self._record_offset( plugin_id, (start + consumed) % len(images), shape) - logger.info( + logger.debug( "[%s] Width budget %dpx: showing %d of %d row(s) (%dpx incl. gaps) " "from offset %d; remainder deferred to a later cycle", plugin_id, budget, len(selected), len(images), used, start @@ -636,7 +636,7 @@ class PluginAdapter: if mode != 'truncate': self._record_offset( plugin_id, 0 if end >= img.width else end, shape) - logger.info( + logger.debug( "[%s] Width budget %dpx: cropped continuous %dpx image to " "[%d:%d] (no item gaps of %dpx+ to align to)%s", plugin_id, budget, img.width, offset, end, min_run, @@ -674,7 +674,7 @@ class PluginAdapter: self._record_offset( plugin_id, 0 if end >= img.width else end_index, shape) - logger.info( + logger.debug( "[%s] Width budget %dpx: cropped single %dpx image to [%d:%d] " "(%dpx) at item boundaries %d-%d of %d, %s", plugin_id, budget, img.width, start, end, end - start, @@ -698,7 +698,7 @@ class PluginAdapter: List of images or None """ try: - logger.info("[%s] Native: calling get_vegas_content()", plugin_id) + logger.debug("[%s] Native: calling get_vegas_content()", plugin_id) # Tell the plugin how much width the ticker wants it to use, and # narrow the canvas for the duration of the call. A plugin that @@ -707,7 +707,7 @@ class PluginAdapter: # be explicit can read get_vegas_render_width(). render_width = self.resolve_render_width(plugin, plugin_id) if render_width != self.display_width: - logger.info( + logger.debug( "[%s] Native: requesting %dpx instead of %dpx", plugin_id, render_width, self.display_width ) @@ -735,19 +735,19 @@ class PluginAdapter: plugin._vegas_render_width = None if result is None: - logger.info("[%s] Native: get_vegas_content() returned None", plugin_id) + logger.debug("[%s] Native: get_vegas_content() returned None", plugin_id) return None # Normalize to list if isinstance(result, Image.Image): images = [result] - logger.info( + logger.debug( "[%s] Native: got single Image %dx%d", plugin_id, result.width, result.height ) elif isinstance(result, (list, tuple)): images = list(result) - logger.info( + logger.debug( "[%s] Native: got %d items in list/tuple", plugin_id, len(images) ) @@ -768,14 +768,14 @@ class PluginAdapter: ) continue - logger.info( + logger.debug( "[%s] Native: item[%d] is %dx%d, mode=%s", plugin_id, i, img.width, img.height, img.mode ) # Ensure correct height if img.height != self.display_height: - logger.info( + logger.debug( "[%s] Native: resizing item[%d]: %dx%d -> %dx%d", plugin_id, i, img.width, img.height, img.width, self.display_height @@ -793,13 +793,13 @@ class PluginAdapter: if valid_images: total_width = sum(img.width for img in valid_images) - logger.info( + logger.debug( "[%s] Native: SUCCESS - %d images, %dpx total width", plugin_id, len(valid_images), total_width ) return valid_images - logger.info("[%s] Native: no valid images after validation", plugin_id) + logger.debug("[%s] Native: no valid images after validation", plugin_id) return None except (AttributeError, TypeError, ValueError, OSError) as e: @@ -833,20 +833,20 @@ class PluginAdapter: logger.debug("[%s] No scroll_helper attribute", plugin_id) return None - logger.info( + logger.debug( "[%s] Found scroll_helper: %s", plugin_id, type(scroll_helper).__name__ ) cached_image = getattr(scroll_helper, 'cached_image', None) if cached_image is None: - logger.info( + logger.debug( "[%s] scroll_helper.cached_image is None, triggering content generation", plugin_id ) if offscreen_only: # Generating it calls display(), which needs the canvas. - logger.info( + logger.debug( "[%s] scroll_helper cache empty; deferring generation " "to the render thread", plugin_id ) @@ -859,13 +859,13 @@ class PluginAdapter: return None if not isinstance(cached_image, Image.Image): - logger.info( + logger.debug( "[%s] scroll_helper.cached_image is not an Image: %s", plugin_id, type(cached_image).__name__ ) return None - logger.info( + logger.debug( "[%s] scroll_helper.cached_image found: %dx%d, mode=%s", plugin_id, cached_image.width, cached_image.height, cached_image.mode ) @@ -888,7 +888,7 @@ class PluginAdapter: # Ensure correct height if img.height != self.display_height: - logger.info( + logger.debug( "[%s] Resizing scroll_helper content: %dx%d -> %dx%d", plugin_id, img.width, img.height, img.width, self.display_height @@ -902,7 +902,7 @@ class PluginAdapter: if img.mode != 'RGB': img = img.convert('RGB') - logger.info( + logger.debug( "[%s] ScrollHelper content ready: %dx%d", plugin_id, img.width, img.height ) @@ -1002,7 +1002,7 @@ class PluginAdapter: with self._capture(): # Method 1: Try _create_scrolling_display (stocks pattern) if hasattr(plugin, '_create_scrolling_display'): - logger.info( + logger.debug( "[%s] Triggering via _create_scrolling_display()", plugin_id ) @@ -1010,7 +1010,7 @@ class PluginAdapter: plugin._create_scrolling_display() cached_image = getattr(scroll_helper, 'cached_image', None) if cached_image is not None and isinstance(cached_image, Image.Image): - logger.info( + logger.debug( "[%s] _create_scrolling_display() SUCCESS: %dx%d", plugin_id, cached_image.width, cached_image.height ) @@ -1022,7 +1022,7 @@ class PluginAdapter: # Method 2: Try display(force_clear=True) which typically builds scroll content if hasattr(plugin, 'display'): - logger.info( + logger.debug( "[%s] Triggering via display(force_clear=True)", plugin_id ) @@ -1031,12 +1031,12 @@ class PluginAdapter: plugin.display(force_clear=True) cached_image = getattr(scroll_helper, 'cached_image', None) if cached_image is not None and isinstance(cached_image, Image.Image): - logger.info( + logger.debug( "[%s] display(force_clear=True) SUCCESS: %dx%d", plugin_id, cached_image.width, cached_image.height ) return cached_image - logger.info( + logger.debug( "[%s] display(force_clear=True) did not populate cached_image", plugin_id ) @@ -1045,7 +1045,7 @@ class PluginAdapter: "[%s] display(force_clear=True) failed", plugin_id ) - logger.info( + logger.debug( "[%s] Could not trigger scroll content generation", plugin_id ) @@ -1077,15 +1077,15 @@ class PluginAdapter: try: # Save current display state original_image = self.display_manager.image.copy() - logger.info("[%s] Fallback: saved original display state", plugin_id) + logger.debug("[%s] Fallback: saved original display state", plugin_id) # Ensure plugin has fresh data before capturing has_update_data = hasattr(plugin, 'update_data') - logger.info("[%s] Fallback: has update_data=%s", plugin_id, has_update_data) + logger.debug("[%s] Fallback: has update_data=%s", plugin_id, has_update_data) if has_update_data: try: plugin.update_data() - logger.info("[%s] Fallback: update_data() called", plugin_id) + logger.debug("[%s] Fallback: update_data() called", plugin_id) except (AttributeError, RuntimeError, OSError): logger.exception("[%s] Fallback: update_data() failed", plugin_id) @@ -1097,41 +1097,41 @@ class PluginAdapter: # arrangement rather than one that has to be cropped afterwards. render_width = self.resolve_render_width(plugin, plugin_id) if render_width != self.display_width: - logger.info( + logger.debug( "[%s] Fallback: rendering at %dpx instead of %dpx", plugin_id, render_width, self.display_width ) with self._capture(), self._render_at(render_width): self.display_manager.clear() - logger.info("[%s] Fallback: display cleared, calling display()", plugin_id) + logger.debug("[%s] Fallback: display cleared, calling display()", plugin_id) # First try without force_clear (some plugins behave better this way) try: plugin.display() - logger.info("[%s] Fallback: display() called successfully", plugin_id) + logger.debug("[%s] Fallback: display() called successfully", plugin_id) except TypeError: # Plugin may require force_clear argument - logger.info("[%s] Fallback: display() failed, trying with force_clear=True", plugin_id) + logger.debug("[%s] Fallback: display() failed, trying with force_clear=True", plugin_id) plugin.display(force_clear=True) # Capture the result captured = self.display_manager.image.copy() - logger.info( + logger.debug( "[%s] Fallback: captured frame %dx%d, mode=%s", plugin_id, captured.width, captured.height, captured.mode ) # Check if captured image has content (not all black) is_blank, bright_ratio = self._is_blank_image(captured, return_ratio=True) - logger.info( + logger.debug( "[%s] Fallback: brightness check - %.3f%% bright pixels (threshold=0.5%%)", plugin_id, bright_ratio * 100 ) if is_blank: - logger.info( + logger.debug( "[%s] Fallback: first capture blank, retrying with force_clear", plugin_id ) @@ -1142,7 +1142,7 @@ class PluginAdapter: captured = self.display_manager.image.copy() is_blank, bright_ratio = self._is_blank_image(captured, return_ratio=True) - logger.info( + logger.debug( "[%s] Fallback: retry brightness - %.3f%% bright pixels", plugin_id, bright_ratio * 100 ) @@ -1159,7 +1159,7 @@ class PluginAdapter: if captured.mode != 'RGB': captured = captured.convert('RGB') - logger.info( + logger.debug( "[%s] Fallback: SUCCESS - captured %dx%d", plugin_id, captured.width, captured.height ) diff --git a/test/test_health_write_churn.py b/test/test_health_write_churn.py new file mode 100644 index 00000000..883ce51a --- /dev/null +++ b/test/test_health_write_churn.py @@ -0,0 +1,119 @@ +"""A healthy plugin must not rewrite its health record every cycle. + +Every successful plugin update called record_success(), which persisted the +record unconditionally. In steady state the only fields that had changed were +total_successes and last_success_time -- a counter and a timestamp that +health_monitor reads for display and that nothing reads back after a restart. + +Measured on a rig running 24 plugins: about 17 health-file rewrites a minute, +roughly 25,000 a day. Each is ~400 bytes, but they land on an SD card where +the unit of cost is an erase-block cycle, not the byte count, and where wear is +what eventually kills the card. + +The circuit breaker still needs its own state to survive a restart, so the +write is kept for exactly the fields it is rebuilt from -- and a failure, a +circuit opening, or a recovery must still be written the moment it happens. +""" +import time + +import copy + +import pytest + +from src.plugin_system.plugin_health import PluginHealthTracker, CircuitState + + +class _Cache: + """Counts writes; serves back whatever was last written. + + Both directions deep-copy, so this behaves like a real cache that + serialises through a file. Storing by reference let the tracker keep + mutating the object already in the store, so a record could appear to + have been persisted when no write ever happened -- which is precisely + what test_durable_state_survives_a_restart is supposed to detect. + """ + + def __init__(self): + self.store = {} + self.writes = 0 + + def set(self, key, data, ttl=None, **kwargs): + self.writes += 1 + self.store[key] = copy.deepcopy(data) + + def get(self, key, max_age=None, memory_ttl=None, **kwargs): + return copy.deepcopy(self.store.get(key)) + + +@pytest.fixture +def tracker(): + cache = _Cache() + t = PluginHealthTracker(cache_manager=cache) + return t, cache + + +def test_steady_state_success_stops_writing(tracker): + """The regression: 100 healthy cycles used to be 100 SD writes.""" + t, cache = tracker + t.record_success("weather") + first = cache.writes + for _ in range(100): + t.record_success("weather") + assert cache.writes == first, ( + f"{cache.writes - first} redundant writes across 100 healthy cycles" + ) + + +def test_the_counters_are_still_accurate_in_memory(tracker): + """Skipping the write must not skip the bookkeeping.""" + t, _ = tracker + for _ in range(10): + t.record_success("weather") + state = t.get_health_state("weather") + assert state["total_successes"] == 10 + assert state["last_success_time"] is not None + assert state["last_success_time"] <= time.time() + + +def test_a_failure_is_written_immediately(tracker): + t, cache = tracker + t.record_success("weather") + before = cache.writes + t.record_failure("weather", RuntimeError("boom")) + assert cache.writes > before, "a failure must reach disk" + + +def test_recovery_after_failure_is_written(tracker): + """consecutive_failures returning to 0 is durable state changing.""" + t, cache = tracker + t.record_failure("weather", RuntimeError("boom")) + before = cache.writes + t.record_success("weather") + assert cache.writes > before, "recovery must reach disk" + assert t.get_health_state("weather")["consecutive_failures"] == 0 + + +def test_a_closing_circuit_is_written(tracker): + """Success in half-open closes the circuit -- that must survive a restart.""" + t, cache = tracker + state = t.get_health_state("weather") + state["circuit_state"] = CircuitState.HALF_OPEN.value + state["half_open_start_time"] = time.time() + before = cache.writes + t.record_success("weather") + assert cache.writes > before, "a circuit transition must reach disk" + assert t.get_health_state("weather")["circuit_state"] == CircuitState.CLOSED.value + + +def test_durable_state_survives_a_restart(tracker): + """What is skipped must genuinely not matter to the breaker.""" + t, cache = tracker + for _ in range(3): + t.record_failure("weather", RuntimeError("boom")) + for _ in range(50): + t.record_success("weather") + + revived = PluginHealthTracker(cache_manager=cache) + state = revived.get_health_state("weather") + assert state["consecutive_failures"] == 0 + assert state["circuit_state"] == CircuitState.CLOSED.value diff --git a/test/test_journald_log_priority.py b/test/test_journald_log_priority.py new file mode 100644 index 00000000..386b024c --- /dev/null +++ b/test/test_journald_log_priority.py @@ -0,0 +1,137 @@ +"""Log lines must reach the journal with their real severity. + +Everything this process writes to stdout lands in the journal as PRIORITY=6, +whatever the Python level was, because journald has no other signal. Measured +on a live rig over 24 hours: 55 lines containing " - ERROR - " and 13 +containing " - WARNING - ", every one of them recorded as informational. So + + journalctl -p err -u ledmatrix + +returned nothing while errors were being logged, and anyone triaging has to +grep the message text instead. That is slower and it is wrong: a search for +"oom" also matches the radar logging "zoom=9", which is exactly the false +positive it produced during this audit. + +systemd reads a leading "" on each stdout line and uses it as the priority +(sd-daemon(3)), so this needs no extra dependency -- and it must only be +applied when systemd is actually reading, or the prefixes become literal noise +in a terminal, the emulator, and test output. +""" +import logging +import os +import sys +from unittest.mock import patch + +import pytest + +from src.logging_config import JournalPriorityFormatter, _SYSLOG_PRIORITY, _under_systemd + + +class _Plain(logging.Formatter): + def format(self, record): + return record.getMessage() + + +def _record(level, msg="hello"): + return logging.LogRecord("t", level, "f.py", 1, msg, None, None) + + +@pytest.mark.parametrize("level,expected", [ + (logging.CRITICAL, 2), + (logging.ERROR, 3), + (logging.WARNING, 4), + (logging.INFO, 6), + (logging.DEBUG, 7), +]) +def test_each_level_maps_to_its_syslog_priority(level, expected): + out = JournalPriorityFormatter(_Plain()).format(_record(level)) + assert out.startswith(f"<{expected}>"), out + assert _SYSLOG_PRIORITY[level] == expected + + +def test_error_and_info_are_distinguishable(): + """The whole point: journalctl -p err must be able to tell them apart.""" + fmt = JournalPriorityFormatter(_Plain()) + assert fmt.format(_record(logging.ERROR))[:3] != fmt.format(_record(logging.INFO))[:3] + + +def test_every_line_of_a_multiline_record_is_tagged(): + """The journal splits them, and an untagged continuation loses its level. + + A traceback is the case that matters -- it is the most important thing in + the log and the longest. + """ + out = JournalPriorityFormatter(_Plain()).format( + _record(logging.ERROR, "Traceback:\nline one\nline two")) + lines = out.split("\n") + assert len(lines) == 3 + assert all(line.startswith("<3>") for line in lines), lines + + +def test_the_message_survives_intact(): + out = JournalPriorityFormatter(_Plain()).format(_record(logging.WARNING, "disk full")) + assert out == "<4>disk full" + + +def test_an_unknown_level_falls_back_to_info(): + out = JournalPriorityFormatter(_Plain()).format(_record(25)) + assert out.startswith("<6>") + + +def _stdout_ids(): + """The dev:ino systemd would publish for this process's stdout.""" + st = os.fstat(sys.stdout.fileno()) + return f"{st.st_dev}:{st.st_ino}" + + +def test_prefixing_is_off_outside_systemd(): + """Otherwise a terminal run, the emulator and pytest all show `<6>`.""" + with patch.dict(os.environ, {}, clear=True): + assert not _under_systemd() + with patch.dict(os.environ, {"JOURNAL_STREAM": _stdout_ids()}): + assert _under_systemd() + + +def test_an_inherited_journal_stream_does_not_count(): + """The variable outlives the descriptor it describes. + + systemd sets JOURNAL_STREAM for the service, and every child inherits it + -- including one whose stdout has been redirected to a pipe or a file. + Trusting the variable alone put literal "<6>" prefixes into that captured + output. Only a descriptor whose dev:ino actually matches is the journal. + """ + with patch.dict(os.environ, {"JOURNAL_STREAM": "8:12345"}): + assert not _under_systemd(), \ + "a stale inherited JOURNAL_STREAM was treated as the journal" + + +@pytest.mark.parametrize("value", ["", "not-a-pair", "8", "8:", ":12345", + "eight:12345", "8:12345:9"]) +def test_a_malformed_journal_stream_is_not_the_journal(value): + with patch.dict(os.environ, {"JOURNAL_STREAM": value}): + assert not _under_systemd() + + +def test_a_closed_stdout_is_not_the_journal(): + """os.fstat raises rather than answers; that must not propagate.""" + with patch.dict(os.environ, {"JOURNAL_STREAM": "8:12345"}), \ + patch("src.logging_config.sys.stdout") as fake_stdout: + fake_stdout.fileno.side_effect = ValueError("I/O operation on closed file") + assert not _under_systemd() + + +def test_setup_uses_the_wrapper_only_under_systemd(): + from src.logging_config import setup_logging + + for env, expect_wrapped in (({}, False), + ({"JOURNAL_STREAM": _stdout_ids()}, True)): + with patch.dict(os.environ, env, clear=True): + setup_logging() + handlers = [h for h in logging.getLogger().handlers + if isinstance(h, logging.StreamHandler)] + assert handlers, "no stream handler installed" + wrapped = any(isinstance(h.formatter, JournalPriorityFormatter) + for h in handlers) + assert wrapped is expect_wrapped, ( + f"JOURNAL_STREAM={env}: wrapped={wrapped}, expected {expect_wrapped}") + logging.getLogger().handlers.clear() diff --git a/test/test_logging_config.py b/test/test_logging_config.py index e0fe7fff..d0cf0fe8 100644 --- a/test/test_logging_config.py +++ b/test/test_logging_config.py @@ -183,15 +183,27 @@ class TestSetupLogging: setup_logging() assert len(logging.getLogger().handlers) == 1 + @staticmethod + def _selected_formatter(): + """The formatter setup_logging() chose, past any journald wrapper. + + Under systemd the console handler's formatter is wrapped so each line + carries its syslog priority. That wrapper is applied only when + JOURNAL_STREAM is set, which is true in CI and false in a terminal, so + asserting on the handler's formatter directly passes locally and fails + on the runner. These tests are about which formatter format_type + selects, so they look through the wrapper. + """ + formatter = logging.getLogger().handlers[0].formatter + return getattr(formatter, "inner", formatter) + def test_json_format_selects_structured_formatter(self): setup_logging(format_type="json") - assert isinstance( - logging.getLogger().handlers[0].formatter, StructuredFormatter) + assert isinstance(self._selected_formatter(), StructuredFormatter) def test_readable_format_selects_contextual_formatter(self): setup_logging(format_type="readable") - assert isinstance( - logging.getLogger().handlers[0].formatter, ContextualFormatter) + assert isinstance(self._selected_formatter(), ContextualFormatter) def test_log_file_adds_file_handler(self, tmp_path): log_file = tmp_path / "test.log" diff --git a/test/test_metrics_cache_unknown_fields.py b/test/test_metrics_cache_unknown_fields.py new file mode 100644 index 00000000..729893a3 --- /dev/null +++ b/test/test_metrics_cache_unknown_fields.py @@ -0,0 +1,128 @@ +"""A malformed metrics cache entry must not take every plugin down with it. + +`ResourceMetrics(**cached)` raises TypeError on a single unexpected key, and +that exception escapes into plugin_manager, which reports it per plugin as +"plugin operation failed". Every plugin fails and the plugin system never +finishes initialising -- the health endpoint reports +`plugin_system: not_initialized` while the display itself keeps running. + +Seen on a live rig, once per plugin, continuously: + + ERROR - src.plugin_system.plugin_manager - plugin geochron operation failed: + ResourceMetrics.__init__() got an unexpected keyword argument + 'consecutive_failures' + +`consecutive_failures` belongs to plugin_health, not to metrics. How a +health-shaped record came to sit under a plugin_metrics key on that machine is +not established -- a restored backup that mixed two machines' caches is the +likeliest explanation, and the same rig had one restored onto it -- but a +loader that turns one bad cache entry into a total outage is the part worth +fixing. plugin_health already repairs its own records field by field rather +than trusting what is on disk. +""" +import logging +from dataclasses import fields +from unittest.mock import MagicMock + +import pytest + +from src.plugin_system.resource_monitor import PluginResourceMonitor, ResourceMetrics + + +class _Cache: + def __init__(self, payload=None): + self.payload = payload + + def get(self, key, max_age=None, memory_ttl=None, **kwargs): + return self.payload + + def set(self, key, data, ttl=None, **kwargs): + pass + + +def _monitor(payload): + m = PluginResourceMonitor(cache_manager=_Cache(payload)) + m.logger = logging.getLogger("test") + return m + + +#: What the rig actually had under the metrics key. +HEALTH_SHAPED = { + "consecutive_failures": 0, "circuit_state": "closed", + "circuit_opened_time": None, "half_open_start_time": None, + "last_error": None, "last_failure_time": None, + "last_success_time": 1_700_000_000.0, "total_failures": 0, + "total_successes": 42, +} + + +def test_a_health_record_under_the_metrics_key_does_not_raise(): + """The exact failure: it must degrade, not take the plugin system down.""" + monitor = _monitor(HEALTH_SHAPED) + metrics = monitor.get_metrics(" plugin-a".strip()) + assert isinstance(metrics, ResourceMetrics) + + +def test_recognised_fields_in_a_mixed_record_are_kept(): + """Dropping the record wholesale would lose real history unnecessarily.""" + mixed = dict(HEALTH_SHAPED, call_count=7, memory_mb=12.5) + metrics = _monitor(mixed).get_metrics("plugin-b") + assert metrics.call_count == 7 + assert metrics.memory_mb == 12.5 + + +def test_a_clean_record_still_loads_unchanged(): + clean = {f.name: 3 for f in fields(ResourceMetrics)} + metrics = _monitor(clean).get_metrics("plugin-c") + for name in (f.name for f in fields(ResourceMetrics)): + assert getattr(metrics, name) == 3 + + +def test_unknown_fields_are_named_in_the_log(caplog): + """Silently discarding them would hide a real schema change.""" + with caplog.at_level(logging.WARNING): + _monitor(HEALTH_SHAPED).get_metrics("plugin-d") + # getMessage(), not .message: the latter is only populated once a handler + # formats the record, so the obvious spelling silently never matches. + assert any("consecutive_failures" in r.getMessage() for r in caplog.records), \ + caplog.text + + +@pytest.mark.parametrize("payload", ["a string", 42, ["a", "list"]]) +def test_a_non_mapping_cache_entry_does_not_raise(payload): + metrics = _monitor(payload).get_metrics("plugin-e") + assert isinstance(metrics, ResourceMetrics) + + +@pytest.mark.parametrize("bad", [ + {"call_count": "not a number"}, + {"memory_mb": None}, + {"execution_time": {"nested": "junk"}}, + {"min_execution_time": ["a", "list"]}, +]) +def test_values_of_the_wrong_type_fall_back_to_usable_defaults(bad): + """isinstance() alone was not enough. + + A dataclass does not enforce its annotations, so the bad value was simply + stored and the old assertion passed -- then monitor_call() raised + "can only concatenate str (not \"int\") to str" on the next call. The + metrics must come back *usable*, not merely constructed. + """ + monitor = _monitor(bad) + metrics = monitor.get_metrics("plugin-f") + assert isinstance(metrics, ResourceMetrics) + + field_name = next(iter(bad)) + assert isinstance(getattr(metrics, field_name), (int, float)), \ + f"{field_name} came back as {getattr(metrics, field_name)!r}" + + # The real proof: arithmetic on the loaded metrics must not explode. + metrics.call_count += 1 + metrics.total_execution_time += 0.5 + metrics.update_average_execution_time() + + +def test_a_numeric_string_is_accepted_rather_than_discarded(): + """JSON round-trips can widen an int to a string; that is recoverable.""" + metrics = _monitor({"call_count": "7"}).get_metrics("plugin-g") + assert metrics.call_count == 7 diff --git a/test/test_resource_monitor.py b/test/test_resource_monitor.py index 6b3dbe48..2f651d1d 100644 --- a/test/test_resource_monitor.py +++ b/test/test_resource_monitor.py @@ -11,7 +11,7 @@ Focus areas: import time import pytest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from src.plugin_system.resource_monitor import ( PluginResourceMonitor, @@ -127,3 +127,87 @@ class TestForceReload: fresh = mon.get_metrics_summary("p", force_reload=True) assert fresh["call_count"] == 7 assert any(c.kwargs.get("memory_ttl") == 0 for c in cache.get.call_args_list) + + +class TestMetricsPersistenceChurn: + """Metrics are telemetry; writing them on every call wore the SD card. + + Each write is a ~350-byte file, which on ext4 costs a 4KB block plus a + journal entry. At roughly nine calls a minute per plugin across fourteen + plugins it dominated the device's write volume. + """ + + def test_the_first_snapshot_is_written_even_seconds_after_boot(self): + """The throttle must key off "have we written?", not process uptime. + + time.monotonic() is time since boot on Linux, and systemd starts this + service at boot. With 0.0 as the missing-timestamp default, + `now - 0.0 < 30` was true for the first half-minute of every run, so + the very first metrics write -- the one that matters most after a + restart -- was silently skipped. + """ + import src.plugin_system.resource_monitor as rm + cache = _cache() + mon = PluginResourceMonitor(cache, enable_monitoring=False) + # 12 seconds after boot: inside the interval, but nothing written yet. + with patch.object(rm.time, "monotonic", return_value=12.0): + mon.monitor_call("p", lambda: None) + writes = [c for c in cache.set.call_args_list + if "plugin_metrics:" in str(c)] + assert writes, \ + "the first snapshot was dropped because the process was young" + + def test_repeated_calls_persist_once_per_interval(self): + cache = _cache() + mon = PluginResourceMonitor(cache, enable_monitoring=False) + for _ in range(50): + mon.monitor_call("p", lambda: None) + writes = [c for c in cache.set.call_args_list + if c.args and str(c.args[0]).startswith("plugin_metrics:")] + assert len(writes) == 1, ( + f"50 calls produced {len(writes)} metric writes; expected 1") + + def test_the_interval_elapsing_allows_the_next_write(self, monkeypatch): + import src.plugin_system.resource_monitor as rm + cache = _cache() + mon = PluginResourceMonitor(cache, enable_monitoring=False) + mon.monitor_call("p", lambda: None) + # pretend the interval has passed + mon._metrics_persisted_at["p"] -= rm._METRICS_PERSIST_INTERVAL + 1 + mon.monitor_call("p", lambda: None) + writes = [c for c in cache.set.call_args_list + if c.args and str(c.args[0]).startswith("plugin_metrics:")] + assert len(writes) == 2 + + def test_in_memory_metrics_stay_exact_while_writes_are_skipped(self): + mon = PluginResourceMonitor(_cache(), enable_monitoring=False) + for _ in range(20): + mon.monitor_call("p", lambda: None) + assert mon.get_metrics("p").call_count == 20 + + def test_reset_lets_the_next_call_persist_immediately(self): + cache = _cache() + mon = PluginResourceMonitor(cache, enable_monitoring=False) + mon.monitor_call("p", lambda: None) + mon.reset_metrics("p") + mon.monitor_call("p", lambda: None) + writes = [c for c in cache.set.call_args_list + if c.args and str(c.args[0]).startswith("plugin_metrics:")] + assert len(writes) == 2, "reset should clear the throttle timestamp" + + def test_a_failed_write_does_not_buy_the_next_interval_of_silence(self): + """A set() that raises must not count as having persisted. + + Marking the timestamp before the write would leave no snapshot in the + cache and still suppress the next 30 seconds of attempts. + """ + cache = _cache() + cache.set.side_effect = [OSError("disk full"), None] + mon = PluginResourceMonitor(cache, enable_monitoring=False) + with pytest.raises(OSError): + mon.monitor_call("p", lambda: None) + # the very next call must try again rather than skip the interval + mon.monitor_call("p", lambda: None) + writes = [c for c in cache.set.call_args_list + if c.args and str(c.args[0]).startswith("plugin_metrics:")] + assert len(writes) == 2, "a failed write should be retried, not skipped" diff --git a/test/test_vegas_log_volume.py b/test/test_vegas_log_volume.py new file mode 100644 index 00000000..5e8d126a --- /dev/null +++ b/test/test_vegas_log_volume.py @@ -0,0 +1,73 @@ +"""The Vegas content path must trace at DEBUG, not INFO. + +plugin_adapter narrates every step of acquiring content from every plugin -- +"Has get_vegas_content", "Native: calling get_vegas_content()", "Native content +returned None", "Has scroll_helper", the per-item sizes -- and it does that for +each plugin on each cycle. + +Measured on a live rig: 13,408 log lines an hour, of which 13,366 were INFO and +35 were WARNING. plugin_adapter alone produced 2,457 of them. That is ~223 +lines a minute of string formatting on a Pi that is also driving the panel, all +of it written through journald to the SD card, and it buries the 35 lines that +actually indicate a problem. + +Nothing is lost by moving it to DEBUG: the 19 warning/error/exception calls in +the module are untouched, so real failures still surface at their own level. + +One INFO call is deliberate and stays -- the padding-strip message chooses its +level at runtime (`logger.warning if (left and right) else logger.info`) and +test_vegas_plugin_adapter.py pins it. +""" +import ast +from pathlib import Path + +import pytest + +ADAPTER = (Path(__file__).resolve().parent.parent / "src" / "vegas_mode" + / "plugin_adapter.py") + + +def _logger_calls(path, *levels): + """Direct logger.(...) call sites in a module.""" + tree = ast.parse(path.read_text(encoding="utf-8")) + found = [] + for node in ast.walk(tree): + if (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr in levels + and getattr(node.func.value, "id", None) == "logger"): + found.append(node.lineno) + return found + + +def _info_calls(path): + """Direct logger.info(...) call sites in a module.""" + return _logger_calls(path, "info") + + +def test_the_content_path_does_not_trace_at_info(): + calls = _info_calls(ADAPTER) + assert not calls, ( + "plugin_adapter should trace at DEBUG; found logger.info at lines " + f"{calls}. This path runs per plugin per cycle and its output goes to " + "the SD card via journald." + ) + + +def test_real_failures_still_have_a_level_of_their_own(): + """Demoting the trace must not have swept up the error reporting. + + Counted from the AST rather than with source.count(): the text form also + matches comments, docstrings and string literals -- including this + module's own docstring, which names these levels -- so a real + logger.error() could be demoted while the tally stayed put. + """ + loud = _logger_calls(ADAPTER, "warning", "error", "exception") + assert len(loud) >= 15, \ + f"only {len(loud)} warning/error/exception calls remain: {loud}" + + +def test_the_deliberate_runtime_chosen_level_survives(): + """The padding-strip message picks its level at runtime; leave it alone.""" + source = ADAPTER.read_text(encoding="utf-8") + assert "logger.warning if (left and right) else logger.info" in source From 1fbe244e49ea7989d706f10e7666fadb6d5ed4d0 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:19:33 -0400 Subject: [PATCH 34/53] fix(plugins): say when discovery skips a directory (#489) * fix(plugins): say when discovery skips a directory A plugin can be enabled in config, enabled in plugin state, present on disk with a valid manifest and an importable entry point -- and simply absent from the running process, with nothing anywhere to say why. That is not hypothetical. hockey-scoreboard on a live rig is enabled in both places, imports cleanly when loaded by hand, and is listed in the Vegas plugin order, but is not among the 22 plugins the process actually holds. Establishing even that much meant comparing cache-file mtimes to find it had last run three days earlier. The journal had nothing, because discovery does not report what it declines to load. Two paths were silent. A directory with no manifest.json was skipped without comment, which is defensible until it is the thing you are trying to explain. Quieter still, a manifest that parsed but carried no "id" was read successfully and then dropped on the floor -- no warning, no trace, and the plugin simply does not exist as far as the rest of the system is concerned. Both now log a warning naming the directory and the reason. This does not explain the rig above; its manifest has an id. It makes the next occurrence diagnosable from the journal instead of from file timestamps. Reverting the change fails both tests. 65 plugin-system tests pass. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(plugins): warn once per directory, not once per scan Self-review catch. Discovery runs on every web UI page load and every config reconcile, so warning unconditionally about an unloadable directory would put a line in the journal each time someone opened a page -- the same log-volume problem this change exists to help diagnose. The skip is now reported once per directory per process. The diagnostic value is unchanged: the reason a plugin is missing still appears in the journal, once, where before it appeared nowhere. Test added covering five consecutive scans producing one warning. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(plugins): one unusable manifest no longer aborts the whole scan json.load accepts any JSON value, so a manifest.json holding null, [], "text" or 42 parses without complaint and then raises AttributeError on manifest.get('id'). Nothing catches that: the outer handler around the scan takes OSError and PermissionError only. So a single malformed manifest did not skip that one directory -- it aborted _scan_directory_for_plugins outright, and every other plugin on disk, however healthy, silently failed to register. Reproduced with three directories, the middle one holding `null`: SCAN ABORTED -> AttributeError: 'NoneType' object has no attribute 'get' the two valid plugins never registered That is the same failure this PR set out to fix, in its most severe form: a plugin enabled in config, enabled in plugin state, present on disk, and absent from the running process with nothing to say why -- except here it takes every other plugin with it. A manifest that is not a JSON object is now skipped like any other unusable directory, named once, with what it actually was: Skipping bad-null: its manifest.json is NoneType, not a JSON object Skipping bad-list: its manifest.json is list, not a JSON object scan returned: ['aaa-good', 'zzz-good'] Verified: removing the guard fails 6 of the 10 tests. Covers null, list, string, int and bool, and asserts the healthy plugins either side of the bad one still register. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 (1M context) --- src/plugin_system/plugin_manager.py | 68 +++++++++-- test/test_plugin_discovery_reports_skips.py | 126 ++++++++++++++++++++ 2 files changed, 182 insertions(+), 12 deletions(-) create mode 100644 test/test_plugin_discovery_reports_skips.py diff --git a/src/plugin_system/plugin_manager.py b/src/plugin_system/plugin_manager.py index 60356385..cac81472 100644 --- a/src/plugin_system/plugin_manager.py +++ b/src/plugin_system/plugin_manager.py @@ -76,6 +76,9 @@ class PluginManager: # Lock protecting plugin_manifests and plugin_directories from # concurrent mutation (background reconciliation) and reads (requests). self._discovery_lock = threading.RLock() + #: Directories already reported as unloadable, so the warning is + #: emitted once rather than on every discovery scan. + self._skip_reported: set = set() # Lock protecting plugin_last_update from concurrent mutation/iteration. # It's written from run_scheduled_updates()/update_all_plugins() (main @@ -195,18 +198,59 @@ class PluginManager: continue manifest_path = item / "manifest.json" - if manifest_path.exists(): - try: - with open(manifest_path, 'r', encoding='utf-8') as f: - manifest = json.load(f) - plugin_id = manifest.get('id') - if plugin_id: - plugin_ids.append(plugin_id) - new_manifests[plugin_id] = manifest - new_directories[plugin_id] = item - except (json.JSONDecodeError, PermissionError, OSError) as e: - self.logger.warning("Error reading manifest from %s: %s", manifest_path, e, exc_info=True) - continue + if not manifest_path.exists(): + # Once per directory per process. Discovery runs on every + # web UI page load and every config reconcile, so warning + # unconditionally would put a line in the journal each + # time someone opened a page -- the same log-volume + # problem this is meant to help diagnose. + # A directory here that carries no manifest is not a + # plugin. Said once, because the alternative is a plugin + # that is enabled in config, enabled in plugin state, + # present on disk, and simply absent from the running + # process with nothing anywhere to say why. Working that + # out afterwards means reading cache-file mtimes. + if item.name not in self._skip_reported: + self._skip_reported.add(item.name) + self.logger.warning( + "Skipping %s: no manifest.json, so it cannot be " + "loaded as a plugin", item.name) + continue + try: + with open(manifest_path, 'r', encoding='utf-8') as f: + manifest = json.load(f) + except (json.JSONDecodeError, PermissionError, OSError) as e: + self.logger.warning("Error reading manifest from %s: %s", manifest_path, e, exc_info=True) + continue + + # json.load accepts any JSON value, so a manifest holding + # null, [] or "text" parses and then raises AttributeError on + # .get(). Nothing here catches that -- the outer handler takes + # OSError/PermissionError only -- so a single malformed + # manifest aborted the whole scan and every other plugin on + # disk, however healthy, silently failed to register. + if not isinstance(manifest, dict): + if item.name not in self._skip_reported: + self._skip_reported.add(item.name) + self.logger.warning( + "Skipping %s: its manifest.json is %s, not a JSON " + "object", item.name, type(manifest).__name__) + continue + + plugin_id = manifest.get('id') + if not plugin_id: + # Parsed but unusable. This was the quietest path of all: + # the manifest is read successfully and then dropped. + if item.name not in self._skip_reported: + self._skip_reported.add(item.name) + self.logger.warning( + "Skipping %s: its manifest.json has no \"id\", so " + "there is nothing to register it under", item.name) + continue + + plugin_ids.append(plugin_id) + new_manifests[plugin_id] = manifest + new_directories[plugin_id] = item except (OSError, PermissionError) as e: self.logger.error("Error scanning directory %s: %s", directory, e, exc_info=True) diff --git a/test/test_plugin_discovery_reports_skips.py b/test/test_plugin_discovery_reports_skips.py new file mode 100644 index 00000000..046134f9 --- /dev/null +++ b/test/test_plugin_discovery_reports_skips.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Discovery must say when it skips a directory. + +A plugin can be enabled in config, enabled in plugin state, present on disk +with a valid entry point -- and simply absent from the running process, with +nothing in the journal to say why. Working that out afterwards meant comparing +cache-file mtimes to find when it had last run. + +Two paths were silent. A directory with no manifest.json was ignored, and -- +quieter still -- a manifest that parsed but carried no "id" was read +successfully and then dropped on the floor. +""" +import json +import logging +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from src.plugin_system.plugin_manager import PluginManager # noqa: E402 + + +def _manager(tmp_path): + pm = PluginManager.__new__(PluginManager) + pm.plugins_dir = tmp_path + pm.logger = logging.getLogger("test.discovery") + pm.plugin_manifests = {} + pm.plugin_directories = {} + pm._discovery_lock = __import__("threading").RLock() + pm._skip_reported = set() + pm.schema_manager = MagicMock() + return pm + + +def test_a_directory_without_a_manifest_is_reported(tmp_path, caplog): + (tmp_path / "not-a-plugin").mkdir() + pm = _manager(tmp_path) + with caplog.at_level(logging.WARNING, logger="test.discovery"): + pm._scan_directory_for_plugins(tmp_path) + joined = " ".join(r.message for r in caplog.records) + assert "not-a-plugin" in joined and "manifest" in joined, ( + f"skip was silent; log said: {joined!r}") + + +def test_a_manifest_without_an_id_is_reported(tmp_path, caplog): + d = tmp_path / "idless" + d.mkdir() + (d / "manifest.json").write_text(json.dumps({"name": "No Id", "version": "1.0.0"})) + pm = _manager(tmp_path) + with caplog.at_level(logging.WARNING, logger="test.discovery"): + pm._scan_directory_for_plugins(tmp_path) + joined = " ".join(r.message for r in caplog.records) + assert "idless" in joined and "id" in joined, ( + f"a parsed-but-unusable manifest vanished silently; log said: {joined!r}") + + +def test_a_good_plugin_still_registers(tmp_path, caplog): + d = tmp_path / "real-plugin" + d.mkdir() + (d / "manifest.json").write_text(json.dumps( + {"id": "real-plugin", "name": "Real", "version": "1.0.0"})) + pm = _manager(tmp_path) + pm._scan_directory_for_plugins(tmp_path) + assert "real-plugin" in pm.plugin_manifests, "a valid plugin was not registered" + + +def test_the_warning_does_not_repeat_on_every_scan(tmp_path, caplog): + """Discovery runs on every web UI page load and every config reconcile. + + Warning unconditionally would put a line in the journal each time someone + opened a page -- the same log-volume problem this is meant to help + diagnose. + """ + (tmp_path / "not-a-plugin").mkdir() + pm = _manager(tmp_path) + with caplog.at_level(logging.WARNING, logger="test.discovery"): + for _ in range(5): + pm._scan_directory_for_plugins(tmp_path) + hits = [r for r in caplog.records if "not-a-plugin" in r.message] + assert len(hits) == 1, f"warned {len(hits)} times across 5 scans" + + +def _plugin(tmp_path, name, body): + d = tmp_path / name + d.mkdir() + (d / "manifest.json").write_text(json.dumps(body)) + return d + + +VALID = {"name": "V", "version": "1.0.0", "class_name": "X", "display_modes": ["m"]} + + +@pytest.mark.parametrize("body", [None, [1, 2], "not an object", 42, True]) +def test_a_manifest_that_is_not_an_object_is_skipped_not_fatal(tmp_path, caplog, body): + """json.load accepts any JSON value, not just objects. + + manifest.get('id') then raised AttributeError, which nothing here caught -- + the outer handler takes OSError/PermissionError only. A single malformed + manifest aborted the entire scan, so every other plugin on disk, however + healthy, silently failed to register. + """ + _plugin(tmp_path, "aaa-good", dict(VALID, id="aaa-good")) + _plugin(tmp_path, "mmm-bad", body) + _plugin(tmp_path, "zzz-good", dict(VALID, id="zzz-good")) + + pm = _manager(tmp_path) + with caplog.at_level(logging.WARNING, logger="test.discovery"): + found = pm._scan_directory_for_plugins(tmp_path) + + assert sorted(found) == ["aaa-good", "zzz-good"], ( + "one unusable manifest took the healthy plugins down with it") + joined = " ".join(r.message for r in caplog.records) + assert "mmm-bad" in joined, f"the skip was silent; log said: {joined!r}" + + +def test_the_bad_manifest_is_named_with_what_it_actually_was(tmp_path, caplog): + _plugin(tmp_path, "listy", [1, 2]) + pm = _manager(tmp_path) + with caplog.at_level(logging.WARNING, logger="test.discovery"): + pm._scan_directory_for_plugins(tmp_path) + joined = " ".join(r.message for r in caplog.records) + assert "listy" in joined and "list" in joined, ( + f"the warning does not say what the manifest was: {joined!r}") From 5f29243e876c68f11974a40d0f575f4317b665ec Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:22:48 -0400 Subject: [PATCH 35/53] fix(config): make the device location the default for plugin location fields (#490) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(config): make the device location the default for plugin location fields A user in Kansas City reported their radar centred on Dallas, TX with nothing in config.json to explain it. The radar is the `ledmatrix-weather` plugin's `radar` mode, and it centres on the same coordinates as every other weather mode: `forecast_data` lat/lon, geocoded from the plugin's own `location_city` / `location_state` / `location_country`. Those ship with schema defaults of Dallas / Texas / US. A user who never opened the weather plugin's config form therefore has no `location_city` on disk, and `PluginManager` merges the schema default in at load time — so the whole plugin (not just the radar) silently runs on Dallas. Radar is just the only mode that draws a recognisable map and gives the mismatch away. Meanwhile the device-wide `location` block that General settings writes was read by nothing at all, despite its own help text promising it was "used for weather, sunrise/sunset, and other location-based content". `SchemaManager.generate_default_config()` now substitutes the device `location` into the three fully-namespaced `location_*` keys before handing defaults back, so the promise holds: - Only `location_city` / `location_state` / `location_country` are substituted. A bare `state` key is left alone — `ledmatrix-elections` uses it for a two-letter code, and rewriting it would break that plugin. - A value the user saved on the plugin still wins: this replaces the schema default, and `merge_with_defaults` puts user config on top. - The substitution is applied on the way out of the defaults cache rather than into it, so changing the device location takes effect immediately. - No config manager, no `location` block, or an unreadable config all fall back to the plugin's own schema defaults. Every caller benefits: the plugin loader, the config form (which now pre-fills the user's real city), config save, and reset-to-defaults. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GNLrSZ32FNKpHRaduKEJsg * docs(web): name the exact plugin keys the device location seeds Review follow-up. The General settings help text said the device location was "the default for every plugin that asks for a city", which overstates what the code does: only the fully-namespaced `location_city` / `location_state` / `location_country` keys are substituted. A plugin with a bare `city` key gets nothing — deliberately, since `ledmatrix-elections` uses `state` for a two-letter code. The tips now name the exact keys. Worth noting for anyone editing these: `ui.help_tip(...)` takes a single-quoted Jinja string, so an apostrophe in the tip text has to be escaped or written around. The wording here avoids them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GNLrSZ32FNKpHRaduKEJsg --------- Co-authored-by: Claude --- docs/CONFIG_REFERENCE.md | 2 +- src/plugin_system/plugin_manager.py | 3 +- src/plugin_system/schema_manager.py | 91 ++++++++- test/test_schema_manager_device_location.py | 179 ++++++++++++++++++ web_interface/app.py | 3 +- .../templates/v3/partials/general.html | 6 +- 6 files changed, 274 insertions(+), 10 deletions(-) create mode 100644 test/test_schema_manager_device_location.py diff --git a/docs/CONFIG_REFERENCE.md b/docs/CONFIG_REFERENCE.md index c54aca08..4af3c0fb 100644 --- a/docs/CONFIG_REFERENCE.md +++ b/docs/CONFIG_REFERENCE.md @@ -18,7 +18,7 @@ tooling against it. | `web_display_autostart` | bool, `true` | Whether the web interface service starts with the system | `scripts/utils/start_web_conditionally.py` | | `timezone` | string, `"America/New_York"` | IANA timezone for schedules and displays | `ConfigManager.get_timezone()` | | `target_fps` | int, `100` | Frame-rate ceiling for plugin rendering | `src/plugin_system/base_plugin.py`, `src/common/sports_scroll.py` | -| `location` | object | `city` / `state` / `country`, offered to plugins that need a location (weather, etc.) | plugins via merged config | +| `location` | object | `city` / `state` / `country`. Supplies the **default** for a plugin's own `location_city` / `location_state` / `location_country` setting, so weather, radar and friends follow this device without being configured twice. A value saved on the plugin itself still overrides it. | `SchemaManager.apply_device_location()`, then plugins via merged config | ## `schedule` — display on/off hours diff --git a/src/plugin_system/plugin_manager.py b/src/plugin_system/plugin_manager.py index cac81472..543017c9 100644 --- a/src/plugin_system/plugin_manager.py +++ b/src/plugin_system/plugin_manager.py @@ -71,7 +71,8 @@ class PluginManager: self.plugin_loader = PluginLoader(logger=self.logger) self.plugin_executor = PluginExecutor(default_timeout=30.0, logger=self.logger) self.state_manager = PluginStateManager(logger=self.logger) - self.schema_manager = SchemaManager(plugins_dir=self.plugins_dir, logger=self.logger) + self.schema_manager = SchemaManager(plugins_dir=self.plugins_dir, logger=self.logger, + config_manager=self.config_manager) # Lock protecting plugin_manifests and plugin_directories from # concurrent mutation (background reconciliation) and reads (requests). diff --git a/src/plugin_system/schema_manager.py b/src/plugin_system/schema_manager.py index 5b894100..3db4a5a2 100644 --- a/src/plugin_system/schema_manager.py +++ b/src/plugin_system/schema_manager.py @@ -26,7 +26,25 @@ class SchemaManager: - Cache invalidation on plugin changes """ - def __init__(self, plugins_dir: Optional[Path] = None, project_root: Optional[Path] = None, logger: Optional[logging.Logger] = None): + # Plugin config keys that mean "where this device is". A plugin declaring + # any of these in its schema gets the device-wide ``location`` block from + # config.json as the *default* for that field, instead of whatever city the + # plugin author happened to ship. A value the user set on the plugin itself + # always wins -- this only ever replaces the schema default, so an explicit + # per-plugin location is still honoured. + # + # Only these fully-namespaced keys are substituted. A bare ``state`` or + # ``city`` key is deliberately left alone: plugins use those for unrelated + # things (ledmatrix-elections' ``state`` is a two-letter code, not a place + # name), and silently rewriting them would break those plugins. + DEVICE_LOCATION_KEYS: Dict[str, str] = { + 'location_city': 'city', + 'location_state': 'state', + 'location_country': 'country', + } + + def __init__(self, plugins_dir: Optional[Path] = None, project_root: Optional[Path] = None, + logger: Optional[logging.Logger] = None, config_manager: Optional[Any] = None): """ Initialize the Schema Manager. @@ -34,10 +52,14 @@ class SchemaManager: plugins_dir: Base plugins directory path project_root: Project root directory path logger: Optional logger instance + config_manager: Optional config manager, used to resolve the + device-wide ``location`` that seeds plugin location defaults. + Omitting it simply leaves schema defaults untouched. """ self.logger = logger or logging.getLogger(__name__) self.plugins_dir = plugins_dir self.project_root = project_root or Path.cwd() + self.config_manager = config_manager # Schema cache: plugin_id -> schema dict self._schema_cache: Dict[str, Dict[str, Any]] = {} @@ -212,10 +234,70 @@ class SchemaManager: return defaults + def get_device_location(self) -> Optional[Dict[str, Any]]: + """ + Return the device-wide ``location`` block from config.json, or None. + + This is the City/State/Country the user sets once under General + settings. Returns None when there is no config manager wired, the + config can't be read, or no location has been configured. + """ + if self.config_manager is None: + return None + try: + config = self.config_manager.load_config() + except Exception as e: + # A config that can't be read must never stop defaults being + # generated -- the plugin's own schema defaults still apply. + self.logger.debug(f"Could not read device location from config: {e}") + return None + if not isinstance(config, dict): + return None + location = config.get('location') + return location if isinstance(location, dict) else None + + def apply_device_location(self, defaults: Dict[str, Any]) -> Dict[str, Any]: + """ + Replace location-shaped schema defaults with the device's own location. + + Without this, a plugin that ships ``"location_city": "Dallas"`` as its + schema default silently reports Dallas weather (and centres its radar + there) for every user who never opened that plugin's config form -- + even though they set their real city under General settings. The + substituted value is still only a *default*: ``merge_with_defaults`` + lets any per-plugin value the user saved win over it. + + Mutates and returns ``defaults`` for convenience. + """ + if not defaults: + return defaults + if not any(key in defaults for key in self.DEVICE_LOCATION_KEYS): + return defaults + + location = self.get_device_location() + if not location: + return defaults + + for key, field in self.DEVICE_LOCATION_KEYS.items(): + if key not in defaults: + continue + value = location.get(field) + # Only a non-empty string is a real answer; a blank or missing + # field means "not configured", which leaves the schema default. + if isinstance(value, str) and value.strip(): + defaults[key] = value.strip() + + return defaults + def generate_default_config(self, plugin_id: str, use_cache: bool = True) -> Dict[str, Any]: """ Generate default configuration for a plugin from its schema. + Location fields (see ``DEVICE_LOCATION_KEYS``) default to the device's + configured location rather than the plugin author's. That substitution + is applied on the way out rather than being cached, so changing the + device location takes effect without invalidating the defaults cache. + Args: plugin_id: Plugin identifier use_cache: If True, return cached defaults if available @@ -225,7 +307,7 @@ class SchemaManager: """ # Check cache first if use_cache and plugin_id in self._defaults_cache: - return self._defaults_cache[plugin_id].copy() + return self.apply_device_location(self._defaults_cache[plugin_id].copy()) schema = self.load_schema(plugin_id, use_cache=use_cache) if not schema: @@ -249,10 +331,11 @@ class SchemaManager: if 'live_priority' not in defaults: defaults['live_priority'] = schema.get('properties', {}).get('live_priority', {}).get('default', False) - # Cache the defaults + # Cache the defaults *before* the device location is layered on, so a + # later change to the device location is picked up by the next call. self._defaults_cache[plugin_id] = defaults.copy() - return defaults + return self.apply_device_location(defaults) def validate_config_against_schema(self, config: Dict[str, Any], schema: Dict[str, Any], plugin_id: Optional[str] = None) -> Tuple[bool, List[str]]: diff --git a/test/test_schema_manager_device_location.py b/test/test_schema_manager_device_location.py new file mode 100644 index 00000000..2908b95a --- /dev/null +++ b/test/test_schema_manager_device_location.py @@ -0,0 +1,179 @@ +""" +Tests for the device-location default: a plugin that ships a location field in +its schema must default to the device's configured City/State/Country, not to +whatever place the plugin author hard-coded. + +The bug this pins: ledmatrix-weather ships ``"location_city": "Dallas"`` as a +schema default, so a user who set Kansas City under General settings but never +opened the weather plugin's own config form got Dallas weather — and a radar +centred on Dallas — with nothing in config.json to explain it. +""" + +import json + +import pytest + +from src.plugin_system.schema_manager import SchemaManager + + +class FakeConfigManager: + """Minimal stand-in exposing the load_config() SchemaManager relies on.""" + + def __init__(self, config): + self.config = config + self.load_count = 0 + + def load_config(self): + self.load_count += 1 + return self.config + + +class ExplodingConfigManager: + def load_config(self): + raise OSError("config.json is unreadable") + + +WEATHER_SCHEMA = { + "type": "object", + "properties": { + "location_city": {"type": "string", "default": "Dallas"}, + "location_state": {"type": "string", "default": "Texas"}, + "location_country": {"type": "string", "default": "US"}, + "units": {"type": "string", "default": "imperial"}, + }, +} + + +def write_plugin(plugins_dir, plugin_id, schema): + plugin_dir = plugins_dir / plugin_id + plugin_dir.mkdir(parents=True, exist_ok=True) + (plugin_dir / "config_schema.json").write_text(json.dumps(schema)) + return plugin_dir + + +@pytest.fixture +def plugins_dir(tmp_path): + d = tmp_path / "plugin-repos" + d.mkdir() + return d + + +def make_sm(plugins_dir, tmp_path, location): + config = {} if location is None else {"location": location} + cm = FakeConfigManager(config) + sm = SchemaManager(plugins_dir=plugins_dir, project_root=tmp_path, + config_manager=cm) + return sm, cm + + +class TestDeviceLocationDefaults: + def test_device_location_replaces_plugin_default(self, plugins_dir, tmp_path): + write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA) + sm, _ = make_sm(plugins_dir, tmp_path, + {"city": "Kansas City", "state": "Missouri", "country": "US"}) + + defaults = sm.generate_default_config("ledmatrix-weather") + + assert defaults["location_city"] == "Kansas City" + assert defaults["location_state"] == "Missouri" + assert defaults["location_country"] == "US" + # Non-location defaults are untouched. + assert defaults["units"] == "imperial" + + def test_user_set_plugin_value_still_wins(self, plugins_dir, tmp_path): + write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA) + sm, _ = make_sm(plugins_dir, tmp_path, + {"city": "Kansas City", "state": "Missouri", "country": "US"}) + + defaults = sm.generate_default_config("ledmatrix-weather") + merged = sm.merge_with_defaults({"location_city": "Denver"}, defaults) + + assert merged["location_city"] == "Denver" + # Fields the user did not override still follow the device. + assert merged["location_state"] == "Missouri" + + def test_blank_and_missing_device_fields_leave_schema_default(self, plugins_dir, tmp_path): + write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA) + sm, _ = make_sm(plugins_dir, tmp_path, {"city": "Kansas City", "state": " "}) + + defaults = sm.generate_default_config("ledmatrix-weather") + + assert defaults["location_city"] == "Kansas City" + assert defaults["location_state"] == "Texas" # blank -> not configured + assert defaults["location_country"] == "US" # absent -> schema default + + def test_no_device_location_configured_is_a_no_op(self, plugins_dir, tmp_path): + write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA) + sm, _ = make_sm(plugins_dir, tmp_path, None) + + defaults = sm.generate_default_config("ledmatrix-weather") + + assert defaults["location_city"] == "Dallas" + + def test_no_config_manager_is_a_no_op(self, plugins_dir, tmp_path): + write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA) + sm = SchemaManager(plugins_dir=plugins_dir, project_root=tmp_path) + + assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Dallas" + + def test_unreadable_config_falls_back_to_schema_defaults(self, plugins_dir, tmp_path): + write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA) + sm = SchemaManager(plugins_dir=plugins_dir, project_root=tmp_path, + config_manager=ExplodingConfigManager()) + + assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Dallas" + + +class TestScopedToNamespacedKeys: + def test_bare_state_key_is_not_rewritten(self, plugins_dir, tmp_path): + """ledmatrix-elections' ``state`` is a two-letter code, not a place name.""" + write_plugin(plugins_dir, "ledmatrix-elections", { + "type": "object", + "properties": { + "state": {"type": "string", "default": "CA"}, + "city": {"type": "string", "default": "Springfield"}, + }, + }) + sm, _ = make_sm(plugins_dir, tmp_path, + {"city": "Kansas City", "state": "Missouri", "country": "US"}) + + defaults = sm.generate_default_config("ledmatrix-elections") + + assert defaults["state"] == "CA" + assert defaults["city"] == "Springfield" + + def test_plugin_without_location_fields_never_reads_config(self, plugins_dir, tmp_path): + write_plugin(plugins_dir, "clock-simple", { + "type": "object", + "properties": {"format": {"type": "string", "default": "12h"}}, + }) + sm, cm = make_sm(plugins_dir, tmp_path, {"city": "Kansas City"}) + + defaults = sm.generate_default_config("clock-simple") + + assert defaults["format"] == "12h" + assert cm.load_count == 0 + + +class TestCachingStaysFresh: + def test_location_change_is_picked_up_through_the_defaults_cache(self, plugins_dir, tmp_path): + write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA) + sm, cm = make_sm(plugins_dir, tmp_path, {"city": "Kansas City"}) + + assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Kansas City" + + cm.config["location"]["city"] = "Omaha" + + # Second call is served from the defaults cache, but must not serve a + # stale location. + assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Omaha" + + def test_cached_defaults_are_not_mutated_by_the_overlay(self, plugins_dir, tmp_path): + write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA) + sm, cm = make_sm(plugins_dir, tmp_path, {"city": "Kansas City"}) + + sm.generate_default_config("ledmatrix-weather") + assert sm._defaults_cache["ledmatrix-weather"]["location_city"] == "Dallas" + + cm.config.pop("location") + assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Dallas" diff --git a/web_interface/app.py b/web_interface/app.py index 8ed9cb8c..98bf0f71 100644 --- a/web_interface/app.py +++ b/web_interface/app.py @@ -118,7 +118,8 @@ saved_repositories_manager = SavedRepositoriesManager() schema_manager = SchemaManager( plugins_dir=plugins_dir, project_root=project_root, - logger=None + logger=None, + config_manager=config_manager ) # Initialize operation queue for plugin operations diff --git a/web_interface/templates/v3/partials/general.html b/web_interface/templates/v3/partials/general.html index eaf87a2c..e9261de7 100644 --- a/web_interface/templates/v3/partials/general.html +++ b/web_interface/templates/v3/partials/general.html @@ -95,7 +95,7 @@
- +
- +
- + Date: Fri, 21 Aug 2026 16:24:15 -0400 Subject: [PATCH 36/53] fix(sports): fetch odds for the games shown, not the whole schedule window (#494) * fix(sports): fetch odds for the games shown, not the whole window SportsUpcoming.update() walked every upcoming game in the schedule window and called _fetch_odds() on each one inside that collection loop, narrowing to upcoming_games_to_show only afterwards. Each call is a separate sequential ESPN request. The comment sitting above it said odds were fetched "only for games that will be displayed". The only narrowing it actually applied was show_favorite_teams_only, which is not the default, so in the usual configuration nothing narrowed it at all. Measured on devpi, where the football plugin has the same shape: 467 odds requests in one 35s burst, 467 distinct events 315 NFL + 152 college-football -- roughly a whole season plugin football-scoreboard operation timed out after 30.0s The burst repeats each time the 1h odds TTL expires: 67 -> 327 -> 957 -> 1261 requests/hour across four consecutive hours. Between expiries the cache works and the rate is zero, so this is a thundering herd on expiry, not a caching failure. The fetch now runs after selection, over team_games -- the list already cut to upcoming_games_to_show. This mirrors the fix the football plugin already carries; the shared base class never got it. SportsLive is deliberately left as it is: it walks the raw event list because it has to find which games are live, but only fetches odds for a game that has already passed the is_live/is_halftime test, so its fan-out is bounded by how many games are actually in progress. The test pins that distinction rather than assuming it. The test reads the AST rather than the source text, and asserts the full set of call sites, so a new one has to be classified deliberately instead of inheriting whichever behaviour it happens to land in. Writing it that way is what turned up the SportsLive site, which I had missed. Verified: reverting the fix fails the test with the offending iterable named ("iterates over 'events'"). 525 passed, 9 skipped across the sports and odds suites. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * test(sports): check the odds guard structurally, not by its text Review caught that _guards_above() collected an `if` test even when the call sat in that if's `else`, so moving _fetch_odds() into the else of the is_live/is_halftime test would still pass -- while fetching odds for exactly the non-live games the guard exists to exclude. Verifying that turned up a wider hole in the same assertion. It matched substrings of the *unparsed source*, so a negated condition satisfied it too: if not (details["is_live"] or details["is_halftime"]): self._fetch_odds(details) # every non-live game Both names still appear in that text, so `"is_live" in guards` held and the test passed on code doing the opposite of what it claims to check. The guard test is now structural. It walks the AST for an enclosing `if` whose *body* (never its `else`) contains the call, and whose test references both names without either sitting under a `not`. Verified by mutation: fetching odds for non-live games now fails with "does not sit in the true branch of a test requiring the game to be in progress". Moving the call into the else of the *favourites* test still passes, which is correct -- the game there is still live, so the in-progress contract holds and the fan-out stays bounded by how many games are actually in play. 525 passed, 9 skipped across the sports and odds suites. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 (1M context) --- src/base_classes/sports/modes.py | 17 +++- test/test_sports_odds_fanout.py | 136 +++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 test/test_sports_odds_fanout.py diff --git a/src/base_classes/sports/modes.py b/src/base_classes/sports/modes.py index 06e73467..8f6e3cce 100644 --- a/src/base_classes/sports/modes.py +++ b/src/base_classes/sports/modes.py @@ -145,8 +145,7 @@ class SportsUpcoming(SportsCore): if (game['home_abbr'] in self.favorite_teams or game['away_abbr'] in self.favorite_teams): favorite_games_found += 1 - if self.show_odds: - self._fetch_odds(game) + # Odds are NOT fetched here -- see after selection below. # Enhanced logging for debugging self.logger.info(f"Found {all_upcoming_games} total upcoming games in data") @@ -190,6 +189,20 @@ class SportsUpcoming(SportsCore): # Limit to the specified number of upcoming games team_games = team_games[:self.upcoming_games_to_show] + # Odds are fetched here, for the games that survived selection, + # rather than in the loop that collects them. That loop walks every + # upcoming game in the schedule window, and for a college league + # the window is enormous -- a live rig logged 946 upcoming games in + # one cycle and displayed 1 of them. The comment up there claimed + # odds were fetched "only for games that will be displayed", but + # the only narrowing it applied was show_favorite_teams_only, which + # is not the default; in the usual case nothing narrowed it at all + # and every game cost a separate ESPN request on a Pi that is also + # driving the panel. + if self.show_odds: + for game in team_games: + self._fetch_odds(game) + # Log changes or periodically should_log = ( current_time - self.last_log_time >= self.log_interval or diff --git a/test/test_sports_odds_fanout.py b/test/test_sports_odds_fanout.py new file mode 100644 index 00000000..c33930e9 --- /dev/null +++ b/test/test_sports_odds_fanout.py @@ -0,0 +1,136 @@ +"""Odds must be fetched for the games shown, not every game in the window. + +SportsUpcoming.update() collected every upcoming game in the schedule window +and called _fetch_odds() on each one *inside* that collection loop, narrowing +to upcoming_games_to_show only afterwards. The comment there said odds were +fetched "only for games that will be displayed", but the sole narrowing it +applied was show_favorite_teams_only, which is not the default -- so in the +usual configuration nothing narrowed it at all. + +Measured on a live rig: a college league produced 946 upcoming games in one +cycle and displayed 1 of them. The same shape on the football plugin produced +a burst of 467 sequential ESPN requests that ran for 35s and blew that +plugin's 30s update budget, and it repeats every time the 1h odds TTL expires. + +SportsLive is deliberately different: it walks the raw event list because it +has to find which games are live, but only fetches odds for a game that has +already passed the is_live/is_halftime test, so the fan-out is bounded by how +many games are actually in progress. +""" +import ast +from pathlib import Path + +import pytest + +MODES = (Path(__file__).resolve().parent.parent + / "src" / "base_classes" / "sports" / "modes.py") +TREE = ast.parse(MODES.read_text(encoding="utf-8")) + + +def _fetch_sites(): + """(class name, method name, lineno) for every self._fetch_odds(...) call.""" + calls = [n.lineno for n in ast.walk(TREE) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) + and n.func.attr == "_fetch_odds"] + sites = [] + for cls in [n for n in ast.walk(TREE) if isinstance(n, ast.ClassDef)]: + for fn in [n for n in cls.body if isinstance(n, ast.FunctionDef)]: + for lineno in calls: + if fn.lineno <= lineno <= (fn.end_lineno or fn.lineno): + sites.append((cls.name, fn.name, lineno)) + assert len(sites) == len(calls), "a _fetch_odds call sits outside any method" + return sites + + +def _innermost_loop_iterable(lineno): + best = None + for node in ast.walk(TREE): + if isinstance(node, ast.For) and \ + node.lineno <= lineno <= (node.end_lineno or node.lineno): + if best is None or node.lineno > best.lineno: + best = node + return None if best is None else ast.unparse(best.iter) + + +def _spans(body, lineno): + """True when `lineno` falls inside this list of statements.""" + return any(n.lineno <= lineno <= (n.end_lineno or n.lineno) for n in body) + + +def _parents(tree): + table = {} + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + table[child] = node + return table + + +PARENTS = _parents(TREE) + + +def _mentions_positively(test, names): + """True when `test` references every name, none of them under a `not`. + + Structural, not textual. Matching the unparsed source would accept + `not (details["is_live"] or details["is_halftime"])` -- which selects + exactly the non-live games this guard exists to exclude -- because the + names still appear in the text. + """ + found = set() + for node in ast.walk(test): + if not (isinstance(node, ast.Constant) and node.value in names): + continue + negated = False + cursor = node + while cursor is not test and cursor in PARENTS: + cursor = PARENTS[cursor] + if isinstance(cursor, ast.UnaryOp) and isinstance(cursor.op, ast.Not): + negated = True + break + if not negated: + found.add(node.value) + return found >= set(names) + + +def _guarded_by_positive(lineno, names): + """True when some enclosing `if` runs this line only if `names` hold. + + Only the TRUE branch counts: an `if` whose `else` contains the call would + otherwise look like a guard while doing the opposite. + """ + for node in ast.walk(TREE): + if isinstance(node, ast.If) and _spans(node.body, lineno) \ + and _mentions_positively(node.test, names): + return True + return False + + +def test_every_fetch_site_is_accounted_for(): + """A new call site must be classified deliberately, not inherited silently.""" + found = {(cls, fn) for cls, fn, _ in _fetch_sites()} + assert found == {("SportsUpcoming", "update"), ("SportsLive", "update")}, ( + f"unexpected _fetch_odds call sites: {sorted(found)}. Each one is a " + "sequential ESPN request per game -- classify it here on purpose.") + + +def test_upcoming_fetches_only_the_selected_games(): + for cls, _fn, lineno in _fetch_sites(): + if cls != "SportsUpcoming": + continue + iterable = _innermost_loop_iterable(lineno) + assert iterable == "team_games", ( + f"SportsUpcoming._fetch_odds at line {lineno} iterates over " + f"{iterable!r}. It must run over team_games -- already narrowed to " + "upcoming_games_to_show -- not over every event in the schedule " + "window. Each item costs one sequential ESPN request.") + + +def test_live_only_fetches_for_games_actually_in_progress(): + for cls, _fn, lineno in _fetch_sites(): + if cls != "SportsLive": + continue + assert _guarded_by_positive(lineno, {"is_live", "is_halftime"}), ( + f"SportsLive._fetch_odds at line {lineno} does not sit in the true " + "branch of a test requiring the game to be in progress. Without " + "that, it fans out across the whole event list -- one sequential " + "ESPN request per game.") From 568cb6d77f26277f8589b8530ebae98e858408f0 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:42:33 -0400 Subject: [PATCH 37/53] perf(vegas): report the frame rate when it is worth reporting (#487) Vegas logged an FPS line at INFO every five seconds for the whole of every run. Measured over two hours on a rig: 1410 samples, 98.5% of them within 10% of target. The 1.5% that were not included a reading of 8.6fps against a target of 60 -- a real stall, completely invisible inside 1389 lines reading "59.6". INFO is now reserved for a shortfall, the recovery from one, and a slow heartbeat so a healthy marquee still shows a pulse. Scroll-progress tracing drops to DEBUG for the same reason: it runs for the whole of every scroll and is what you turn debug on to watch. Three review findings, all fixed here. 1. Per-frame timing used the wall clock (critical). The loop sleeps the remainder of each frame budget: frame_elapsed = - frame_started time.sleep(max(0.0, frame_interval - frame_elapsed)) These devices have no RTC, so the clock jumps by however wrong boot time was when NTP first syncs. A backward step makes frame_elapsed negative, `frame_interval - frame_elapsed` then exceeds the whole budget, and the render loop stalls for the size of the correction. A forward step instead inflates the p99 and worst-frame figures this telemetry exists to report. Both per-frame timestamps are monotonic now. start_time stays wall-clock: it is only used for the iteration duration report, where a human-readable clock is the point. 2. FPS health state reset every iteration. last_fps_health_log and was_degraded were locals of run_iteration(), which is called once per cycle. Starting at 0.0 against a monotonic clock, `due` was true on the first sample of every iteration, so the 300s heartbeat degenerated into one report per cycle -- reintroducing the noise this change is about. A recovery that crossed an iteration boundary was never reported either, since was_degraded had already gone back to False. Both now live on the coordinator and reset in start(). 3. The degraded threshold read as an off-by-one. 90% of target is deliberate -- a marquee jitters constantly, so "anything below target" would report forever and mean nothing -- but nothing said so, leaving 55fps-against-60 looking like a missed case. The constant now states the band and gives that exact example. Also drops two soccer logo PNGs that a `git add -A` had swept into the first commit. They are unreferenced, unrelated to frame-rate telemetry, and 210KB. Verified: each fix mutation-checked -- restoring the wall clock on either per-frame timestamp, or making the health state local again, fails the new tests. 566 passed across the vegas, coordinator and scroll suites. Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW Co-authored-by: Claude Opus 5 (1M context) --- src/common/scroll_helper.py | 4 ++ src/vegas_mode/coordinator.py | 86 ++++++++++++++++++++++++++---- test/test_vegas_fps_health.py | 99 +++++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 11 deletions(-) create mode 100644 test/test_vegas_fps_health.py diff --git a/src/common/scroll_helper.py b/src/common/scroll_helper.py index 88c6d498..6e2f4218 100644 --- a/src/common/scroll_helper.py +++ b/src/common/scroll_helper.py @@ -328,6 +328,10 @@ class ScrollHelper: elapsed_time = current_time - (self.scroll_start_time or current_time) # The image already includes display_width padding, so we only need total_scroll_width required_total_distance = self.total_scroll_width + # Progress telemetry, emitted every few seconds for the whole of + # every scroll. It says how far along a marquee is, which is what + # you turn debug on to watch and not something an operator needs + # in the journal on a device that scrolls all day. self.logger.debug( "Scroll progress: elapsed=%.2fs, target=%.2fs, total_scrolled=%.0f/%d px (%.1f%%)", elapsed_time, diff --git a/src/vegas_mode/coordinator.py b/src/vegas_mode/coordinator.py index 430cbef6..b9af5ea6 100644 --- a/src/vegas_mode/coordinator.py +++ b/src/vegas_mode/coordinator.py @@ -31,6 +31,18 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +#: Degradation threshold, as a fraction of target_fps. A marquee jitters a +#: little all the time, so "anything under target" would report constantly and +#: mean nothing; 90% of target is the point where a shortfall is real. At a +#: 60fps target that is 54fps -- 55fps is a normal wobble and stays at DEBUG, +#: which is deliberate, not an off-by-one. +_FPS_HEALTHY_FRACTION = 0.9 + +#: A healthy marquee still reports this often, so silence means stopped +#: rather than fine. +_FPS_HEARTBEAT_INTERVAL = 300.0 + + def _percentile(ordered: List[float], fraction: float) -> float: """Nearest-rank percentile of an already-sorted list. @@ -96,6 +108,11 @@ class VegasModeCoordinator: self._is_active = False self._is_paused = False self._should_stop = False + # Frame-rate health, tracked across run_iteration() calls so the + # heartbeat is one-per-interval rather than one-per-cycle, and so a + # recovery spanning two cycles is still reported. Reset on start(). + self._fps_last_health_log = 0.0 + self._fps_was_degraded = False self._state_lock = threading.Lock() # Live priority tracking @@ -248,6 +265,11 @@ class VegasModeCoordinator: self._is_active = True self._should_stop = False self._start_time = time.time() + # A fresh run starts with a clean health slate: no stale + # "was degraded" from the previous run, and a heartbeat that is + # due immediately so the first sample confirms the marquee is up. + self._fps_last_health_log = 0.0 + self._fps_was_degraded = False # Line up the next group immediately, so the first extension is already # warm rather than stalling the scroll to fetch it. @@ -395,8 +417,18 @@ class VegasModeCoordinator: duration = self.render_pipeline.get_dynamic_duration() start_time = time.time() frame_count = 0 - fps_log_interval = 5.0 # Log FPS every 5 seconds - last_fps_log_time = start_time + fps_log_interval = 5.0 # Sample FPS every 5 seconds + # Health state lives on the coordinator, not here: run_iteration() is + # called once per cycle, so locals reset every few seconds. That made + # `last_fps_health_log = 0.0` fire the "heartbeat" on the first sample + # of every iteration rather than once per interval, and a recovery + # that crossed an iteration boundary was never reported at all -- + # was_degraded had already gone back to False. + # Monotonic, and deliberately not start_time: start_time is wall + # clock and is used below to report the iteration's duration. Mixing + # the two here would make every delta hugely negative and silence the + # frame-rate reporting altogether. + last_fps_log_time = time.monotonic() fps_frame_count = 0 # A mean hides stutter completely. At 120fps a five-second window is # ~600 frames, so a 200ms freeze -- plainly visible on a marquee -- @@ -408,7 +440,13 @@ class VegasModeCoordinator: logger.info("Starting Vegas iteration for %.1fs", duration) while True: - frame_started = time.time() + # Monotonic, like the FPS window below. These devices have no RTC, + # so the wall clock jumps by however wrong boot time was the moment + # NTP first syncs. A backward jump makes frame_elapsed negative, + # and `frame_interval - frame_elapsed` then sleeps for longer than + # the whole budget -- the render loop stalls for the size of the + # correction. A forward jump inflates p99 and worst-frame instead. + frame_started = time.monotonic() # Check for STATIC mode plugin that should pause scroll static_plugin = self._check_static_plugin_trigger() @@ -436,7 +474,7 @@ class VegasModeCoordinator: # quarter of the budget spent not rendering. Subtracting the work # already done keeps the pacing target while reclaiming that time, # and yields the GIL either way so other threads still run. - frame_elapsed = time.time() - frame_started + frame_elapsed = time.monotonic() - frame_started time.sleep(max(0.0, frame_interval - frame_elapsed)) # Measured before the sleep: time spent working, not pacing. @@ -448,16 +486,42 @@ class VegasModeCoordinator: frame_count += 1 fps_frame_count += 1 - # Periodic FPS logging - current_time = time.time() + # Periodic FPS logging. Reported at INFO only when the frame rate + # is actually worth an operator's attention -- a shortfall against + # target, or the recovery from one -- with a slow heartbeat so a + # healthy marquee still shows a pulse. + # + # Measured over two hours on a running rig: 1410 samples, 98.5% + # of them within 10% of target. The 1.5% that were not included a + # reading of 8.6fps against a target of 60 -- a real stall, and + # completely invisible inside 1389 lines reading "59.6". + # Monotonic: every use of this value in the block below is a + # duration, and these devices have no RTC, so the wall clock jumps + # by however wrong boot time was the moment NTP first syncs. That + # would not only mis-fire the heartbeat, it would corrupt the + # frame rate itself, since fps is frames divided by this delta. + current_time = time.monotonic() if current_time - last_fps_log_time >= fps_log_interval: fps = fps_frame_count / (current_time - last_fps_log_time) p99 = _percentile(sorted(frame_times), 0.99) - logger.info( - "Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms", - fps, self.vegas_config.target_fps, fps_frame_count, - p99 * 1000.0, frame_worst * 1000.0 - ) + target = self.vegas_config.target_fps + degraded = target > 0 and fps < target * _FPS_HEALTHY_FRACTION + due = (current_time - self._fps_last_health_log + >= _FPS_HEARTBEAT_INTERVAL) + if degraded or self._fps_was_degraded or due: + logger.info( + "Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms", + fps, target, fps_frame_count, + p99 * 1000.0, frame_worst * 1000.0 + ) + self._fps_last_health_log = current_time + else: + logger.debug( + "Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms", + fps, target, fps_frame_count, + p99 * 1000.0, frame_worst * 1000.0 + ) + self._fps_was_degraded = degraded last_fps_log_time = current_time fps_frame_count = 0 frame_worst = 0.0 diff --git a/test/test_vegas_fps_health.py b/test/test_vegas_fps_health.py new file mode 100644 index 00000000..7db126e6 --- /dev/null +++ b/test/test_vegas_fps_health.py @@ -0,0 +1,99 @@ +"""Frame pacing and FPS health reporting must not depend on the wall clock. + +These devices have no RTC, so the system clock jumps by however wrong boot +time was the moment NTP first syncs. The render loop sleeps the *remainder* +of each frame budget: + + frame_elapsed = - frame_started + time.sleep(max(0.0, frame_interval - frame_elapsed)) + +With a wall-clock `now`, a backward jump makes frame_elapsed negative, so +`frame_interval - frame_elapsed` exceeds the whole budget and the render loop +stalls for the size of the correction. A forward jump instead inflates the +p99 and worst-frame numbers the telemetry reports. +""" +import ast +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +COORD = (Path(__file__).resolve().parent.parent + / "src" / "vegas_mode" / "coordinator.py") +TREE = ast.parse(COORD.read_text(encoding="utf-8")) + + +def _assignments_of(name): + """Every `name = ` in the module, as unparsed source.""" + out = [] + for node in ast.walk(TREE): + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == name: + out.append((node.lineno, ast.unparse(node.value))) + return out + + +def test_per_frame_timestamps_are_monotonic(): + for name in ("frame_started", "frame_elapsed"): + assigns = _assignments_of(name) + assert assigns, f"{name} is no longer assigned -- has the loop changed?" + for lineno, expr in assigns: + assert "time.time()" not in expr, ( + f"{name} at line {lineno} uses the wall clock ({expr!r}). A " + "backward NTP step makes the per-frame delta negative and the " + "loop then sleeps longer than the whole frame budget.") + assert "time.monotonic()" in expr, ( + f"{name} at line {lineno} is {expr!r}, expected monotonic") + + +def test_the_fps_window_is_monotonic(): + for lineno, expr in _assignments_of("current_time"): + assert "time.monotonic()" in expr, ( + f"current_time at line {lineno} is {expr!r}; fps is frames divided " + "by this delta, so a clock step would corrupt the rate itself") + + +def test_health_state_is_not_reset_every_iteration(): + """run_iteration() runs once per cycle -- locals here reset every few seconds. + + As locals, `last_fps_health_log = 0.0` made the 300s heartbeat fire on the + first sample of every iteration, and a recovery spanning two iterations was + never reported because was_degraded had already gone back to False. + """ + run_iteration = next( + (n for n in ast.walk(TREE) + if isinstance(n, ast.FunctionDef) and n.name == "run_iteration"), None) + assert run_iteration is not None, "run_iteration() not found" + + local_names = {t.id for n in ast.walk(run_iteration) + if isinstance(n, ast.Assign) + for t in n.targets if isinstance(t, ast.Name)} + for leaked in ("last_fps_health_log", "was_degraded"): + assert leaked not in local_names, ( + f"{leaked} is a local of run_iteration() again, so it resets every " + "cycle -- the heartbeat degenerates to once per iteration") + + body = ast.unparse(run_iteration) + assert "self._fps_last_health_log" in body and "self._fps_was_degraded" in body, ( + "the health state should live on the coordinator, across iterations") + + +def test_start_clears_stale_health_state(): + """A new run must not inherit "was degraded" from the previous one.""" + start = next((n for n in ast.walk(TREE) + if isinstance(n, ast.FunctionDef) and n.name == "start"), None) + assert start is not None, "start() not found" + body = ast.unparse(start) + assert "self._fps_last_health_log" in body and "self._fps_was_degraded" in body, ( + "start() does not reset the FPS health state") + + +def test_the_degraded_threshold_is_documented(): + """The 90% band is deliberate; say so where the constant is defined.""" + source = COORD.read_text(encoding="utf-8") + idx = source.index("_FPS_HEALTHY_FRACTION = ") + preamble = source[max(0, idx - 700):idx] + assert "90%" in preamble or "0.9" in preamble, ( + "the degradation threshold is not explained at its definition, so " + "'below target' reads as a bug rather than a deliberate band") From 6138a3cbefb5c78ded5aec412120c1e51e0a72cb Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:42:51 -0400 Subject: [PATCH 38/53] test(install): stop assuming pytest's tmp_path is on disk (#492) test_returns_nothing_when_tmpdir_is_already_disk_backed asserted that lm_disk_backed_tmpdir prints nothing when TMPDIR is already disk-backed, and used pytest's tmp_path as the "disk-backed" directory: # tmp_path is on the regular filesystem, so the default must be kept. assert call("lm_disk_backed_tmpdir", env={"TMPDIR": str(tmp_path)}) == "" That premise is false on the platform the helper was written for. Debian 13 mounts /tmp as tmpfs -- which is the entire reason lm_disk_backed_tmpdir exists -- and pytest puts tmp_path under /tmp. So on the target platform TMPDIR is memory-backed, the helper correctly answers /var/tmp, and the test fails: E AssertionError: assert '/var/tmp' == '' The helper is right; the test was wrong. Reproduced on a box where /tmp is tmpfs and / is ext4. The test now looks for a directory whose backing store is actually disk -- tmp_path, else a scratch dir under /var/tmp, else beside the library -- using the same findmnt lookup the helper itself uses, and skips only if no disk-backed directory exists anywhere. An earlier version of this fix skipped whenever tmp_path was tmpfs, which made it skip on every machine with a tmpfs /tmp; that is barely better than asserting the wrong thing, so it now searches instead of giving up. Verified: 31 passed, 0 skipped. Mutation-checked -- deleting the "is the current TMPDIR memory-backed?" guard from lm_disk_backed_tmpdir fails this test, so it still catches the regression it is there for. Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW Co-authored-by: Claude Opus 5 (1M context) --- test/test_install_lowmem.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/test/test_install_lowmem.py b/test/test_install_lowmem.py index 7143176a..7c9fbbc8 100644 --- a/test/test_install_lowmem.py +++ b/test/test_install_lowmem.py @@ -13,6 +13,7 @@ need root and mutate the system, so they are exercised manually instead. """ import subprocess +import tempfile from pathlib import Path import pytest @@ -31,6 +32,16 @@ def run_lib(snippet: str, env: dict | None = None) -> subprocess.CompletedProces ) +def _fstype_of(path: object) -> str: + """Filesystem type backing ``path``, via the same tool the helper uses.""" + result = subprocess.run( + ["findmnt", "-no", "FSTYPE", "--target", str(path)], + capture_output=True, text=True, + env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin"}, + ) + return result.stdout.strip() + + def call(fn: str, *args: object, env: dict | None = None) -> str: joined = " ".join(str(a) for a in args) result = run_lib(f"{fn} {joined}", env=env) @@ -195,8 +206,29 @@ class TestOomDetection: class TestDiskBackedTmpdir: def test_returns_nothing_when_tmpdir_is_already_disk_backed(self, tmp_path): - # tmp_path is on the regular filesystem, so the default must be kept. - assert call("lm_disk_backed_tmpdir", env={"TMPDIR": str(tmp_path)}) == "" + # Do not assume tmp_path is disk-backed. Debian 13 -- the platform this + # helper exists for -- mounts /tmp as tmpfs, and pytest puts tmp_path + # under /tmp, so this asserted against a *memory*-backed directory and + # failed on the target platform while the helper behaved exactly as + # designed. Search for a directory whose backing store is really disk. + scratch = None + disk_backed = None + for candidate in (tmp_path, Path("/var/tmp"), LIB.parent): + if _fstype_of(candidate) not in ("tmpfs", "ramfs", ""): + if candidate is tmp_path: + disk_backed = candidate + else: + scratch = Path(tempfile.mkdtemp(dir=str(candidate))) + disk_backed = scratch + break + if disk_backed is None: + pytest.skip("no disk-backed directory available to test against") + try: + assert call("lm_disk_backed_tmpdir", + env={"TMPDIR": str(disk_backed)}) == "" + finally: + if scratch is not None: + scratch.rmdir() def test_redirects_away_from_a_memory_backed_tmpdir(self): # Debian 13 mounts /tmp as tmpfs, which would otherwise hold the whole From 5a1f121e6be43513f197608a4fc603cecd186905 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:43:08 -0400 Subject: [PATCH 39/53] Stop array-item secrets being wiped, and logging them (#493) Three review findings from #485 that I missed when addressing that PR; it has since merged, so they land here. 1. Array-item secrets destroyed by any unrelated save (data loss). remove_empty_secrets recursed into dicts but let a list fall through to the scalar branch and kept it verbatim. Lists merge by *replacement*, so the blanks the masked form posts back went straight over the stored array: stored [{"name":"a","token":"REAL-A"}, {"name":"b","token":"REAL-B"}] posted [{"name":"a","token":""}, {"name":"b","token":""}] merged [{"name":"a","token":""}, {"name":"b","token":""}] -> both credentials gone Same failure as the scalar api_key case fixed earlier, one container deeper. Lists now prune element-wise, and a list with nothing real in it is dropped so the stored one is left alone. Where one entry does change, the new merge_secrets merges by index instead of replacing. Two details the first attempt got wrong, both caught by existing tests: - An emptied dict item must stay {}, not None. ConfigManager's _strip_secrets_recursive treats a secrets list as *parallel* to the regular one ({} = "item i has no secrets"); a None makes it stop looking parallel, and it then drops the whole key from the main config -- silently deleting the items' non-secret fields too. - The incoming list's length wins. The regular config's list is authoritative about how many items exist, so preserving surplus stored entries would let the two fall out of step and make deleting an entry impossible. 2. Submitted credentials written to the journal (security). save_plugin_config logged `Full config: {plugin_config}` at INFO and `Config that failed: {plugin_config}` at ERROR. Both run before separate_secrets, so plugin_config still held the values just typed into the form. Now keys only. Swept the rest of web_interface/ and src/ for the same shape -- these were the only two. 3. Restart banner kept stale wording. showRestartPending() cleared the stored custom text but left the DOM element alone, so a config save could show the previous update's message. The default is read back from the server-rendered copy rather than duplicated in JS, so the template stays the one owner of the string. Verified: 556 passed, 1 skipped across the web suite. Mutation-checked -- reverting api_v3 fails the logging guard and the array-merge test; reverting either half of the secret_helpers change fails the unit tests. New end-to-end coverage drives the real endpoint, not just the helpers. Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW Co-authored-by: Claude Opus 5 (1M context) --- src/web_interface/secret_helpers.py | 80 ++++++++++++++++++- .../test_api_v3_secret_roundtrip.py | 38 +++++++++ .../test_config_logging_omits_secrets.py | 45 +++++++++++ test/web_interface/test_secret_helpers.py | 65 +++++++++++++++ web_interface/blueprints/api_v3.py | 19 +++-- web_interface/static/v3/app.js | 12 ++- 6 files changed, 252 insertions(+), 7 deletions(-) create mode 100644 test/web_interface/test_config_logging_omits_secrets.py diff --git a/src/web_interface/secret_helpers.py b/src/web_interface/secret_helpers.py index 6a00cb94..9aa2eccc 100644 --- a/src/web_interface/secret_helpers.py +++ b/src/web_interface/secret_helpers.py @@ -5,7 +5,7 @@ Provides functions for identifying, masking, separating, and filtering secret fields in plugin configurations based on JSON Schema x-secret markers. """ -from typing import Any, Dict, Set, Tuple +from typing import Any, Dict, Optional, Set, Tuple def find_secret_fields(properties: Dict[str, Any], prefix: str = '') -> Set[str]: @@ -202,11 +202,89 @@ def remove_empty_secrets(secrets: Dict[str, Any]) -> Dict[str, Any]: nested = remove_empty_secrets(v) if nested: result[k] = nested + elif isinstance(v, list): + # Lists used to fall through to the scalar branch below and be + # kept verbatim, blanks and all. Because lists merge by + # *replacement*, saving any unrelated setting then wrote + # [{"token": ""}, ...] straight over the stored list and + # destroyed every credential in it. + pruned = _prune_secret_list(v) + if pruned is not None: + result[k] = pruned elif v is not None and not (isinstance(v, str) and v.strip() == ''): result[k] = v return result +def _prune_secret_list(items: list) -> Optional[list]: + """Strip blanks from inside a list of secrets, preserving every index. + + The rest of the system treats a secrets list as *parallel* to the regular + one -- ``sec[i]`` holds the secret fields of item ``i``, and ``{}`` means + "item i has none" (see ConfigManager._strip_secrets_recursive). So an + emptied dict item stays ``{}``: putting ``None`` there makes that list stop + looking parallel, and the stripper then drops the whole key from the main + config, taking the non-secret fields with it. + + A blank *scalar* becomes ``None``, meaning "no update at this index" -- + :func:`merge_secrets` substitutes whatever is stored there. Returns + ``None`` when nothing in the list carries a real value, so the caller drops + the key and leaves the stored list untouched. + """ + pruned: list = [] + has_real_value = False + for item in items: + if isinstance(item, dict): + kept = remove_empty_secrets(item) + pruned.append(kept) + has_real_value = has_real_value or bool(kept) + elif isinstance(item, list): + sub = _prune_secret_list(item) + pruned.append(sub if sub is not None else []) + has_real_value = has_real_value or sub is not None + elif item is not None and not (isinstance(item, str) and item.strip() == ''): + pruned.append(item) + has_real_value = True + else: + pruned.append(None) + return pruned if has_real_value else None + + +def merge_secrets(stored: Any, incoming: Any) -> Any: + """Merge submitted secrets over stored ones, element-wise inside lists. + + ``deep_merge`` replaces a list wholesale. For secrets that is destructive: + an incoming list that carries a real value for one entry and ``None`` for + the rest would drop the stored credentials of every other entry. Here a + list merges by index, and ``None`` means "keep what is stored". + + Entries are matched by *position*, which is what the config form gives us + -- there is no schema-declared identity to key on, and it is the same + contract ConfigManager._strip_secrets_recursive already relies on. The + incoming list's length wins, so deleting an item deletes its secrets; + an item the client left blank keeps whatever is stored at that index. + """ + if isinstance(stored, dict) and isinstance(incoming, dict): + merged = dict(stored) + for key, value in incoming.items(): + merged[key] = (merge_secrets(stored[key], value) + if key in stored else value) + return merged + if isinstance(stored, list) and isinstance(incoming, list): + # The incoming list sets the length -- the regular config's list is + # authoritative about how many items exist, and this one runs parallel + # to it. Removing an entry must therefore remove its secrets too. + merged_list = [] + for index, item in enumerate(incoming): + stored_item = stored[index] if index < len(stored) else None + merged_list.append(stored_item if item is None + else merge_secrets(stored_item, item)) + return merged_list + if incoming is None: + return stored + return incoming + + def strip_masked_values(secrets: Dict[str, Any]) -> Dict[str, Any]: """Remove values a client echoed back rather than changed. diff --git a/test/web_interface/test_api_v3_secret_roundtrip.py b/test/web_interface/test_api_v3_secret_roundtrip.py index 1d70f009..9d07a011 100644 --- a/test/web_interface/test_api_v3_secret_roundtrip.py +++ b/test/web_interface/test_api_v3_secret_roundtrip.py @@ -229,6 +229,44 @@ class TestSavePluginConfig: "REAL-KEY-0123456789", "an unrelated edit destroyed the API key" assert env.fresh_load()[PLUGIN_ID]["city"] == "Dallas" + def test_an_unrelated_edit_does_not_erase_array_item_secrets(self, env): + """The scalar api_key case above, but for a list of credentials. + + remove_empty_secrets recursed into dicts only, so a list went into + deep_merge untouched -- and lists merge by *replacement*. Saving any + unrelated field posted [{"token": ""}, ...] straight over the stored + array and destroyed every token in it at once. + """ + assert self._save(env, {"accounts": [ + {"name": "a", "token": "REAL-A"}, + {"name": "b", "token": "REAL-B"}, + ], "city": "Austin"}).status_code == 200 + + # the user changes the city; both masked tokens ride along blank + assert self._save(env, {"accounts": [ + {"name": "a", "token": ""}, + {"name": "b", "token": ""}, + ], "city": "Dallas"}).status_code == 200 + + merged = env.fresh_load()[PLUGIN_ID] + assert [a.get("token") for a in merged["accounts"]] == \ + ["REAL-A", "REAL-B"], "an unrelated edit destroyed the array secrets" + assert [a["name"] for a in merged["accounts"]] == ["a", "b"] + assert merged["city"] == "Dallas" + + def test_one_array_secret_can_be_changed_without_losing_the_rest(self, env): + assert self._save(env, {"accounts": [ + {"name": "a", "token": "REAL-A"}, + {"name": "b", "token": "REAL-B"}, + ]}).status_code == 200 + assert self._save(env, {"accounts": [ + {"name": "a", "token": ""}, + {"name": "b", "token": "NEW-B"}, + ]}).status_code == 200 + + merged = env.fresh_load()[PLUGIN_ID] + assert [a.get("token") for a in merged["accounts"]] == ["REAL-A", "NEW-B"] + def test_a_secret_can_still_be_changed(self, env): """Dropping blanks must not stop a real new value from being saved.""" self._save(env, {"api_key": "first-key"}) diff --git a/test/web_interface/test_config_logging_omits_secrets.py b/test/web_interface/test_config_logging_omits_secrets.py new file mode 100644 index 00000000..031eb4f1 --- /dev/null +++ b/test/web_interface/test_config_logging_omits_secrets.py @@ -0,0 +1,45 @@ +"""The validation logging ran before separate_secrets, so it logged credentials. + +api_v3's plugin-config save logged `Full config: {plugin_config}` at INFO and +`Config that failed: {plugin_config}` at ERROR. Both run *before* +separate_secrets(), so plugin_config still held the values the user just typed +into the form -- API keys and tokens went to the journal in clear text. +""" +import re +from pathlib import Path + +import pytest + +SOURCE = (Path(__file__).resolve().parents[2] + / "web_interface" / "blueprints" / "api_v3.py") + +#: Objects that still hold submitted secret values at the point these log +#: calls run. Interpolating one whole into a log message leaks credentials. +UNREDACTED = ("plugin_config", "secrets_config", "current_secrets") + + +def _logging_lines(): + for number, line in enumerate(SOURCE.read_text(encoding="utf-8").splitlines(), 1): + stripped = line.strip() + if stripped.startswith("#"): + continue + if re.match(r"logger\.(debug|info|warning|error|critical|exception)\(", stripped): + yield number, stripped + + +@pytest.mark.parametrize("name", UNREDACTED) +def test_no_log_call_interpolates_a_whole_secret_bearing_object(name): + # {name} or {name['k']} leaks; {list(name.keys())} and {len(name)} do not. + bare = re.compile(r"\{" + re.escape(name) + r"(\[[^\]]*\])*\}") + offenders = [f"{n}: {text}" for n, text in _logging_lines() if bare.search(text)] + assert not offenders, ( + f"{name} still holds submitted secrets where these log calls run:\n " + + "\n ".join(offenders)) + + +def test_the_guard_would_notice_a_reintroduced_leak(): + """Pin the detector itself, so a rewrite cannot silently stop matching.""" + bare = re.compile(r"\{" + re.escape("plugin_config") + r"(\[[^\]]*\])*\}") + assert bare.search('logger.info(f"Full config: {plugin_config}")') + assert bare.search("logger.error(f\"{plugin_config['api_key']}\")") + assert not bare.search('logger.info(f"{list(plugin_config.keys())}")') diff --git a/test/web_interface/test_secret_helpers.py b/test/web_interface/test_secret_helpers.py index 5f33e899..a9eb4a7a 100644 --- a/test/web_interface/test_secret_helpers.py +++ b/test/web_interface/test_secret_helpers.py @@ -17,6 +17,7 @@ from src.web_interface.secret_helpers import ( separate_secrets, mask_secret_fields, mask_all_secret_values, + merge_secrets, remove_empty_secrets, ) @@ -239,3 +240,67 @@ class TestRemoveEmptySecrets: def test_keeps_falsey_non_string_values(self): # 0 and False are neither None nor blank strings — they are kept. assert remove_empty_secrets({"a": 0, "b": False}) == {"a": 0, "b": False} + + +class TestArrayItemSecrets: + """Lists merge by replacement, so a blanked array wipes stored credentials. + + remove_empty_secrets recursed into dicts but let a list through untouched, + so [{"token": ""}] went straight into deep_merge and overwrote the stored + list. Saving any unrelated setting destroyed every token in the array. + """ + + STORED = {"accounts": [{"name": "a", "token": "REAL-A"}, + {"name": "b", "token": "REAL-B"}]} + + def test_an_unrelated_save_keeps_every_stored_token(self): + posted = {"accounts": [{"name": "a", "token": ""}, + {"name": "b", "token": ""}]} + merged = merge_secrets(self.STORED, remove_empty_secrets(posted)) + assert [a["token"] for a in merged["accounts"]] == ["REAL-A", "REAL-B"] + + def test_editing_one_entry_leaves_the_others_alone(self): + posted = {"accounts": [{"name": "a", "token": ""}, + {"name": "b", "token": "NEW-B"}]} + merged = merge_secrets(self.STORED, remove_empty_secrets(posted)) + assert [a["token"] for a in merged["accounts"]] == ["REAL-A", "NEW-B"] + + def test_a_new_entry_is_appended(self): + posted = {"accounts": [{"name": "a", "token": ""}, + {"name": "b", "token": ""}, + {"name": "c", "token": "NEW-C"}]} + merged = merge_secrets(self.STORED, remove_empty_secrets(posted)) + assert [a["token"] for a in merged["accounts"]] == \ + ["REAL-A", "REAL-B", "NEW-C"] + + def test_a_list_of_bare_strings_merges_by_index(self): + merged = merge_secrets({"keys": ["K1", "K2", "K3"]}, + remove_empty_secrets({"keys": ["", "K2-NEW", ""]})) + assert merged["keys"] == ["K1", "K2-NEW", "K3"] + + def test_an_all_blank_list_is_dropped_entirely(self): + posted = {"accounts": [{"token": ""}, {"token": ""}]} + assert "accounts" not in remove_empty_secrets(posted) + + def test_plain_dict_secrets_are_unaffected(self): + merged = merge_secrets({"api_key": "OLD", "other": "keep"}, + remove_empty_secrets({"api_key": "", "other": "changed"})) + assert merged == {"api_key": "OLD", "other": "changed"} + + def test_a_removed_entry_takes_its_secret_with_it(self): + """The regular config's list is authoritative about how many items + exist, and the secrets list runs parallel to it -- see + ConfigManager._strip_secrets_recursive. So a shorter incoming list + must shorten the stored secrets too, or the two fall out of step.""" + posted = {"accounts": [{"name": "a", "token": "NEW-A"}]} + merged = merge_secrets(self.STORED, remove_empty_secrets(posted)) + assert [a["token"] for a in merged["accounts"]] == ["NEW-A"] + + def test_an_emptied_item_stays_a_dict_not_none(self): + """None there stops the list looking parallel, and + _strip_secrets_recursive then drops the whole key from the main + config -- deleting the item's non-secret fields as well.""" + pruned = remove_empty_secrets( + {"accounts": [{"token": "real"}, {"token": ""}]}) + assert pruned["accounts"] == [{"token": "real"}, {}] + assert None not in pruned["accounts"] diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 37df1141..958f5f0c 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -22,7 +22,8 @@ logger = logging.getLogger(__name__) from src.web_interface.api_helpers import success_response, error_response, validate_request_json from src.web_interface.errors import ErrorCode from src.web_interface.secret_helpers import (find_secret_fields, mask_all_secret_values, - remove_empty_secrets, separate_secrets, + merge_secrets, remove_empty_secrets, + separate_secrets, strip_masked_values) from src.web_interface.error_handler import describe_exception, redact_text from src.plugin_system.operation_types import OperationType @@ -1296,7 +1297,10 @@ def save_main_config(): if secrets_config: if plugin_id not in current_secrets: current_secrets[plugin_id] = {} - current_secrets[plugin_id] = deep_merge(current_secrets[plugin_id], secrets_config) + # Lists merge by replacement, so deep_merge here wrote a + # blanked array straight over the stored credentials. + current_secrets[plugin_id] = merge_secrets( + current_secrets[plugin_id], secrets_config) # Save secrets file api_v3.config_manager.save_raw_file_content('secrets', current_secrets) @@ -5675,8 +5679,10 @@ def save_plugin_config(): if schema: # Log what we're validating for debugging logger.info(f"Validating config for {plugin_id}") + # Only the shape. plugin_config still holds the submitted secret + # values at this point -- separate_secrets does not run until + # below -- so logging it wrote live credentials to the journal. logger.info(f"Config keys being validated: {list(plugin_config.keys())}") - logger.info(f"Full config: {plugin_config}") # Get enhanced schema keys (including injected core properties) # We need to create an enhanced schema to get the actual allowed keys @@ -5699,7 +5705,8 @@ def save_plugin_config(): # Log validation errors for debugging logger.error(f"Config validation failed for {plugin_id}") logger.error(f"Validation errors: {validation_errors}") - logger.error(f"Config that failed: {plugin_config}") + # Keys only, for the same reason as above. + logger.error(f"Config keys that failed: {list(plugin_config.keys())}") logger.error(f"Schema properties: {list(enhanced_schema.get('properties', {}).keys())}") # Also print to console for immediate visibility @@ -5750,7 +5757,9 @@ def save_plugin_config(): if secrets_config: if plugin_id not in current_secrets: current_secrets[plugin_id] = {} - current_secrets[plugin_id] = deep_merge(current_secrets[plugin_id], secrets_config) + # See above -- secrets lists must merge element-wise. + current_secrets[plugin_id] = merge_secrets( + current_secrets[plugin_id], secrets_config) # Save secrets file try: api_v3.config_manager.save_raw_file_content('secrets', current_secrets) diff --git a/web_interface/static/v3/app.js b/web_interface/static/v3/app.js index 581c66f2..638204b3 100644 --- a/web_interface/static/v3/app.js +++ b/web_interface/static/v3/app.js @@ -126,7 +126,17 @@ window.showRestartPending = function(message) { } catch { /* private browsing */ } const banner = document.getElementById('restart-pending-banner'); const text = document.getElementById('restart-pending-text'); - if (text && message) text.textContent = message; + if (text) { + // Without the else-branch a config save inherited whatever wording the + // previous update left in the DOM: showRestartPending() clears the + // stored text but used to leave the element itself alone. The default + // is read back from the server-rendered copy rather than duplicated + // here, so the template stays the one place that owns the string. + if (text.dataset.defaultText === undefined) { + text.dataset.defaultText = text.textContent.trim(); + } + text.textContent = message || text.dataset.defaultText; + } if (banner) banner.style.display = 'block'; }; From c321b94085bdf5c1e813274a930c0064f71d5612 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:43:39 -0400 Subject: [PATCH 40/53] fix(display): retry a plugin that is enabled but failed to load (#495) * fix(display): retry a plugin that is enabled but failed to load A plugin whose validate_config() returns False is treated as a hard load failure. The API then reports enabled=true, loaded=false, error=null: the plugin is simply absent, with nothing saying why. hockey-scoreboard sat in that state on a live rig for four days. The recovery path existed but could not be reached. _reconcile_enabled_plugins computes to_add = desired - current, and a plugin that failed to load is never in current, so it stays in to_add and would be retried. But the reconcile is queued by _enabled_set_changed(), which compares only top-level `enabled` flags -- and the edit that actually fixes such a plugin (enabling a league, filling in an API key) is nested inside the plugin's own config section. No top-level flag changes, so no reconcile is queued, and the save that should have fixed it does nothing. Only toggling some unrelated plugin -- which does change a top-level flag -- queues the global reconcile that recovers it. Add a second gate: queue a reconcile when a discovered plugin is enabled in config but absent from the running set. It is deliberately narrow rather than "reconcile on any config change". Reconcile calls discover_plugins(), a ~39-manifest filesystem scan, and it runs on the render thread; doing that on every config save would trade this bug for a frame hitch. Gating on plugin_manifests also keeps non-plugin sections that carry their own `enabled` flag (schedule, display) from queueing a reconcile they can never satisfy. In the steady state -- every enabled plugin loaded -- the new check is False and costs nothing. The same valid-but-unconfigured => hard-fail shape still exists in text-display, youtube-stats, birdnet-go, ledmatrix-flights and mqtt-notifications; this makes all of them recoverable without a restart. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(display): snapshot the plugin mappings under their locks Addresses the review finding on the cross-thread reads. _enabled_plugin_not_running runs on the config-watcher thread and read two mappings the render thread mutates. Catching RuntimeError was not a fix: it turned a torn read into a coin flip between an unnecessary discovery scan and a missed retry, which is the bug this PR exists to remove. Both reads are now snapshots taken under the lock that guards their writes: - plugin_manifests via a new PluginManager.discovered_plugin_ids(), which copies the ids while holding the existing _discovery_lock. Discovery rebuilds that mapping entry by entry, so an unsynchronised reader can see it half-populated. - plugin_display_modes under a new controller lock, taken at the only two sites that mutate it (_register_loaded_plugin / _unregister_plugin). The locks are never nested -- each snapshot is taken and released before the next -- so this cannot deadlock against discovery, which holds _discovery_lock while it rebuilds. No cost on the per-frame path. Both mutation sites run during reconcile, which is rare, and every hot-path read of plugin_display_modes is on the render thread itself, same thread as the writes, so those stay lock-free. Tests: the accessor returns a snapshot rather than a live view, and actually takes the discovery lock (proved from a second thread, since an RLock is reentrant on the owning one) so a later refactor cannot quietly drop it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(display): consume the reconcile request before serving it Addresses the second review finding: a lost update on _pending_plugin_reconcile. The flag was cleared after a successful reconcile. Reconcile has already read its config by that point, so a config change arriving mid-flight set a flag that the trailing clear then erased -- a request that was never served, and the newest config never reconciled. That is the same "my save did nothing" symptom this PR exists to remove, so leaving it would have undercut the fix. Consume the request before running it instead, and re-arm only on a retryable failure. A change that lands during reconcile now stays set and is picked up on the next pass. The per-frame read stays lock-free. It is a fast path that can only produce a false negative -- the watcher setting the flag just after it is read is seen on the next iteration -- never a false positive that loses a request. The lock is taken only when a reconcile is actually pending or a config change arrives. Extracted _service_pending_reconcile() so the sequence is testable rather than buried in run()'s loop; the review asked for a regression test that invokes the subscriber during reconciliation, which is not reachable otherwise. Tests: 4 new, covering a request racing in mid-reconcile, the quiet success, the retryable-failure re-arm, and not reconciling when nothing is pending. Two of them fail against the previous clear-after-success semantics. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 (1M context) --- src/display_controller.py | 92 ++++++++- src/plugin_system/plugin_manager.py | 11 ++ test/test_display_controller_plugin_toggle.py | 180 ++++++++++++++++++ test/test_plugin_manager_discovered_ids.py | 63 ++++++ 4 files changed, 338 insertions(+), 8 deletions(-) create mode 100644 test/test_plugin_manager_discovered_ids.py diff --git a/src/display_controller.py b/src/display_controller.py index e739e08e..bb650698 100644 --- a/src/display_controller.py +++ b/src/display_controller.py @@ -181,6 +181,16 @@ class DisplayController: self.plugin_modes = {} # mode -> plugin_instance mapping for plugin-first dispatch self.mode_to_plugin_id: Dict[str, str] = {} self.plugin_display_modes: Dict[str, List[str]] = {} + # plugin_display_modes is mutated only by _register_loaded_plugin / + # _unregister_plugin on the render thread, but the config-watcher + # thread reads it in _enabled_plugin_not_running. Both mutation sites + # run during reconcile (rare), so this lock never touches the per-frame + # path -- the hot-path reads are same-thread as the writes. + self._plugin_modes_lock = threading.Lock() + # Guards the consume-and-clear of _pending_plugin_reconcile. Only taken + # when a reconcile is actually pending or a config change arrives, both + # rare -- the per-frame path just reads the bool. + self._reconcile_flag_lock = threading.Lock() # Per-plugin config-change callbacks, kept so we can unsubscribe a # plugin when it is disabled live. self._plugin_config_callbacks: Dict[str, Callable] = {} @@ -463,8 +473,10 @@ class DisplayController: self._refresh_config_cache(new_config) # If a plugin was enabled/disabled, flag a reconcile for the main # loop to apply (loading/unloading off the watcher thread is unsafe). - if self._enabled_set_changed(old_config, new_config): - self._pending_plugin_reconcile = True + if (self._enabled_set_changed(old_config, new_config) + or self._enabled_plugin_not_running(new_config)): + with self._reconcile_flag_lock: + self._pending_plugin_reconcile = True self.config_service.subscribe(_controller_config_change) @@ -1749,11 +1761,12 @@ class DisplayController: # rebuilding available_modes happens here on the render thread so # it can't race with rendering. Deferred while on-demand is active # (the flag stays set) so we don't fight its temporary-enable. + # The lock-free read is a fast path only; it can be a false + # negative (the watcher setting the flag just after it is read + # is seen next iteration), never a false positive that loses a + # request. if self._pending_plugin_reconcile and not self.on_demand_active: - # Only clear the flag on success -- a retryable failure - # (e.g. discovery) leaves it set so the request isn't lost. - if self._reconcile_enabled_plugins(): - self._pending_plugin_reconcile = False + self._service_pending_reconcile() if not self.available_modes: # Nothing to render yet. Re-check _pending_plugin_reconcile @@ -2813,7 +2826,8 @@ class DisplayController: logger.debug("Using manifest display_modes for %s: %s", plugin_id, display_modes) if not (isinstance(display_modes, list) and display_modes): display_modes = [plugin_id] - self.plugin_display_modes[plugin_id] = list(display_modes) + with self._plugin_modes_lock: + self.plugin_display_modes[plugin_id] = list(display_modes) # Subscribe to config changes for per-plugin hot-reload. Bind plugin_id # and instance as defaults so each plugin's callback targets its own @@ -2847,7 +2861,8 @@ class DisplayController: def _unregister_plugin(self, plugin_id: str) -> None: """Remove a plugin's modes, config subscription and instance, then unload it. Used by live disable hot-reload.""" - modes = self.plugin_display_modes.pop(plugin_id, []) + with self._plugin_modes_lock: + modes = self.plugin_display_modes.pop(plugin_id, []) for mode in modes: if mode in self.available_modes: self.available_modes.remove(mode) @@ -2892,6 +2907,67 @@ class DisplayController: } return enabled_map(old_config) != enabled_map(new_config) + def _service_pending_reconcile(self) -> None: + """Consume a pending reconcile request and run it. + + The request is consumed BEFORE reconciling, not cleared after. Clearing + after would drop any config change that lands while reconcile is + running: reconcile has already read its config by then, so the clear + erases a request it never served and the newest config never + reconciles -- the same "your save did nothing" failure this whole path + exists to prevent. Consuming first means such a request stays set and + is picked up on the next pass. + + A retryable failure (e.g. discovery) re-arms the flag. + """ + with self._reconcile_flag_lock: + pending = self._pending_plugin_reconcile + self._pending_plugin_reconcile = False + if pending and not self._reconcile_enabled_plugins(): + with self._reconcile_flag_lock: + self._pending_plugin_reconcile = True + + def _enabled_plugin_not_running(self, new_config: Dict[str, Any]) -> bool: + """True when a discovered plugin is enabled in config but not running. + + ``_enabled_set_changed`` compares only top-level ``enabled`` flags, which + misses the case that strands a plugin: one whose ``validate_config()`` + returned False is absent from the running set, and the edit that fixes it + (enabling a league, filling in an API key) lives *nested* inside that + plugin's own section. No top-level flag changes, so no reconcile is + queued, and the save that should have fixed it appears to do nothing -- + only toggling some unrelated plugin recovers it. hockey-scoreboard sat + enabled-but-absent on a live rig for four days this way. + + Deliberately narrow: it fires only for ids the plugin manager has + actually discovered, so non-plugin sections that carry their own + ``enabled`` flag (``schedule``, ``display``, ...) don't queue a reconcile + on every save. In the steady state -- everything enabled is loaded -- + this is False and costs nothing. That matters because reconcile runs + ``discover_plugins()`` on the render thread, where a needless + filesystem scan per config save would show up as a frame hitch. + + Runs on the config-watcher thread, so both mappings it reads are + snapshotted under the lock that guards their writes. + """ + if self.plugin_manager is None: + return False + # Two snapshots, each taken under its own lock and never nested, so a + # half-written mapping is never observed and this can't deadlock + # against discovery (which holds the discovery lock while rebuilding). + try: + known = self.plugin_manager.discovered_plugin_ids() + except AttributeError: + # Older manager without the accessor: fall back to a plain read. + known = set(getattr(self.plugin_manager, 'plugin_manifests', ()) or ()) + with self._plugin_modes_lock: + running = set(self.plugin_display_modes) + for key, value in new_config.items(): + if (key in known and isinstance(value, dict) + and value.get('enabled', False) and key not in running): + return True + return False + def _reconcile_enabled_plugins(self) -> bool: """Load/unload plugins so the running set matches the enabled set in config. Runs on the main display thread (never the config-watcher diff --git a/src/plugin_system/plugin_manager.py b/src/plugin_system/plugin_manager.py index 543017c9..0dc7a428 100644 --- a/src/plugin_system/plugin_manager.py +++ b/src/plugin_system/plugin_manager.py @@ -631,6 +631,17 @@ class PluginManager: return self.load_plugin(plugin_id) + def discovered_plugin_ids(self) -> set: + """Snapshot of the discovered plugin ids, taken under the discovery lock. + + Callers on other threads (the config watcher) must not iterate + ``plugin_manifests`` directly: discovery rebuilds it entry by entry, so + an unsynchronised reader can see a half-populated mapping or raise + "dictionary changed size during iteration". + """ + with self._discovery_lock: + return set(self.plugin_manifests) + def get_plugin(self, plugin_id: str) -> Optional[Any]: """ Get a loaded plugin instance by ID. diff --git a/test/test_display_controller_plugin_toggle.py b/test/test_display_controller_plugin_toggle.py index 0980e729..e09933d0 100644 --- a/test/test_display_controller_plugin_toggle.py +++ b/test/test_display_controller_plugin_toggle.py @@ -6,6 +6,7 @@ These tests cover the reconcile path that loads/unloads plugins and rebuilds the dispatch maps on the main thread when the enabled set changes. """ +import copy from unittest.mock import MagicMock @@ -253,3 +254,182 @@ class TestEnabledSetChanged: {"a": {"enabled": True, "duration": 30}}, {"a": {"enabled": True, "duration": 45}}, ) is False + + +class TestEnabledPluginNotRunning: + """A plugin that fails validate_config() is enabled but absent, and the + config edit that fixes it is nested inside the plugin's own section -- so + the top-level ``enabled`` comparison never sees it. These cover the second + gate that queues a reconcile in that case. + """ + + def test_nested_edit_is_invisible_to_the_enabled_set_check(self, test_display_controller): + """The original gate: proves why a second one is needed.""" + controller = test_display_controller + old = {"hockey-scoreboard": {"enabled": True, "nhl": {"enabled": False}}} + new = {"hockey-scoreboard": {"enabled": True, "nhl": {"enabled": True}}} + # Enabling a league changes no top-level flag. + assert controller._enabled_set_changed(old, new) is False + + def test_queues_reconcile_when_enabled_plugin_is_absent(self, test_display_controller): + controller = test_display_controller + controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}} + controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"} + controller.plugin_display_modes = {} # failed to load + cfg = {"hockey-scoreboard": {"enabled": True, "nhl": {"enabled": True}}} + assert controller._enabled_plugin_not_running(cfg) is True + + def test_quiet_when_every_enabled_plugin_is_running(self, test_display_controller): + controller = test_display_controller + controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}} + controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"} + controller.plugin_display_modes = {"hockey-scoreboard": ["nhl"]} + cfg = {"hockey-scoreboard": {"enabled": True}} + assert controller._enabled_plugin_not_running(cfg) is False + + def test_disabled_plugin_does_not_queue(self, test_display_controller): + controller = test_display_controller + controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}} + controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"} + controller.plugin_display_modes = {} + cfg = {"hockey-scoreboard": {"enabled": False}} + assert controller._enabled_plugin_not_running(cfg) is False + + def test_non_plugin_sections_do_not_queue(self, test_display_controller): + """``schedule``/``display`` carry their own ``enabled`` and are never + in plugin_display_modes -- without the manifest check they would queue + a reconcile, and therefore a filesystem scan, on every config save.""" + controller = test_display_controller + controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}} + controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"} + controller.plugin_display_modes = {"hockey-scoreboard": ["nhl"]} + cfg = { + "hockey-scoreboard": {"enabled": True}, + "schedule": {"enabled": True}, + "display": {"enabled": True}, + } + assert controller._enabled_plugin_not_running(cfg) is False + + def test_non_dict_section_is_ignored(self, test_display_controller): + controller = test_display_controller + controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}} + controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"} + controller.plugin_display_modes = {} + assert controller._enabled_plugin_not_running({"hockey-scoreboard": "nonsense"}) is False + + def test_no_plugin_manager_is_quiet(self, test_display_controller): + controller = test_display_controller + controller.plugin_manager = None + assert controller._enabled_plugin_not_running({"x": {"enabled": True}}) is False + + +class TestReconcileQueuedThroughSubscriber: + """End-to-end through the real config-change subscriber, not the helper. + + Without the second gate this is the four-day-outage path: the plugin is + enabled, absent, and the save that enables its league sets no flag. + """ + + @staticmethod + def _subscriber(controller): + subs = controller.config_service._subscribers['*'] + for cb in subs: + if getattr(cb, '__name__', '') == '_controller_config_change': + return cb + raise AssertionError(f"controller subscriber not found among {subs}") + + @staticmethod + def _configs(controller, plugin_section_old, plugin_section_new): + """Build two full configs differing only inside the plugin section -- + the subscriber refreshes its cache from these, so they must be real.""" + base = copy.deepcopy(controller.config) + old = copy.deepcopy(base) + new = copy.deepcopy(base) + old["hockey-scoreboard"] = plugin_section_old + new["hockey-scoreboard"] = plugin_section_new + return old, new + + def test_nested_edit_queues_reconcile_for_absent_plugin(self, test_display_controller): + controller = test_display_controller + controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}} + controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"} + controller.plugin_display_modes = {} # validate_config() said False + controller._pending_plugin_reconcile = False + + old, new = self._configs( + controller, + {"enabled": True, "nhl": {"enabled": False}}, + {"enabled": True, "nhl": {"enabled": True}}, + ) + # The original gate is blind to this edit ... + assert controller._enabled_set_changed(old, new) is False + self._subscriber(controller)(old, new) + # ... but the reconcile is queued anyway. + assert controller._pending_plugin_reconcile is True + + def test_steady_state_does_not_queue_reconcile(self, test_display_controller): + """Everything enabled is running: an unrelated edit must not queue a + reconcile, or every config save drags a filesystem scan onto the + render thread.""" + controller = test_display_controller + controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}} + controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"} + controller.plugin_display_modes = {"hockey-scoreboard": ["nhl"]} + controller._pending_plugin_reconcile = False + + old, new = self._configs( + controller, + {"enabled": True, "scroll_speed": 1}, + {"enabled": True, "scroll_speed": 2}, + ) + self._subscriber(controller)(old, new) + + assert controller._pending_plugin_reconcile is False + + +class TestPendingReconcileNotLost: + """A config change arriving *during* reconcile must not be discarded. + + The flag used to be cleared after a successful reconcile. Reconcile has + already read its config by then, so that clear erased a request it never + served and the newest config never reconciled -- the same "my save did + nothing" symptom this path exists to prevent. + """ + + def test_request_arriving_during_reconcile_survives(self, test_display_controller): + controller = test_display_controller + controller._pending_plugin_reconcile = True + + def reconcile_and_race(): + # The watcher thread queues another change while we are mid-flight. + with controller._reconcile_flag_lock: + controller._pending_plugin_reconcile = True + return True + + controller._reconcile_enabled_plugins = reconcile_and_race + controller._service_pending_reconcile() + + assert controller._pending_plugin_reconcile is True, \ + "a config change landing during reconcile was discarded" + + def test_flag_cleared_on_a_quiet_success(self, test_display_controller): + controller = test_display_controller + controller._pending_plugin_reconcile = True + controller._reconcile_enabled_plugins = lambda: True + controller._service_pending_reconcile() + assert controller._pending_plugin_reconcile is False + + def test_retryable_failure_rearms(self, test_display_controller): + controller = test_display_controller + controller._pending_plugin_reconcile = True + controller._reconcile_enabled_plugins = lambda: False + controller._service_pending_reconcile() + assert controller._pending_plugin_reconcile is True + + def test_no_reconcile_when_nothing_pending(self, test_display_controller): + controller = test_display_controller + controller._pending_plugin_reconcile = False + calls = [] + controller._reconcile_enabled_plugins = lambda: calls.append(1) or True + controller._service_pending_reconcile() + assert calls == [] diff --git a/test/test_plugin_manager_discovered_ids.py b/test/test_plugin_manager_discovered_ids.py new file mode 100644 index 00000000..5bedba8d --- /dev/null +++ b/test/test_plugin_manager_discovered_ids.py @@ -0,0 +1,63 @@ +"""Tests for PluginManager.discovered_plugin_ids(). + +The config-watcher thread needs the set of discovered plugin ids while the +render thread may be rebuilding plugin_manifests. Iterating that dict directly +can observe a half-populated mapping or raise "dictionary changed size during +iteration", so the accessor snapshots it under the discovery lock. +""" + +import tempfile +import threading +from pathlib import Path + +import pytest + +from src.plugin_system.plugin_manager import PluginManager + + +@pytest.fixture +def pm(): + with tempfile.TemporaryDirectory() as tmp: + yield PluginManager(plugins_dir=str(Path(tmp) / "plugins")) + + +def test_returns_the_discovered_ids(pm): + pm.plugin_manifests = {"clock-simple": {}, "hockey-scoreboard": {}} + assert pm.discovered_plugin_ids() == {"clock-simple", "hockey-scoreboard"} + + +def test_empty_when_nothing_discovered(pm): + pm.plugin_manifests = {} + assert pm.discovered_plugin_ids() == set() + + +def test_is_a_snapshot_not_a_live_view(pm): + """The caller iterates the result on another thread; it must not alias + the mapping discovery is still writing to.""" + pm.plugin_manifests = {"clock-simple": {}} + snapshot = pm.discovered_plugin_ids() + pm.plugin_manifests["hockey-scoreboard"] = {} + assert snapshot == {"clock-simple"} + + +def test_takes_the_discovery_lock(pm): + """Guards against the lock being dropped in a later refactor: with the + lock held by another thread the call must block rather than read.""" + pm.plugin_manifests = {"clock-simple": {}} + finished = threading.Event() + + def call(): + pm.discovered_plugin_ids() + finished.set() + + pm._discovery_lock.acquire() + try: + # RLock is reentrant per-thread, so use a *different* thread to prove + # the accessor actually waits on it. + t = threading.Thread(target=call, daemon=True) + t.start() + assert not finished.wait(timeout=0.3), "accessor did not take the discovery lock" + finally: + pm._discovery_lock.release() + t.join(timeout=2) + assert finished.is_set() From 085fb93a8745ffa80aab2d797a9c57594c5e0fea Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:44:00 -0400 Subject: [PATCH 41/53] test(logging): stop the location assertion matching the clock (#496) test_location_toggle asserted that ":42" -- a bare colon plus the record's hardcoded lineno -- is absent from a line formatted with include_location=False. But every formatted line starts with an HH:MM:SS.mmm timestamp, so ":42" also matches the clock whenever the minute or the second is 42. The test fails for roughly 3% of runs with nothing wrong: 2026-08-22 08:05:42.274 - INFO - test.logger - hello ^^^ matches ":42" Assert on the whole "module.funcName:lineno" token the format string actually emits ('%(module)s.%(funcName)s:%(lineno)d') instead of a fragment of it. That cannot collide with a timestamp, and it checks the thing the test is named for. Confirmed by formatting a record stamped 08:42:42 -- both minute and second colliding: the old assertion fails, the new one passes. Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW Co-authored-by: Claude Opus 5 (1M context) --- test/test_logging_config.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test_logging_config.py b/test/test_logging_config.py index d0cf0fe8..044a5d92 100644 --- a/test/test_logging_config.py +++ b/test/test_logging_config.py @@ -89,11 +89,17 @@ class TestContextualFormatter: assert "hello" in out def test_location_toggle(self): + # Assert on the whole "module.func:lineno" token, not a bare ":42". + # The formatted line starts with an HH:MM:SS timestamp, so a bare + # ":{lineno}" also matches the clock whenever the minute or second + # happens to equal the line number -- about 3% of runs, which is a + # flaky failure with nothing wrong. record = make_record() + location = f"{record.module}.{record.funcName}:{record.lineno}" with_loc = ContextualFormatter(include_location=True).format(record) without = ContextualFormatter(include_location=False).format(record) - assert f":{record.lineno}" in with_loc - assert f":{record.lineno}" not in without + assert location in with_loc + assert location not in without def test_record_not_mutated_no_double_prefix(self): # Regression: a record is formatted once PER HANDLER. The formatter From a4a55a23fc69bccd3883ad89974b6393fe9cb585 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sun, 23 Aug 2026 11:44:23 -0400 Subject: [PATCH 42/53] fix(web): reject non-finite JSON numbers instead of raising (#497) * fix(web): reject non-finite JSON numbers instead of raising POST /api/v3/config/dim-schedule with {"dim_brightness": Infinity} answered 500. So did /api/v3/errors/clear with max_age_hours, and /api/v3/config/main with multiplexing or row_address_type. json.loads accepts Infinity/-Infinity/NaN by default -- they are not valid JSON, but Python's parser emits them -- and Flask's get_json passes them straight through. int(float('inf')) raises OverflowError, which is neither ValueError nor TypeError, so validation blocks that carefully caught those let it past and Flask turned it into a 500. The status code was not the real damage. dim-schedule answered with CONFIG_SAVE_FAILED and suggested "Check file permissions on config directory" and "Check available disk space" for what was an invalid number. Every one of these sites already had a correct 400 response written; they just never reached it. NaN already returned 400, because int(nan) raises ValueError. That is why this only ever showed up for the infinities, and why it survived: the obvious test case passes. OverflowError is now caught alongside ValueError/TypeError at the 27 sites in this file whose try block performs a numeric coercion. An AST sweep confirms no int()/float() of request-derived data is left outside a block that catches it. Verified end to end through Flask's test client rather than by reasoning about the parser: all four routes returned 500 before and 400 after. Tests: five Infinity cases (which fail against the previous except tuples), two NaN cases pinned so narrowing the tuple cannot quietly break them, and a check that ordinary input is not rejected by the widened guard. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * test(web): assert 400 exactly, and prove valid input is accepted Both review points were right, and the first is the failure mode this file exists to catch. Accepting any 4xx meant a 404 would have passed. Renaming one of these routes would have left the test green while it tested nothing -- the same "looks like coverage, points somewhere safe" shape that hid the composer injections. Now asserts exactly 400. Both infinity signs are exercised for every route. int() raises OverflowError either way, but only +Infinity was in the original report, and a guard that special-cased the sign would have passed a one-sided test. The valid-input test previously asserted "not a 400", which did not show what it claimed: the mocked save path fails for any input, so that assertion held whether or not validation had accepted the value. It now gives load_config a real dict and stubs _save_config_atomic, so the endpoint reaches its success response and the test can assert 200 -- which only happens if the value passed validation. 8 of the 11 checks fail with OverflowError removed from the except tuples. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 (1M context) --- test/test_api_v3_non_finite_numbers.py | 85 ++++++++++++++++++++++++++ web_interface/blueprints/api_v3.py | 54 ++++++++-------- 2 files changed, 112 insertions(+), 27 deletions(-) create mode 100644 test/test_api_v3_non_finite_numbers.py diff --git a/test/test_api_v3_non_finite_numbers.py b/test/test_api_v3_non_finite_numbers.py new file mode 100644 index 00000000..4e6c217b --- /dev/null +++ b/test/test_api_v3_non_finite_numbers.py @@ -0,0 +1,85 @@ +"""Non-finite JSON numbers must be rejected, not raise. + +json.loads accepts Infinity/-Infinity/NaN by default (they are not valid JSON, +but Python's parser emits them) and Flask's get_json passes them straight +through. int(float('inf')) raises OverflowError, which is neither ValueError +nor TypeError -- so validation blocks that carefully caught those let it +through and Flask turned it into a 500. + +The damage was not the status code. /config/dim-schedule answered with +CONFIG_SAVE_FAILED and suggested "Check file permissions on config directory" +and "Check available disk space" for what was actually an invalid number. + +NaN already returned 400 (int(nan) raises ValueError), which is why this only +showed up for the infinities. +""" +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402 + + +#: (route, field) that returned 500 before OverflowError was caught. Both +#: infinity signs are exercised: int() raises OverflowError for either, but +#: only one of them was in the original report, and a guard that special-cased +#: the sign would pass a one-sided test. +NON_FINITE_ROUTES = [ + ('/api/v3/config/dim-schedule', 'dim_brightness'), + ('/api/v3/errors/clear', 'max_age_hours'), + ('/api/v3/config/main', 'multiplexing'), + ('/api/v3/config/main', 'row_address_type'), +] +NON_FINITE_CASES = [ + (route, '{"%s": %s}' % (field, literal)) + for route, field in NON_FINITE_ROUTES + for literal in ('Infinity', '-Infinity') +] + + +@pytest.mark.parametrize("route,body", NON_FINITE_CASES) +def test_infinity_is_a_client_error_not_a_server_error(api_v3_client, route, body): + """Exactly 400, not merely "some 4xx". + + Accepting any 4xx would let a 404 pass, so renaming one of these routes + would leave the test green while testing nothing -- the failure mode this + whole file exists to catch. + """ + response = api_v3_client.post(route, data=body, content_type='application/json') + assert response.status_code == 400, ( + f"{route} with {body} answered {response.status_code}; expected 400" + ) + + +@pytest.mark.parametrize("route,body", [ + ('/api/v3/config/dim-schedule', '{"dim_brightness": NaN}'), + ('/api/v3/errors/clear', '{"max_age_hours": NaN}'), +]) +def test_nan_is_also_a_client_error(api_v3_client, route, body): + """int(nan) raises ValueError so this path already worked -- pinned so a + refactor that narrows the except tuple cannot quietly break it.""" + response = api_v3_client.post(route, data=body, content_type='application/json') + assert response.status_code == 400 + + +def test_a_valid_number_is_accepted(api_v3_client, api_v3_module, monkeypatch): + """Prove the widened except did not start swallowing ordinary input. + + Asserting "not a 400" would not show that: the mocked save path fails for + any input, so the assertion would hold even if validation had rejected the + value. Give load_config a real dict and stub the atomic save, and the + endpoint reaches its success response -- which only happens if 30 passed + validation. + """ + api_v3_module.api_v3.config_manager.load_config.return_value = {} + monkeypatch.setattr(api_v3_module, '_save_config_atomic', + lambda *a, **k: (True, '')) + response = api_v3_client.post( + '/api/v3/config/dim-schedule', + data='{"dim_brightness": 30}', + content_type='application/json', + ) + assert response.status_code == 200, response.get_data(as_text=True)[:200] diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 958f5f0c..a5a2979f 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -598,7 +598,7 @@ def save_dim_schedule_config(): dim_brightness = 30 else: dim_brightness = int(dim_brightness_raw) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return error_response( ErrorCode.VALIDATION_ERROR, "dim_brightness must be an integer between 0 and 100", @@ -798,7 +798,7 @@ def save_main_config(): }), 400 try: target_fps = int(raw_target_fps) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({ 'status': 'error', 'message': "Invalid value for target_fps: must be an integer" @@ -868,7 +868,7 @@ def save_main_config(): mux_val = int(data['multiplexing']) if mux_val < 0 or mux_val > 22: return jsonify({'status': 'error', 'message': f"Invalid multiplexing value '{data['multiplexing']}'. Must be an integer from 0 to 22."}), 400 - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({'status': 'error', 'message': f"Invalid multiplexing value '{data['multiplexing']}'. Must be an integer from 0 to 22."}), 400 # Validate pixel_mapper_config (free-form mapper string, e.g. "U-mapper;Rotate:90") @@ -886,7 +886,7 @@ def save_main_config(): rat_val = int(data['row_address_type']) if rat_val < 0 or rat_val > 4: return jsonify({'status': 'error', 'message': f"Invalid row_address_type '{data['row_address_type']}'. Must be an integer from 0 to 4."}), 400 - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({'status': 'error', 'message': f"Invalid row_address_type '{data['row_address_type']}'. Must be an integer from 0 to 4."}), 400 # Handle hardware settings @@ -911,7 +911,7 @@ def save_main_config(): if rp1_val not in (0, 1): return jsonify({'status': 'error', 'message': "rp1_rio must be 0 (PIO) or 1 (RIO)"}), 400 current_config['display']['runtime']['rp1_rio'] = rp1_val - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({'status': 'error', 'message': "rp1_rio must be 0 or 1"}), 400 # Handle checkboxes - coerce to bool to ensure proper JSON types @@ -964,7 +964,7 @@ def save_main_config(): copies = None try: copies = int(data['double_sided_copies']) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): if enabled: return jsonify({'status': 'error', 'message': "Double-sided copies must be an integer"}), 400 if copies is not None and not (2 <= copies <= 8): @@ -1037,7 +1037,7 @@ def save_main_config(): if data.get('vegas_extend_threshold_screens') not in ('', None): try: screens = float(data['vegas_extend_threshold_screens']) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({ 'status': 'error', 'message': "Invalid value for vegas_extend_threshold_screens: " @@ -1054,7 +1054,7 @@ def save_main_config(): if data.get('vegas_max_plugin_width_ratio') not in ('', None): try: ratio = float(data['vegas_max_plugin_width_ratio']) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({ 'status': 'error', 'message': "Invalid value for vegas_max_plugin_width_ratio: " @@ -1102,7 +1102,7 @@ def save_main_config(): continue try: int_value = int(raw_value) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({ 'status': 'error', 'message': f"Invalid value for {field_name}: must be an integer" @@ -1154,7 +1154,7 @@ def save_main_config(): if not (1024 <= port_val <= 65535): return jsonify({'status': 'error', 'message': "sync_port must be between 1024 and 65535"}), 400 current_config['sync']['port'] = port_val - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({'status': 'error', 'message': "sync_port must be an integer"}), 400 if "sync_follower_position" in data: @@ -1198,7 +1198,7 @@ def save_main_config(): raw_value = data.pop(field) try: int_value = int(raw_value) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({'status': 'error', 'message': f"Invalid duration for {field}: must be an integer"}), 400 current_config['display']['display_durations'][field] = int_value @@ -1221,7 +1221,7 @@ def save_main_config(): continue try: int_value = int(raw_value) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({'status': 'error', 'message': f"Invalid duration for mode '{mode_key}': must be an integer"}), 400 current_config['display']['display_durations'][mode_key] = int_value @@ -5122,7 +5122,7 @@ def save_plugin_config(): converted_array.append(int(v)) else: converted_array.append(float(v)) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): converted_array.append(v) else: converted_array.append(v) @@ -5147,7 +5147,7 @@ def save_plugin_config(): converted_array.append(int(v)) else: converted_array.append(float(v)) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): converted_array.append(v) else: converted_array.append(v) @@ -5184,7 +5184,7 @@ def save_plugin_config(): converted_array.append(int(v)) else: converted_array.append(float(v)) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): converted_array.append(v) else: converted_array.append(v) @@ -5208,7 +5208,7 @@ def save_plugin_config(): converted_array.append(int(v)) else: converted_array.append(float(v)) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): converted_array.append(v) else: converted_array.append(v) @@ -5375,7 +5375,7 @@ def save_plugin_config(): if isinstance(v, str): try: converted.append(int(v) if item_type == 'integer' else float(v)) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): converted.append(v) else: converted.append(v) @@ -5500,7 +5500,7 @@ def save_plugin_config(): try: normalized[key] = int(value_stripped) continue - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): pass elif isinstance(value, (int, float)): normalized[key] = int(value) @@ -5518,7 +5518,7 @@ def save_plugin_config(): try: normalized[key] = float(value_stripped) continue - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): pass elif isinstance(value, (int, float)): normalized[key] = float(value) @@ -5573,7 +5573,7 @@ def save_plugin_config(): try: normalized_array.append(int(v)) continue - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): pass elif isinstance(v, (int, float)): normalized_array.append(int(v)) @@ -5583,7 +5583,7 @@ def save_plugin_config(): try: normalized_array.append(float(v)) continue - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): pass elif isinstance(v, (int, float)): normalized_array.append(float(v)) @@ -5599,7 +5599,7 @@ def save_plugin_config(): if isinstance(v, str): try: normalized_array.append(int(v)) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): normalized_array.append(v) elif isinstance(v, (int, float)): normalized_array.append(int(v)) @@ -5613,7 +5613,7 @@ def save_plugin_config(): if isinstance(v, str): try: normalized_array.append(float(v)) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): normalized_array.append(v) else: normalized_array.append(v) @@ -5636,7 +5636,7 @@ def save_plugin_config(): if isinstance(value, str): try: normalized[key] = int(value) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): normalized[key] = value else: normalized[key] = value @@ -5645,7 +5645,7 @@ def save_plugin_config(): if isinstance(value, str): try: normalized[key] = float(value) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): normalized[key] = value else: normalized[key] = value @@ -6788,7 +6788,7 @@ def get_font_preview() -> tuple[Response, int] | Response: # Safe integer parsing for size try: size = int(request.args.get('size', 12)) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return jsonify({'status': 'error', 'message': 'Invalid font size'}), 400 if not font_filename: @@ -8369,7 +8369,7 @@ def clear_old_errors(): context={'provided_value': raw_max_age}, status_code=400 ) - except (ValueError, TypeError): + except (ValueError, TypeError, OverflowError): return error_response( error_code=ErrorCode.INVALID_INPUT, message="max_age_hours must be a valid integer", From 333fd17d28a39e34cf21ee087b7dde27509a9e93 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:32:08 -0400 Subject: [PATCH 43/53] fix(memory): release fetched payloads once they have been delivered (#499) * fix(memory): release fetched payloads once they have been delivered BackgroundDataService kept the fetched body on the FetchResult it filed in completed_requests, which is swept hourly and capped at 500 entries by count. For a status record that costs nothing; for a season schedule it costs a tenth of the board. Measured on a 1GB Pi 3B+ with a 1-second RSS profile: the display process sat at 404MB after plugin load, then stepped +21MB when NFL fetched its season and +90MB when NCAA football fetched 946 games for 2026 -- and stayed at 494MB. Not a leak; a staircase that never came down. When a later fetch landed while headroom was low, available memory reached ~70MB, fork() began failing, and the board stopped being able to start a process at all: sshd accepted connections and closed them before its banner, systemd could not respawn the display, and the panel went dark while the kernel carried on answering pings. The cache-hit path was the worse of the two. It runs once per update interval per sport, mints a fresh request_id each time, and files whatever the cache returned. The memory tier is capped at 150 entries on a 1GB board, so a miss re-parses the payload from disk into a genuinely new object -- separate copies accumulating toward the 500-entry cap, not shared references. Releasing is safe: the payload is written to the cache under the request's cache_key before the result is built, the callback is handed the object directly, and consumers read it back from the cache afterwards (the plugins' callbacks use it only in passing, to log a count, before reading the cache). Nothing is lost -- it moves from RAM to the disk cache that was already holding it. Requests submitted without a callback keep their payload, since polling get_result() is then the only way to collect it. That keeps the existing contract, and the existing tests covering it, intact. Not addressed here: max_workers=3 allows three concurrent fetches, so three large parses can peak at once, and there is no in-flight dedupe by cache_key -- a second submit for a key already being fetched starts a second fetch. Both bound the transient peak rather than what stays resident, and both are behaviour changes worth their own review. Co-Authored-By: Claude Opus 5 * fix(memory): file the cache-hit result before running its callback Restores the original ordering. Releasing the payload after the callback meant filing the result after it too, so a callback that queried get_result() or is_request_complete() for its own request would not have found it -- a behaviour change unrelated to the memory fix. The dict holds a reference to the same object, so releasing after filing still clears the payload. Co-Authored-By: Claude Opus 5 * test: wait for the payload release, not just the filing The callback test waited on is_request_complete(), which goes true as soon as the worker files the result in completed_requests. The worker then runs the cleanup pass, then the callback, then releases the payload. Both of the test's assertions therefore raced the worker: `seen` is populated by the callback, and `data is None` only after the release that follows it. It passes today because a one-line callback usually finishes inside the 20ms poll interval. Confirmed by making the callback sleep 0.4s: _wait() returns with seen == {} and the payload still resident. _wait_for_release() polls for the released payload instead. Release happens strictly after the callback returns, so a released payload also means the callback has finished and one wait covers both assertions. Verified against the same 0.4s callback. _wait() stays for the other three fetch-path tests, which assert only what is already true when the result is filed -- the success flag, the error, and the cache write that happened during the fetch itself. Its docstring now says so, so the next reader picks the right one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 --- src/background_data_service.py | 42 +++++- test/test_background_payload_release.py | 187 ++++++++++++++++++++++++ 2 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 test/test_background_payload_release.py diff --git a/src/background_data_service.py b/src/background_data_service.py index 355c93c4..99e101a4 100644 --- a/src/background_data_service.py +++ b/src/background_data_service.py @@ -57,7 +57,15 @@ class FetchRequest: @dataclass class FetchResult: - """Result of a background fetch operation.""" + """Result of a background fetch operation. + + ``data`` survives on the stored result only for requests submitted without + a ``callback``, where polling ``get_result()`` is the sole way to collect + it. When a callback was given, the payload has already been delivered and + the service releases it -- see :meth:`BackgroundDataService._release_payload`. + Either way the data remains in the cache under the request's ``cache_key``, + which is where consumers read it from. + """ request_id: str success: bool data: Optional[Any] = None @@ -191,14 +199,19 @@ class BackgroundDataService: cached=True, fetch_time=0.0 ) + # Filed before the callback runs, as it always was: a callback + # that queries get_result()/is_request_complete() for its own + # request must still find it. Releasing afterwards mutates the + # same object the dict holds. self.completed_requests[request_id] = result - + if callback: try: callback(result) except Exception as e: logger.error(f"Error in callback for request {request_id}: {e}") - + self._release_payload(result) + logger.debug(f"Cache hit for {sport} {year} data") return request_id @@ -333,8 +346,29 @@ class BackgroundDataService: request.callback(result) except Exception as e: logger.error(f"Error in callback for request {request.id}: {e}") - + # Delivered. Drop both references -- they point at the same + # object, so one survivor keeps the whole payload resident. + self._release_payload(result) + request.result = None + return result + + @staticmethod + def _release_payload(result: FetchResult) -> None: + """Drop a delivered payload, keeping the result's status and timings. + + Only called once a callback has been handed the data. Consumers read + fetched data back from the cache under ``cache_key``; the copy carried + here was pinning a parsed season schedule -- 946 games for NCAA + football, roughly a tenth of total RAM on a 1GB Pi -- in memory until + the hourly sweep. + + The cache-hit path matters most: it runs once per update interval per + sport, mints a fresh request_id each time, and a memory-tier miss + re-parses the payload from disk. Those were genuinely separate copies + accumulating toward the 500-entry cap, not shared references. + """ + result.data = None def _make_request_with_retry(self, request: FetchRequest) -> requests.Response: """ diff --git a/test/test_background_payload_release.py b/test/test_background_payload_release.py new file mode 100644 index 00000000..3f6a72c1 --- /dev/null +++ b/test/test_background_payload_release.py @@ -0,0 +1,187 @@ +"""A delivered fetch payload must not stay resident on the stored result. + +BackgroundDataService kept the fetched body on the FetchResult it filed in +`completed_requests`, which is swept only hourly and capped at 500 entries by +count. For status records that is free; for a season schedule it is not. NCAA +football's 2026 schedule is 946 games, and on a 1GB Pi 3B+ the parsed payload +measured ~90MB -- a tenth of the board's memory, pinned for an hour after the +consumer had already been handed it. + +The cache-hit path was the worse of the two. It runs once per update interval +per sport, mints a fresh request_id each time, and hands back whatever the +cache returns -- so a memory-tier miss (the tier is capped at 150 entries) +re-parses the payload from disk into a genuinely new object. Those accumulate +as separate copies rather than shared references, which is the staircase seen +in the field: RSS stepping up ~90MB per sport as seasons loaded and never +coming back down. + +Releasing is safe because the payload is written to the cache under the +request's cache_key before the result is built, and that is where consumers +read it from -- the callback is handed the object directly and the plugins use +it only in passing before reading the cache back. + +Requests submitted *without* a callback keep their payload: polling +get_result() is then the only way to collect it, so releasing would break that +contract. +""" + +import time +import pytest +from unittest.mock import MagicMock, Mock, patch + +from src.background_data_service import BackgroundDataService + + +PAYLOAD = {"events": [{"id": f"g{i}"} for i in range(50)]} + + +@pytest.fixture +def cache(): + m = MagicMock() + m.get.return_value = None + m.set.return_value = None + return m + + +@pytest.fixture +def service(cache): + svc = BackgroundDataService(cache, max_workers=2, request_timeout=5) + yield svc + svc.shutdown(wait=False) + + +def _wait(service, req_id, timeout=5): + """Wait for the result to be FILED. + + Enough for anything that is true by the time the worker stores the result: + its success flag, its error, the cache write that happened during the + fetch. + """ + deadline = time.time() + timeout + while not service.is_request_complete(req_id) and time.time() < deadline: + time.sleep(0.02) + + +def _wait_for_release(service, req_id, timeout=5): + """Wait for the payload to be RELEASED, which is strictly later. + + The worker files the result, then runs the callback, then releases. So + is_request_complete() goes true while the callback still has not run -- + waiting on it alone leaves a window in which `seen` is empty and the + payload is still resident, and the assertions race the worker. It passes + in practice only because a one-line callback usually beats the 20ms poll. + + Release happens after the callback returns, so a released payload also + means the callback has finished: one wait covers both. + """ + deadline = time.time() + timeout + while time.time() < deadline: + result = service.get_result(req_id) + if result is not None and result.data is None: + return + time.sleep(0.02) + raise AssertionError( + f"payload for {req_id} was never released (callback may not have run)") + + +def _resp(): + r = Mock() + r.json.return_value = PAYLOAD + r.raise_for_status.return_value = None + return r + + +class TestFetchPath: + def test_callback_receives_the_payload_then_it_is_released(self, service, cache): + seen = {} + + def callback(result): + # The consumer's one look at the data happens here. + seen['events'] = len(result.data['events']) + + with patch.object(service.session, "get", return_value=_resp()): + req_id = service.submit_fetch_request( + sport="ncaa_fb", year=2026, url="https://example.com/s", + cache_key="ncaa_fb_2026", callback=callback, max_retries=0, + ) + _wait_for_release(service, req_id) + + assert seen['events'] == 50, "callback must still be handed the payload" + + stored = service.get_result(req_id) + assert stored is not None + assert stored.success is True + assert stored.data is None, "payload must not stay on the stored result" + + def test_nothing_is_lost_the_cache_holds_it(self, service, cache): + with patch.object(service.session, "get", return_value=_resp()): + req_id = service.submit_fetch_request( + sport="ncaa_fb", year=2026, url="https://example.com/s", + cache_key="ncaa_fb_2026", callback=lambda r: None, max_retries=0, + ) + _wait(service, req_id) + + cache.set.assert_called_once() + key, written = cache.set.call_args[0][:2] + assert key == "ncaa_fb_2026" + assert written == PAYLOAD, "the payload must be persisted before release" + + def test_without_a_callback_the_payload_is_kept(self, service, cache): + # Polling get_result() is then the only delivery mechanism. + with patch.object(service.session, "get", return_value=_resp()): + req_id = service.submit_fetch_request( + sport="nfl", year=2026, url="https://example.com/s", + cache_key="nfl_2026", max_retries=0, + ) + _wait(service, req_id) + + assert service.get_result(req_id).data == PAYLOAD + + def test_a_failed_fetch_still_records_its_error(self, service, cache): + with patch.object(service.session, "get", side_effect=Exception("boom")): + req_id = service.submit_fetch_request( + sport="nfl", year=2026, url="https://example.com/s", + cache_key="nfl_2026", callback=lambda r: None, max_retries=0, + ) + _wait(service, req_id) + + stored = service.get_result(req_id) + assert stored.success is False + assert stored.error is not None + + +class TestCacheHitPath: + def test_cache_hit_releases_after_the_callback(self, service, cache): + cache.get.return_value = PAYLOAD + seen = {} + + req_id = service.submit_fetch_request( + sport="ncaa_fb", year=2026, url="https://example.com/s", + cache_key="ncaa_fb_2026", + callback=lambda r: seen.update(events=len(r.data['events'])), + ) + + assert seen['events'] == 50 + assert service.get_result(req_id).data is None + + def test_repeated_cache_hits_do_not_accumulate_payloads(self, service, cache): + # The staircase: one entry per update interval per sport, each one + # potentially a freshly parsed copy after a memory-tier miss. + cache.get.return_value = PAYLOAD + + for _ in range(25): + service.submit_fetch_request( + sport="ncaa_fb", year=2026, url="https://example.com/s", + cache_key="ncaa_fb_2026", callback=lambda r: None, + ) + + retained = [r for r in service.completed_requests.values() if r.data is not None] + assert retained == [], f"{len(retained)} payloads still resident" + + def test_cache_hit_without_a_callback_is_unchanged(self, service, cache): + cache.get.return_value = PAYLOAD + req_id = service.submit_fetch_request( + sport="nfl", year=2026, url="https://example.com/s", + cache_key="nfl_2026", + ) + assert service.get_result(req_id).data == PAYLOAD From f90638a9eca146c93bd3236e23ae1e3fd46e4d49 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:40:28 -0400 Subject: [PATCH 44/53] feat(web): show available memory in Tools diagnostics (#500) * feat(web): show available memory in Tools diagnostics System Diagnostics reported memory as used-percent plus used/total GB. Neither distinguishes a healthy board from one about to fail, because page cache counts as used and is reclaimable on demand -- a Pi can read 70% used and be fine, or read the same and be minutes from trouble. MemAvailable is the kernel's own estimate of what a new allocation can actually obtain, and it is the number that tracked the failure on a 1GB Pi 3B+: healthy running sat above 500MB, and the crash came at 73MB. By that point fork() was failing, so sshd could not spawn a session and systemd could not respawn the display, while the kernel carried on answering pings at 0% loss. Used-percent gave no warning at any point on the way there; available memory fell steadily for hours. /api/v3/system/status now returns memory_available_mb from psutil.virtual_memory().available, and Tools renders it as its own tile, coloured against the thresholds that failure implies: red under 150MB, amber under 300MB, green above. The existing memory tile is left alone -- used/total is still what you want when sizing a workload; this answers the different question of how much room is left right now. Co-Authored-By: Claude Opus 5 * fix(web): round available memory once, so the tile agrees with itself The colour was classified from the raw value while the label was rounded, and the API sends one decimal place. At the boundaries the two disagreed: 149.6 rendered as "150 MB" in red, and 299.6 as "300 MB" in amber -- each contradicting the threshold its own colour claims to apply ("red under 150MB"). A reader checking the tile against the documented thresholds would conclude the readout was broken. Rounding once and using that number for both restores agreement. It moves those two boundary cases up a band, which does not matter: the thresholds come from a measured failure at 73MB, so which side of the line a spare 0.4MB falls on carries no information. The tile agreeing with itself does. Null handling and the thresholds themselves are unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 --- test/test_system_status_available_memory.py | 92 +++++++++++++++++++ web_interface/blueprints/api_v3.py | 5 + .../templates/v3/partials/tools.html | 23 +++++ 3 files changed, 120 insertions(+) create mode 100644 test/test_system_status_available_memory.py diff --git a/test/test_system_status_available_memory.py b/test/test_system_status_available_memory.py new file mode 100644 index 00000000..c08bde22 --- /dev/null +++ b/test/test_system_status_available_memory.py @@ -0,0 +1,92 @@ +"""/api/v3/system/status must report MemAvailable, not just used/total. + +"Memory used %" cannot tell a healthy board from one about to fail. Page cache +counts as used and is reclaimable on demand, so a Pi can read 70% used and be +perfectly fine, or read the same and be minutes from trouble. MemAvailable is +the kernel's own estimate of what a new allocation can actually obtain, and it +is the number that tracked the failure on a 1GB Pi 3B+: healthy running sat at +500MB+, the crash happened at 73MB, and by then fork() was failing -- sshd +could not spawn a session and systemd could not respawn the display, while the +kernel carried on answering pings. + +psutil.virtual_memory().available is MemAvailable on Linux. total - used is not +a substitute: they diverge exactly when unreclaimable memory (shmem, tmpfs) is +in play, which is when the distinction matters. +""" + +import json +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +MB = 1024 * 1024 + + +@pytest.fixture +def client(): + pytest.importorskip("psutil") + app = Flask(__name__) + app.config["TESTING"] = True + from web_interface.blueprints.api_v3 import api_v3 + for attr in ("config_manager", "plugin_manager", "cache_manager"): + setattr(api_v3, attr, MagicMock()) + if "api_v3" not in app.blueprints: + app.register_blueprint(api_v3, url_prefix="/api/v3") + return app.test_client() + + +def _memory(total_mb, used_mb, available_mb): + m = MagicMock() + m.total = total_mb * MB + m.used = used_mb * MB + m.available = available_mb * MB + m.percent = round(used_mb / total_mb * 100, 1) + return m + + +def _get_status(client, memory): + # The endpoint caches for 10s; bypass so each case is measured fresh. + with patch("web_interface.cache.get_cached", return_value=None), \ + patch("psutil.virtual_memory", return_value=memory), \ + patch("psutil.cpu_percent", return_value=5.0), \ + patch("psutil.boot_time", return_value=0.0): + resp = client.get("/api/v3/system/status") + assert resp.status_code == 200, resp.data + return json.loads(resp.data)["data"] + + +def test_available_memory_is_reported(client): + data = _get_status(client, _memory(total_mb=905, used_mb=620, available_mb=284)) + assert "memory_available_mb" in data + assert data["memory_available_mb"] == pytest.approx(284, abs=0.5) + + +def test_available_is_not_total_minus_used(client): + # The case the readout exists for: 600MB is "not used", but only 300MB can + # actually be allocated. Reporting used% alone would call this healthy. + data = _get_status(client, _memory(total_mb=1000, used_mb=400, available_mb=300)) + + derived = data["memory_total_mb"] - data["memory_used_mb"] + assert derived == pytest.approx(600, abs=1) + assert data["memory_available_mb"] == pytest.approx(300, abs=0.5) + assert data["memory_available_mb"] != pytest.approx(derived, abs=1), \ + "available must come from MemAvailable, not be derived from used" + + +def test_existing_memory_fields_are_unchanged(client): + data = _get_status(client, _memory(total_mb=905, used_mb=620, available_mb=284)) + assert data["memory_total_mb"] == pytest.approx(905, abs=0.5) + assert data["memory_used_mb"] == pytest.approx(620, abs=0.5) + assert "memory_used_percent" in data + + +def test_a_nearly_exhausted_board_reports_a_small_number(client): + # 73MB available is what the board actually read when it stopped being able + # to fork. The readout has to surface that rather than round it away. + data = _get_status(client, _memory(total_mb=905, used_mb=800, available_mb=73)) + assert data["memory_available_mb"] == pytest.approx(73, abs=0.5) diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index a5a2979f..35dfd885 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -1563,6 +1563,11 @@ def get_system_status(): 'memory_used_percent': round(memory_percent, 1), 'memory_total_mb': round(memory.total / (1024 * 1024), 1), 'memory_used_mb': round(memory.used / (1024 * 1024), 1), + # MemAvailable, not total-minus-used: it accounts for reclaimable + # page cache, so it is what actually predicts memory trouble. A + # board can read 70% "used" and be fine, or read the same and be + # about to fail fork(), and only this number tells them apart. + 'memory_available_mb': round(memory.available / (1024 * 1024), 1), 'cpu_temp': round(cpu_temp, 1) if cpu_temp is not None else None, 'disk_used_percent': round(disk_percent, 1), 'disk_total_gb': round(disk.total / (1024 * 1024 * 1024), 1), diff --git a/web_interface/templates/v3/partials/tools.html b/web_interface/templates/v3/partials/tools.html index f4f7cfdd..1d0fb104 100644 --- a/web_interface/templates/v3/partials/tools.html +++ b/web_interface/templates/v3/partials/tools.html @@ -687,12 +687,35 @@ const mUsedGb = d.memory_used_mb != null ? (d.memory_used_mb / 1024).toFixed(1) : null; const mTotGb = d.memory_total_mb != null ? (d.memory_total_mb / 1024).toFixed(1) : null; const temp = d.cpu_temp != null ? d.cpu_temp + '°C' : 'N/A'; + // Available memory is the number that predicts trouble. When it + // runs out the board does not fail cleanly: fork() starts + // returning ENOMEM, so sshd cannot spawn a session and systemd + // cannot respawn the display, while the kernel keeps answering + // pings. Thresholds are drawn from that failure -- it was + // measured at 73MB free, and healthy running sits well above. + // Round ONCE, then colour and label off the same number. + // The API sends one decimal place, so classifying the raw + // value and displaying the rounded one disagreed at the + // boundaries: 149.6 rendered as "150 MB" in red, and 299.6 as + // "300 MB" in amber, both contradicting the threshold the + // colour claims to apply. Which side of the line a spare + // 0.4MB falls on does not matter; the tile agreeing with + // itself does. + const availMb = d.memory_available_mb == null + ? null : Math.round(d.memory_available_mb); + const availColor = availMb == null ? 'text-gray-400' + : availMb < 150 ? 'text-red-600' + : availMb < 300 ? 'text-amber-500' + : 'text-green-600'; panel.innerHTML = diagTile('fa-microchip', 'text-blue-600', 'CPU Usage', (d.cpu_percent != null ? d.cpu_percent : '--') + '%', null) + diagTile('fa-memory', 'text-green-600', 'Memory', (d.memory_used_percent != null ? d.memory_used_percent : '--') + '%', (mUsedGb && mTotGb) ? `${mUsedGb} / ${mTotGb} GB` : null) + + diagTile('fa-memory', availColor, 'Available Memory', + availMb != null ? `${availMb} MB` : '--', + mTotGb ? `of ${mTotGb} GB total` : null) + diagTile('fa-thermometer-half', 'text-red-600', 'CPU Temp', temp, null) + diagTile('fa-hdd', 'text-indigo-600', 'Disk', (d.disk_used_percent != null ? d.disk_used_percent : '--') + '%', From 39e7f8cbe0043cbdf8346ea48d722e8586b247f0 Mon Sep 17 00:00:00 2001 From: Ron Pierce Date: Tue, 25 Aug 2026 08:45:17 -0500 Subject: [PATCH 45/53] fix(plugins): cap the per-plugin state transition history (#501) * fix(plugins): cap the per-plugin state transition history PluginStateManager recorded every state transition in a per-plugin list and never trimmed it. The only code that removed entries was clear_state(), called solely from PluginManager.unload_plugin(), so a plugin that stays loaded -- normal operation -- never released one. The list is written on the hot scheduling path. Every update cycle appends twice: _reserve_for_update() sets RUNNING and _finish() sets ENABLED back again. At the default 60s update interval that is 2,880 entries per plugin per day, and nothing reads them -- get_state_info() only takes their len(). Pure dead weight. Measured against the unpatched class, ten plugins on a 60s interval: sim uptime history entries heap growth 1 day 28,810 7.7 MB 7 days 201,610 53.9 MB 30 days 864,010 230.9 MB (still climbing) With the cap it is flat at 2,000 entries / 0.5 MB from day one. On a 1 GB board 231 MB of garbage is fatal on its own, and the failure is not a clean OOM: once MemAvailable falls far enough fork() starts returning ENOMEM, so sshd accepts connections and closes them before its banner while the kernel still answers pings. The board looks like a hardware fault and needs a power cycle. Same family as the ceilings added in #464. Retain the most recent 200 transitions per plugin in a deque and let the rest age out. state_history_count is surfaced through the web API, so the lifetime total is tracked separately rather than plateauing at the cap. get_state_history() now returns a copy under the lock; it was handing out the manager's own list, which a caller could mutate. Co-Authored-By: Claude Opus 5 * fix(plugins): copy history entries out, lock clear_state Review follow-ups on the transition history. get_state_history() copied only the outer list, so a caller holding a returned transition could rewrite the manager's record of what happened -- which contradicted the defensive-copy guarantee in its own docstring. Copy each entry too. Every value in a transition is immutable, so a shallow copy per entry is enough. test_get_state_history_entries_are_copies pins it; without the change it fails with 'tampered' == 'enabled'. clear_state() mutated five shared dicts without holding _lock, while every other mutator takes it. A concurrent set_state() could interleave and leave a plugin with history but no state. Drop the five as one unit. This does not close the wider unload-vs-worker race, which lives in PluginManager.unload_plugin() and predates this change: an update worker still in flight can call set_state() after clear_state() returns and recreate the entry. Serialising that needs the per-plugin lock held across worker join in unload_plugin(), which is a separate change. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- src/plugin_system/plugin_state.py | 87 ++++++++++---- test/test_plugin_state_history_cap.py | 166 ++++++++++++++++++++++++++ 2 files changed, 232 insertions(+), 21 deletions(-) create mode 100644 test/test_plugin_state_history_cap.py diff --git a/src/plugin_system/plugin_state.py b/src/plugin_system/plugin_state.py index 269bb423..8b52823e 100644 --- a/src/plugin_system/plugin_state.py +++ b/src/plugin_system/plugin_state.py @@ -6,14 +6,24 @@ with state transitions and queries. """ import threading +from collections import deque from enum import Enum -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, Deque, List from datetime import datetime import logging from src.logging_config import get_logger +# Transitions retained per plugin. The history is diagnostic only -- nothing +# reads the entries themselves, just their count -- but it is appended to on the +# hot scheduling path: every update cycle records RUNNING on reserve and ENABLED +# on finish. Unbounded, that is 2,880 entries per plugin per day at the default +# 60s interval, which on a 1 GB Pi exhausts memory in weeks. Keep the recent +# tail for debugging and let the rest age out. +MAX_STATE_HISTORY_PER_PLUGIN = 200 + + class PluginState(Enum): """Plugin state enumeration.""" UNLOADED = "unloaded" # Plugin not loaded @@ -37,11 +47,34 @@ class PluginStateManager: self.logger = logger or get_logger(__name__) self._lock = threading.RLock() self._states: Dict[str, PluginState] = {} - self._state_history: Dict[str, list] = {} + self._state_history: Dict[str, Deque[Dict[str, Any]]] = {} + # Lifetime transition totals, kept separately so the count reported by + # get_state_info() stays truthful once the history above starts rolling. + self._state_transition_counts: Dict[str, int] = {} self._error_info: Dict[str, Dict[str, Any]] = {} self._last_update: Dict[str, datetime] = {} self._last_display: Dict[str, datetime] = {} + def _record_transition( + self, + plugin_id: str, + transition: Dict[str, Any] + ) -> None: + """Append a transition to the plugin's bounded history. + + Callers must already hold ``_lock``. The deque discards its oldest + entry once it is full, so the history cannot grow without bound; the + lifetime total is tracked separately for get_state_info(). + """ + history = self._state_history.get(plugin_id) + if history is None: + history = deque(maxlen=MAX_STATE_HISTORY_PER_PLUGIN) + self._state_history[plugin_id] = history + history.append(transition) + self._state_transition_counts[plugin_id] = ( + self._state_transition_counts.get(plugin_id, 0) + 1 + ) + def set_state( self, plugin_id: str, @@ -60,16 +93,13 @@ class PluginStateManager: old_state = self._states.get(plugin_id, PluginState.UNLOADED) self._states[plugin_id] = state - if plugin_id not in self._state_history: - self._state_history[plugin_id] = [] - transition = { 'timestamp': datetime.now(), 'from': old_state.value, 'to': state.value, 'error': str(error) if error else None } - self._state_history[plugin_id].append(transition) + self._record_transition(plugin_id, transition) # Store error info if transitioning to ERROR state if state == PluginState.ERROR and error: @@ -126,17 +156,27 @@ class PluginStateManager: state = self.get_state(plugin_id) return state == PluginState.ENABLED - def get_state_history(self, plugin_id: str) -> list: + def get_state_history(self, plugin_id: str) -> List[Dict[str, Any]]: """ Get state transition history for a plugin. - + + Only the most recent MAX_STATE_HISTORY_PER_PLUGIN transitions are + retained; older ones age out. + Args: plugin_id: Plugin identifier - + Returns: - List of state transitions + List of recent state transitions, oldest first. Both the list and + the transition dicts are copies, so callers cannot mutate the + manager's own history. The values inside a transition are all + immutable, so a shallow copy per entry is enough. """ - return self._state_history.get(plugin_id, []) + with self._lock: + return [ + dict(transition) + for transition in self._state_history.get(plugin_id, ()) + ] def set_error_info(self, plugin_id: str, error_info: Dict[str, Any]) -> None: """ @@ -179,9 +219,7 @@ class PluginStateManager: old_state = self._states.get(plugin_id, PluginState.UNLOADED) self._states[plugin_id] = state - if plugin_id not in self._state_history: - self._state_history[plugin_id] = [] - self._state_history[plugin_id].append({ + self._record_transition(plugin_id, { 'timestamp': datetime.now(), 'from': old_state.value, 'to': state.value, @@ -252,15 +290,22 @@ class PluginStateManager: 'last_update': self.get_last_update(plugin_id), 'last_display': self.get_last_display(plugin_id), 'error_info': self.get_error_info(plugin_id), - 'state_history_count': len(self.get_state_history(plugin_id)) + 'state_history_count': self._state_transition_counts.get(plugin_id, 0) } return info def clear_state(self, plugin_id: str) -> None: - """Clear all state information for a plugin.""" - self._states.pop(plugin_id, None) - self._state_history.pop(plugin_id, None) - self._error_info.pop(plugin_id, None) - self._last_update.pop(plugin_id, None) - self._last_display.pop(plugin_id, None) + """Clear all state information for a plugin. + + Held under ``_lock`` so the five dicts are dropped as one unit: every + other mutator takes the lock, and without it a concurrent set_state() + could interleave and leave a plugin with history but no state. + """ + with self._lock: + self._states.pop(plugin_id, None) + self._state_history.pop(plugin_id, None) + self._state_transition_counts.pop(plugin_id, None) + self._error_info.pop(plugin_id, None) + self._last_update.pop(plugin_id, None) + self._last_display.pop(plugin_id, None) diff --git a/test/test_plugin_state_history_cap.py b/test/test_plugin_state_history_cap.py new file mode 100644 index 00000000..1d5e20d5 --- /dev/null +++ b/test/test_plugin_state_history_cap.py @@ -0,0 +1,166 @@ +"""Plugin state history must not grow without bound. + +`PluginStateManager` recorded every state transition in a per-plugin list and +never trimmed it. The only code that removed entries was `clear_state()`, called +solely from `PluginManager.unload_plugin()`, so a plugin that stays loaded -- +i.e. normal operation -- never released a single entry. + +The list is written on the hot scheduling path. Every update cycle appends +twice: `_reserve_for_update()` sets RUNNING and `_finish()` sets ENABLED back +again. At the default 60-second update interval that is 2,880 entries per +plugin per day, and nothing ever reads the entries -- `get_state_info()` only +takes their `len()`. It is pure dead weight. + +Measured against the unpatched class, ten plugins on a 60s interval retain +864,010 transitions after thirty simulated days, for 231 MB of heap. On a 1 GB +Pi that is fatal on its own, and the failure is not a clean OOM: once +MemAvailable falls far enough, fork() starts returning ENOMEM, so sshd accepts +connections and closes them before its banner while the kernel still answers +pings. The board looks like a hardware fault and needs a power cycle. + +These tests pin the cap, the retention order, and the one piece of behaviour the +cap must not change: `state_history_count` is surfaced through the web API, so +it has to keep reporting the lifetime total rather than plateauing at the cap. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from src.plugin_system.plugin_state import ( # noqa: E402 + MAX_STATE_HISTORY_PER_PLUGIN, + PluginState, + PluginStateManager, +) + + +def _cycle_updates(manager, plugin_id, cycles): + """Drive the real scheduling path: RUNNING on reserve, ENABLED on finish.""" + for _ in range(cycles): + manager.set_state(plugin_id, PluginState.RUNNING) + manager.set_state(plugin_id, PluginState.ENABLED) + + +def test_state_history_is_capped(): + """A day of updates must not retain a day of transitions.""" + manager = PluginStateManager() + manager.set_state("clock", PluginState.ENABLED) + + # One simulated day at the default 60s update interval. + _cycle_updates(manager, "clock", 1440) + + history = manager.get_state_history("clock") + assert len(history) <= MAX_STATE_HISTORY_PER_PLUGIN, ( + f"history grew to {len(history)} entries; it is never trimmed" + ) + + +def test_state_history_keeps_the_most_recent_transitions(): + """Trimming drops the oldest entries, not the newest.""" + manager = PluginStateManager() + manager.set_state("clock", PluginState.ENABLED) + _cycle_updates(manager, "clock", MAX_STATE_HISTORY_PER_PLUGIN) + + history = manager.get_state_history("clock") + + # The scheduling cycle ends on ENABLED, so the newest entry is the + # RUNNING -> ENABLED half of the last cycle. + assert history[-1]["from"] == PluginState.RUNNING.value + assert history[-1]["to"] == PluginState.ENABLED.value + + # And the very first ENABLED transition has aged out. + assert history[0]["from"] != PluginState.UNLOADED.value + + +def test_state_history_count_reports_lifetime_total(): + """The count exposed through the API must not plateau at the cap. + + `get_state_info()['state_history_count']` is surfaced by the web UI. Capping + the retained list must not turn it into "entries we happen to still hold". + """ + manager = PluginStateManager() + manager.set_state("clock", PluginState.ENABLED) + total = 1 + + cycles = MAX_STATE_HISTORY_PER_PLUGIN * 2 + _cycle_updates(manager, "clock", cycles) + total += cycles * 2 + + info = manager.get_state_info("clock") + assert info["state_history_count"] == total + assert len(manager.get_state_history("clock")) <= MAX_STATE_HISTORY_PER_PLUGIN + + +def test_error_transitions_are_capped_too(): + """set_state_with_error() appends to the same list and needs the same cap.""" + manager = PluginStateManager() + manager.set_state("clock", PluginState.ENABLED) + + for _ in range(MAX_STATE_HISTORY_PER_PLUGIN * 2): + manager.set_state_with_error( + "clock", + PluginState.ENABLED, + {"reason": "update timeout"}, + error=RuntimeError("boom"), + ) + + assert len(manager.get_state_history("clock")) <= MAX_STATE_HISTORY_PER_PLUGIN + + +def test_history_is_isolated_per_plugin(): + """The cap is per plugin, not shared across the manager.""" + manager = PluginStateManager() + for plugin_id in ("clock", "weather"): + manager.set_state(plugin_id, PluginState.ENABLED) + _cycle_updates(manager, plugin_id, 50) + + assert len(manager.get_state_history("clock")) == 101 + assert len(manager.get_state_history("weather")) == 101 + + +def test_get_state_history_returns_a_copy(): + """Callers must not be able to mutate the manager's internal history.""" + manager = PluginStateManager() + manager.set_state("clock", PluginState.ENABLED) + + history = manager.get_state_history("clock") + history.clear() + + assert len(manager.get_state_history("clock")) == 1 + + +def test_get_state_history_entries_are_copies(): + """Copying the outer list is not enough -- the entries are handed out too. + + A caller holding a returned transition must not be able to rewrite the + manager's record of what happened. + """ + manager = PluginStateManager() + manager.set_state("clock", PluginState.ENABLED) + + entry = manager.get_state_history("clock")[0] + entry["to"] = "tampered" + entry["error"] = "injected" + + stored = manager.get_state_history("clock")[0] + assert stored["to"] == PluginState.ENABLED.value + assert stored["error"] is None + + +def test_clear_state_drops_history(): + """Unloading a plugin still releases everything it accumulated.""" + manager = PluginStateManager() + manager.set_state("clock", PluginState.ENABLED) + _cycle_updates(manager, "clock", 10) + + manager.clear_state("clock") + + assert manager.get_state_history("clock") == [] + assert manager.get_state_info("clock")["state_history_count"] == 0 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) From bdced206dc05d738914ed7ab6a2c2fc564dafe22 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:56:45 -0400 Subject: [PATCH 46/53] fix(plugins): retain state history by age, with the count as a ceiling (#502) * fix(plugins): retain state history by age, with the count as a ceiling Follow-up to the cap in this PR. A flat entry count answers the wrong question: what a reader wants from this history is "the last couple of hours", and how many transitions that is depends entirely on the plugin's update interval. On a real board those span 2s to 3600s, so 200 entries is interval 200 entries covers 2s 3.3 minutes (flights, live) 10s 16.7 minutes (jellyfin) 60s 1.7 hours (default) 300s 8.3 hours (news) 3600s 4.2 days -- the plugin churning hardest, the one worth looking at, keeps the least. So transitions are now trimmed by AGE first (STATE_HISTORY_MAX_AGE_SECONDS, two hours), which makes the retained window comparable whatever the cadence, and the count cap becomes purely a memory ceiling for pollers fast enough to exceed it inside that window. The ceiling rises 200 -> 2000: at ~230 bytes an entry that is ~0.5MB per plugin worst case, and only plugins updating faster than roughly every 4s can reach it. Steady-state memory is unchanged for everything slower, since the age trim binds first. Two details worth stating: - The trim reads time.monotonic(), stored alongside each transition, rather than the datetime already inside it. A DST shift or an NTP step would otherwise make every entry look ancient and flush the history in one go. The human-readable timestamp is untouched and still what get_state_history() returns. - Trimming happens on append, so a plugin that goes quiet keeps its last window until it writes again. That is deliberate: it is bounded either way, and a lazy trim costs nothing on the hot scheduling path. The guarantee is therefore about the SPAN of retained history, not its age against the current clock, and the test asserts it that way. The public shape is unchanged: get_state_history() still returns the same list of transition dicts, and state_history_count is still the lifetime total. test_plugin_state_history_retention.py adds 7 tests. Verified against this branch with only the age trim removed: 4 fail, 3 pass -- the three that survive are testing the count ceiling and the monotonic clock, which this commit does not change. Full suite 3753 passed, 60 skipped. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(plugins): build get_state_info() as one locked snapshot Every field was read under its own lock, so an unload running concurrently could be observed half-done: 'state' read before clear_state() removed it and 'state_history_count' read after, handing PluginManager.get_plugin_info() a plugin that is ENABLED with zero transitions. The whole payload is now built in one critical section. _lock is an RLock, so the helpers called inside it can still take it. The regression test runs a reader against a thread that repeatedly fills and clears the same plugin, and fails on the first torn snapshot. Verified by removing only the lock: fails on 3 of 3 runs, passes on 3 of 3 with it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 (1M context) --- src/plugin_system/plugin_state.py | 86 +++++--- test/test_plugin_state_history_retention.py | 209 ++++++++++++++++++++ 2 files changed, 269 insertions(+), 26 deletions(-) create mode 100644 test/test_plugin_state_history_retention.py diff --git a/src/plugin_system/plugin_state.py b/src/plugin_system/plugin_state.py index 8b52823e..f9b9d7d0 100644 --- a/src/plugin_system/plugin_state.py +++ b/src/plugin_system/plugin_state.py @@ -6,22 +6,38 @@ with state transitions and queries. """ import threading +import time from collections import deque from enum import Enum -from typing import Optional, Dict, Any, Deque, List +from typing import Optional, Dict, Any, Deque, List, Tuple from datetime import datetime import logging from src.logging_config import get_logger -# Transitions retained per plugin. The history is diagnostic only -- nothing -# reads the entries themselves, just their count -- but it is appended to on the -# hot scheduling path: every update cycle records RUNNING on reserve and ENABLED -# on finish. Unbounded, that is 2,880 entries per plugin per day at the default -# 60s interval, which on a 1 GB Pi exhausts memory in weeks. Keep the recent -# tail for debugging and let the rest age out. -MAX_STATE_HISTORY_PER_PLUGIN = 200 +# The history is diagnostic only -- nothing reads the entries themselves, just +# their count -- but it is appended to on the hot scheduling path: every update +# cycle records RUNNING on reserve and ENABLED on finish. Unbounded, that is +# 2,880 entries per plugin per day at the default 60s interval, which on a 1 GB +# Pi exhausts memory in weeks. +# +# Two limits, because a single entry count answers the wrong question. What a +# reader wants is "the last couple of hours", and how many transitions that is +# depends entirely on the plugin's update interval -- which on a real board +# spans 2s to 3600s. A flat 200 entries is 4.2 days for the slowest plugin and +# 3.3 minutes for the fastest, so the plugin churning hardest, the one worth +# looking at, keeps the least history. +# +# So: trim by AGE first, which makes the retained window comparable across +# plugins whatever their cadence... +STATE_HISTORY_MAX_AGE_SECONDS = 2 * 60 * 60 + +# ...and cap by COUNT second, purely as a memory ceiling for the fast pollers +# whose age window would otherwise run to thousands of entries. At ~230 bytes +# an entry this is ~0.5 MB per plugin worst case, and only plugins updating +# faster than roughly every 4s can reach it. +MAX_STATE_HISTORY_PER_PLUGIN = 2000 class PluginState(Enum): @@ -47,7 +63,10 @@ class PluginStateManager: self.logger = logger or get_logger(__name__) self._lock = threading.RLock() self._states: Dict[str, PluginState] = {} - self._state_history: Dict[str, Deque[Dict[str, Any]]] = {} + # (monotonic timestamp, transition). The clock is monotonic so a DST + # shift or an NTP step cannot make entries look old and flush the + # history; the human-readable timestamp lives inside the transition. + self._state_history: Dict[str, Deque[Tuple[float, Dict[str, Any]]]] = {} # Lifetime transition totals, kept separately so the count reported by # get_state_info() stays truthful once the history above starts rolling. self._state_transition_counts: Dict[str, int] = {} @@ -70,7 +89,13 @@ class PluginStateManager: if history is None: history = deque(maxlen=MAX_STATE_HISTORY_PER_PLUGIN) self._state_history[plugin_id] = history - history.append(transition) + now = time.monotonic() + history.append((now, transition)) + # Age out first; the deque's maxlen is the backstop for plugins that + # produce more than the ceiling within the window. + cutoff = now - STATE_HISTORY_MAX_AGE_SECONDS + while history and history[0][0] < cutoff: + history.popleft() self._state_transition_counts[plugin_id] = ( self._state_transition_counts.get(plugin_id, 0) + 1 ) @@ -160,8 +185,10 @@ class PluginStateManager: """ Get state transition history for a plugin. - Only the most recent MAX_STATE_HISTORY_PER_PLUGIN transitions are - retained; older ones age out. + Retention is by age first -- transitions older than + STATE_HISTORY_MAX_AGE_SECONDS are dropped -- and by count second, at + MAX_STATE_HISTORY_PER_PLUGIN, which only binds for plugins updating + fast enough to exceed it inside that window. Args: plugin_id: Plugin identifier @@ -175,7 +202,7 @@ class PluginStateManager: with self._lock: return [ dict(transition) - for transition in self._state_history.get(plugin_id, ()) + for _stamp, transition in self._state_history.get(plugin_id, ()) ] def set_error_info(self, plugin_id: str, error_info: Dict[str, Any]) -> None: @@ -279,19 +306,26 @@ class PluginStateManager: Returns: Dictionary with state information """ - state = self.get_state(plugin_id) - info = { - 'state': state.value, - 'is_loaded': self.is_loaded(plugin_id), - 'is_enabled': self.is_enabled(plugin_id), - 'is_running': self.is_running(plugin_id), - 'is_error': self.is_error(plugin_id), - 'can_execute': self.can_execute(plugin_id), - 'last_update': self.get_last_update(plugin_id), - 'last_display': self.get_last_display(plugin_id), - 'error_info': self.get_error_info(plugin_id), - 'state_history_count': self._state_transition_counts.get(plugin_id, 0) - } + # One snapshot, one critical section. Each field was read under its own + # lock, so an unload running concurrently could be observed half-done: + # 'state' read before clear_state() removed it and + # 'state_history_count' read after, giving a caller a plugin that is + # ENABLED with zero transitions. _lock is an RLock, so the helpers + # below can still take it. + with self._lock: + state = self.get_state(plugin_id) + info = { + 'state': state.value, + 'is_loaded': self.is_loaded(plugin_id), + 'is_enabled': self.is_enabled(plugin_id), + 'is_running': self.is_running(plugin_id), + 'is_error': self.is_error(plugin_id), + 'can_execute': self.can_execute(plugin_id), + 'last_update': self.get_last_update(plugin_id), + 'last_display': self.get_last_display(plugin_id), + 'error_info': self.get_error_info(plugin_id), + 'state_history_count': self._state_transition_counts.get(plugin_id, 0) + } return info def clear_state(self, plugin_id: str) -> None: diff --git a/test/test_plugin_state_history_retention.py b/test/test_plugin_state_history_retention.py new file mode 100644 index 00000000..a7cf1edf --- /dev/null +++ b/test/test_plugin_state_history_retention.py @@ -0,0 +1,209 @@ +"""Retention is bounded by age first and by count second. + +The cap added in the parent change is a flat entry count, and an entry count +answers the wrong question. What a reader wants from this history is "the last +couple of hours"; how many transitions that is depends entirely on the +plugin's update interval, which on a real board spans 2s to 3600s. A flat 200 +entries is 4.2 days of history for the slowest plugin and 3.3 minutes for the +fastest -- so the plugin churning hardest, the one actually worth looking at, +keeps the least. + +Trimming by age makes the retained window comparable whatever the cadence, and +the count then serves only as a memory ceiling for pollers fast enough to +produce thousands of transitions inside that window. +""" + +import time +import pytest + +from src.plugin_system.plugin_state import ( + PluginState, + PluginStateManager, + MAX_STATE_HISTORY_PER_PLUGIN, + STATE_HISTORY_MAX_AGE_SECONDS, +) + + +class FakeClock: + """A monotonic clock the test drives, so no test has to sleep.""" + + def __init__(self): + self.t = 1000.0 + + def __call__(self): + return self.t + + def advance(self, seconds): + self.t += seconds + + +@pytest.fixture +def clock(monkeypatch): + c = FakeClock() + monkeypatch.setattr("src.plugin_system.plugin_state.time.monotonic", c) + return c + + +def _cycle(manager, plugin_id, clock, interval, cycles): + """One update cycle: RUNNING on reserve, ENABLED on finish.""" + for _ in range(cycles): + manager.set_state(plugin_id, PluginState.RUNNING) + manager.set_state(plugin_id, PluginState.ENABLED) + clock.advance(interval) + + +def test_transitions_older_than_the_window_are_dropped(clock): + m = PluginStateManager() + _cycle(m, "clock", clock, interval=60, cycles=10) + assert len(m.get_state_history("clock")) == 20 + + # Nothing happens for longer than the window, then one more cycle. + clock.advance(STATE_HISTORY_MAX_AGE_SECONDS + 1) + _cycle(m, "clock", clock, interval=60, cycles=1) + + assert len(m.get_state_history("clock")) == 2, ( + "only the transitions inside the window should survive") + + +def test_every_plugin_keeps_the_same_WINDOW_not_the_same_COUNT(clock): + """The point of the age policy, stated as the property that distinguishes it. + + Run both plugins for three times the retention window. Under a flat count + cap the slow one would still be holding transitions from hours before the + window, because it never produces enough entries to evict them. Under the + age policy each plugin retains its own last two hours and no more -- + different entry counts, same span of time. + """ + window = STATE_HISTORY_MAX_AGE_SECONDS + m = PluginStateManager() + + _cycle(m, "slow", clock, interval=60, cycles=(3 * window) // 60) + slow = len(m.get_state_history("slow")) + + # Assert the property directly rather than a derived count. The guarantee + # is about the SPAN of retained history, not its age against the current + # clock: trimming happens on append, so a plugin that has gone quiet keeps + # its last window until it writes again. That is intentional -- it is + # bounded either way, and a lazy trim costs nothing on the hot path. + stamps = [stamp for stamp, _ in m._state_history["slow"]] + assert stamps[-1] - stamps[0] <= window, ( + f"retained history spans {stamps[-1] - stamps[0]:.0f}s, " + f"window is {window}s") + assert slow < 2 * ((3 * window) // 60), ( + f"slow plugin kept {slow} entries -- three windows' worth was retained") + + clock.t = 1000.0 + _cycle(m, "fast", clock, interval=2, cycles=(3 * window) // 2) + fast = len(m.get_state_history("fast")) + + # Different counts, and the fast poller keeps more of them -- under a flat + # count cap these would be equal and the fast one would cover minutes. + assert fast > slow, f"fast={fast} slow={slow}" + + +def test_the_count_ceiling_still_bounds_a_fast_poller(clock): + """Age alone would let a 2s plugin hold 7,200 entries.""" + m = PluginStateManager() + _cycle(m, "flights", clock, interval=2, cycles=STATE_HISTORY_MAX_AGE_SECONDS) + assert len(m.get_state_history("flights")) <= MAX_STATE_HISTORY_PER_PLUGIN + + +def test_a_burst_inside_the_window_is_capped_not_kept(clock): + """Transitions with no time between them still cannot grow without bound.""" + m = PluginStateManager() + for _ in range(MAX_STATE_HISTORY_PER_PLUGIN * 3): + m.set_state("flapping", PluginState.RUNNING) # clock never advances + assert len(m.get_state_history("flapping")) <= MAX_STATE_HISTORY_PER_PLUGIN + + +def test_ageing_out_does_not_disturb_the_lifetime_count(clock): + m = PluginStateManager() + _cycle(m, "clock", clock, interval=60, cycles=10) + clock.advance(STATE_HISTORY_MAX_AGE_SECONDS + 1) + _cycle(m, "clock", clock, interval=60, cycles=1) + + assert len(m.get_state_history("clock")) == 2 + assert m.get_state_info("clock")["state_history_count"] == 22, ( + "the lifetime total must survive trimming, it is the flap signal") + + +def test_the_surviving_entries_are_the_recent_ones(clock): + m = PluginStateManager() + _cycle(m, "clock", clock, interval=60, cycles=5) + clock.advance(STATE_HISTORY_MAX_AGE_SECONDS + 1) + m.set_state("clock", PluginState.ERROR) + + history = m.get_state_history("clock") + assert [h["to"] for h in history] == ["error"] + + +def test_a_monotonic_clock_is_used_not_the_wall_clock(clock): + """A DST shift or NTP step must not flush the history. + + The trim reads time.monotonic(); the human-readable datetime inside each + transition is for display only. + """ + m = PluginStateManager() + _cycle(m, "clock", clock, interval=60, cycles=3) + before = len(m.get_state_history("clock")) + + import datetime as real_datetime + + class ShiftedDatetime(real_datetime.datetime): + @classmethod + def now(cls, tz=None): + return real_datetime.datetime(1999, 1, 1) # clock jumps backwards + + import src.plugin_system.plugin_state as ps + original = ps.datetime + ps.datetime = ShiftedDatetime + try: + m.set_state("clock", PluginState.ENABLED) + finally: + ps.datetime = original + + assert len(m.get_state_history("clock")) == before + 1, ( + "a wall-clock jump must not trim anything") + + +def test_get_state_info_is_a_consistent_snapshot(): + """An unload running concurrently must not be observed half-done. + + Each field used to be read under its own lock, so clear_state() could + interleave: 'state' read before the removal, 'state_history_count' after, + handing a caller a plugin that is ENABLED with zero transitions. The whole + payload is now built in one critical section. + """ + import threading + + m = PluginStateManager() + for _ in range(50): + m.set_state("clock", PluginState.RUNNING) + m.set_state("clock", PluginState.ENABLED) + + inconsistent = [] + stop = threading.Event() + + def reader(): + while not stop.is_set(): + info = m.get_state_info("clock") + # Either fully present or fully cleared -- never a live state with + # a wiped count. + if info["state"] != PluginState.UNLOADED.value and \ + info["state_history_count"] == 0: + inconsistent.append(info) + return + + def clearer(): + for _ in range(200): + for _ in range(20): + m.set_state("clock", PluginState.ENABLED) + m.clear_state("clock") + + t = threading.Thread(target=reader, daemon=True) + t.start() + clearer() + stop.set() + t.join(timeout=5) + + assert not inconsistent, f"observed a torn snapshot: {inconsistent[:1]}" From 5e5979973ed401c545e2a69250081f0f5498f4ee Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:13:23 -0400 Subject: [PATCH 47/53] Update README.md (#504) Signed-off-by: Chuck <33324927+ChuckBuilds@users.noreply.github.com> --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 681176ef..2d61b614 100644 --- a/README.md +++ b/README.md @@ -161,9 +161,11 @@ The system supports live, recent, and upcoming game information for multiple spo ### LED Matrix Panels (2x in a horizontal chain is recommended) - [Adafruit 64×32](https://www.adafruit.com/product/2278) – designed for 128×32 but works with dynamic scaling on many displays (pixel pitch is user preference) +**Warning: Lately the Waveshare Panels have had different variations - only some are compatible with this project. I hope to identify what is different to fix it but so far there is a decent chance you get a mis-matched set of panels if you don't buy them all at once! ** - [Waveshare 64×32](https://amzn.to/3Kw55jK) - Does not require E addressable pad -- [Waveshare 96×48](https://amzn.to/4bydNcv) – higher resolution, requires soldering the **E addressable pad** on the [Adafruit RGB Bonnet](https://www.adafruit.com/product/3211) to “8” **OR** toggling the DIP switch on the Adafruit Triple LED Matrix Bonnet *(no soldering required!)* - > Amazon Affiliate Link – ChuckBuilds receives a small commission on purchases +- [Waveshare 96×48](https://amzn.to/4bydNcv) – higher resolution, requires soldering the **E addressable pad** on the [Adafruit RGB Bonnet](https://www.adafruit.com/product/3211) to “8” **OR** toggling the DIP switch on the Adafruit Triple LED Matrix Bonnet *(no soldering required!)* +- There are some Panels on Aliexpress that have worked fine for me, shop around! I think Adafruit is probably the "safest" but they do have some limitation on resolution and layout. + > Amazon Affiliate Links – ChuckBuilds receives a small commission on purchases ### Power Supply - [5V 4A DC Power Supply](https://www.adafruit.com/product/658) (good for 2 -3 displays, depending on brightness and pixel density, you'll need higher amperage for more) From af96bd5cb063d4f92b7f6dd9fc1376753cf2d744 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:03:43 -0400 Subject: [PATCH 48/53] fix(memory): join an in-flight fetch instead of starting a duplicate (#503) * fix(memory): join an in-flight fetch instead of starting a duplicate submit_fetch_request() had no notion of "already fetching this". request_id embeds a millisecond timestamp, so every submit looked new, and active_requests is keyed by that id rather than by what is being fetched. Two submits for the same cache_key therefore started two identical fetches. It is not a rare race. _fetch_data in the sports managers branches: the Live manager fetches only today's games, but Recent and Upcoming both pull the full season schedule under the SAME cache_key. On a cache miss both miss, both submit, and nothing stops the second. On a running 512x64 board: 138 background fetches in 24 hours, arriving in pairs at identical millisecond timestamps, roughly hourly: 2 2026-08-25 11:47:26.612 2 2026-08-25 10:46:28.064 2 2026-08-25 08:01:55.962 Half of them redundant. Each duplicate costs a second download, a second JSON parse -- the expensive part on a Pi -- and a second parsed copy resident at the same time. Schedules on that board run 256KB to 20MB, 106MB across all sports. Because the pairs land in the same millisecond they also occupy two of the three executor slots with identical work, which is what makes two large parses peak simultaneously. A submit for a cache_key already in flight now joins that request: its callback is added to the existing one and the existing request_id is returned, so get_result() works for both. Different keys are untouched, and dedupe applies only while a fetch is in flight -- a submit after completion fetches again, because this is not a second cache layer. Three details: - The in-flight entry is dropped and the callback list snapshotted in the SAME critical section as filing the result. Otherwise a submitter could join a fetch whose callbacks had already run and never be called back. - Cancellation is the other way a request leaves active_requests, so it releases the key too. And the join path looks the request up rather than trusting the id, so an entry stranded any other way cannot wedge a key permanently -- it is dropped and a fresh fetch starts. - One callback raising no longer prevents the others being delivered. Previously there was only ever one. Interaction with #499, whichever merges second: that PR releases the payload after the callback runs. With several callbacks the release must happen after ALL of them, and must not happen at all if a joined submitter passed no callback, since polling get_result() would then be its only delivery path. The callback list built here is the hook for that. test_background_fetch_dedupe.py -- 8 tests, covering the join, callback delivery to both submitters, one callback raising, distinct keys not being coalesced, a post-completion submit fetching again, cancellation releasing the key, a stranded entry not wedging one, and the reported count. Verified non-vacuous by removing only the join branch: 3 fail. The callback tests assert the ids coalesced, without which they would pass trivially on two independent requests. Full suite: 3698 passed, 60 skipped, 1 failure that reproduces identically on unmodified main (test_install_lowmem, environment-dependent: /var/tmp is disk-backed on this machine). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(memory): discard a cancelled fetch instead of letting it commit Review follow-up on the dedupe. Cancelling releases the cache_key, so a replacement fetch for that key can start immediately. But _fetch_data_worker() had no cancellation check: the cancelled worker still wrote its response to the cache, flipped its own status from CANCELLED to COMPLETED, and ran its callbacks. The stale response could therefore land on top of the replacement's fresher data. The worker cannot abort an HTTP call in flight, so the response is discarded on return instead: no cache write, no callbacks, status left CANCELLED. The check sits immediately before the cache write, which is the first side effect. Also fixed, found by the new test rather than by reading: request_id was f"{sport}_{year}_{milliseconds}", which is not unique. Two submits inside the same millisecond produced the SAME id -- the test's two sequential fetches collided on a fast mocked response, and one request silently replaced the other in active_requests and completed_requests. Rare before this PR; load-bearing now, because dedupe hands that id back to every joiner as their handle for get_result(). A per-service counter is appended. Two test problems of my own, both fixed here rather than left to flake: - The cancellation test synchronised with time.sleep(0.4). A slow worker would have made it pass for the wrong reason. It now waits for the request to be filed in completed_requests. - The id-uniqueness test patched session.get, but submits are async: the 50 workers outlived the patch and made real DNS calls to the dummy host. It stubs the executor instead, which is what a submit-time test should exercise. 20 consecutive runs of the dedupe file: 0 failures. Full suite: 3700 passed, 60 skipped, 1 failure that reproduces identically on unmodified main (test_install_lowmem, environment-dependent). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * fix(memory): make cancellation terminal, not advisory Three paths wrote request.status without checking whether the request had already been cancelled, so a cancel could be silently undone and the work it was meant to stop went ahead anyway. - A request cancelled while queued had CANCELLED overwritten with IN_PROGRESS the moment its worker started, defeating the discard check entirely: it downloaded, cached and called back for work the caller had withdrawn. It now skips the fetch outright, which is also the cheapest possible cancel. - The cancelled-check and the cache write were separate critical sections, so a cancel landing between them left the payload in the cache with the callbacks suppressed -- every submitter that joined the fetch waited for a call that never came. The worker now claims the commit in the same critical section that reads the status, and cancel_request refuses once claimed. The write stays outside the lock: it serialises a multi-megabyte payload to the SD card, and holding the service lock across that would stall every submit, status query and cancel behind it. - A cancelled request that then failed was relabelled FAILED, which slipped past the CANCELLED-only callback gate and delivered a spurious error callback. The except path now leaves CANCELLED alone. get_request_status() also reported a cancelled request as FAILED, since it inferred status from result.success; the final status is now recorded on the result. Both early returns assign to `result` so completed_requests files the outcome that was reported rather than the untouched placeholder. Tests cover cancellation before worker start, during the commit, and during an HTTP failure; all three fail against the unfixed code. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW * test(memory): reach the exception path as a cancelled request test_a_failure_after_cancelling_stays_cancelled cancelled the request while its worker was still queued, so once the pre-start branch landed the worker returned there and never reached the exception handler the test is named for. It passed against the unfixed code only because that branch did not exist yet; with it, the test passed for the wrong reason and reverting the except-path guard did not fail it. Cancel while the worker is parked inside the HTTP call instead, and assert the fetch actually started so the test cannot silently degrade into the pre-start case again. Reverting each of the three guards now fails exactly one test. Also read the payload inside the callback rather than off the FetchResult afterwards: #499 releases result.data once delivery is done, so the later read saw the released object and not what the caller was handed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW --------- Co-authored-by: Claude Opus 5 (1M context) --- src/background_data_service.py | 162 +++++++++- test/test_background_fetch_dedupe.py | 450 +++++++++++++++++++++++++++ 2 files changed, 605 insertions(+), 7 deletions(-) create mode 100644 test/test_background_fetch_dedupe.py diff --git a/src/background_data_service.py b/src/background_data_service.py index 99e101a4..b3838322 100644 --- a/src/background_data_service.py +++ b/src/background_data_service.py @@ -14,11 +14,12 @@ Key Features: - Memory-efficient data storage """ +import itertools import time import logging import threading import requests -from typing import Dict, Any, Optional, Callable +from typing import Dict, Any, Optional, Callable, List from dataclasses import dataclass, field from enum import Enum import queue @@ -50,8 +51,19 @@ class FetchRequest: max_retries: int = 3 priority: int = 1 # Higher number = higher priority callback: Optional[Callable] = None + # Callbacks from submitters that JOINED this fetch instead of starting a + # duplicate one. The primary `callback` above belongs to whoever created + # the request; these belong to everyone who asked for the same cache_key + # while it was still in flight. + extra_callbacks: List[Callable] = field(default_factory=list) created_at: float = field(default_factory=time.time) status: FetchStatus = FetchStatus.PENDING + # Set once the worker has decided this response will be cached, while it + # still holds the lock. From that point cancelling is refused: the write + # is already authorised, and abandoning it here would put the payload in + # the cache with the callbacks suppressed -- joiners waiting forever for a + # fetch that did, in fact, succeed. + commit_claimed: bool = False result: Optional[Any] = None error: Optional[str] = None @@ -74,6 +86,10 @@ class FetchResult: fetch_time: float = 0.0 retry_count: int = 0 completed_at: float = field(default_factory=time.time) # Timestamp when request completed + # The request's final status, recorded so a finished request can still be + # reported accurately. Without it a caller can only be told COMPLETED or + # FAILED, which turns "you cancelled this" into "this errored". + final_status: Optional[FetchStatus] = None class BackgroundDataService: """ @@ -98,6 +114,20 @@ class BackgroundDataService: # Thread management self.executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="BackgroundData") + # cache_key -> request_id for fetches currently in flight. Submitting + # the same key twice used to start two identical fetches: request_id + # carries a millisecond timestamp, so every submit looked new, and + # active_requests is keyed by it rather than by what is being fetched. + # On a real board the season-schedule key is requested by both the + # Recent and the Upcoming manager, which miss the cache in the same + # millisecond and each download and parse the same payload. + self._inflight_by_cache_key: Dict[str, str] = {} + # request_id was sport_year_milliseconds, which is not unique: two + # submits inside the same millisecond produced the SAME id, so one + # silently replaced the other in active_requests and completed_requests. + # Rare before, but dedupe hands this id back to every joiner as their + # handle for get_result(), so it has to be unique. A counter is enough. + self._request_seq = itertools.count() self.active_requests: Dict[str, FetchRequest] = {} self.completed_requests: Dict[str, FetchResult] = {} self.request_queue = queue.PriorityQueue() @@ -185,7 +215,9 @@ class BackgroundDataService: if cache_key is None: cache_key = self.get_sport_cache_key(sport) - request_id = f"{sport}_{year}_{int(time.time() * 1000)}" + with self._lock: + request_id = (f"{sport}_{year}_{int(time.time() * 1000)}" + f"_{next(self._request_seq)}") # Check cache first cached_data = self.cache_manager.get(cache_key) @@ -231,7 +263,29 @@ class BackgroundDataService: ) with self._lock: + existing_id = self._inflight_by_cache_key.get(cache_key) + existing = self.active_requests.get(existing_id) if existing_id else None + if existing_id and existing is None: + # Stranded index entry: the request it names is gone. Drop it and + # fetch normally. Looking the request up rather than trusting the + # id is what stops a stale entry wedging a key forever. + del self._inflight_by_cache_key[cache_key] + if existing is not None: + # Someone is already fetching this key. Ride along rather than + # duplicating the download, the parse and the resident copy. + if callback: + existing.extra_callbacks.append(callback) + self.stats['deduplicated_requests'] = ( + self.stats.get('deduplicated_requests', 0) + 1 + ) + logger.info( + "Joined in-flight fetch %s for %s (cache_key=%s) instead of " + "starting a duplicate", existing_id, sport, cache_key + ) + return existing_id + self.active_requests[request_id] = request + self._inflight_by_cache_key[cache_key] = request_id self.stats['total_requests'] += 1 self.stats['cache_misses'] += 1 @@ -256,7 +310,31 @@ class BackgroundDataService: try: with self._lock: - request.status = FetchStatus.IN_PROGRESS + # A request cancelled while it sat in the executor queue must + # stay cancelled. Overwriting the status here undid the cancel + # outright: the worker went on to download, cache and call back + # for work the caller had already withdrawn. + if request.status == FetchStatus.CANCELLED: + cancelled_before_start = True + else: + cancelled_before_start = False + request.status = FetchStatus.IN_PROGRESS + if cancelled_before_start: + logger.info( + "Request %s was cancelled before its worker started; " + "skipping the fetch entirely", request.id + ) + # Assign before returning: the finally block stores `result` + # in completed_requests, so building a fresh one here would + # file the untouched placeholder instead of this outcome. + result = FetchResult( + request_id=request.id, + success=False, + error="cancelled", + fetch_time=time.time() - start_time, + retry_count=request.retry_count + ) + return result logger.info(f"Starting background fetch for {request.sport} {request.year}") @@ -282,6 +360,37 @@ class BackgroundDataService: # Log data validation logger.debug(f"Validated {len(events)} events for {request.sport} {request.year}") + # A cancelled request must not commit anything. Cancelling + # releases the cache_key, so a replacement fetch for the same key + # may already be in flight or finished -- writing this response to + # the cache now would overwrite fresher data with the response + # nobody wanted. The worker has no way to abort the HTTP call, so + # this is where the work gets discarded. + with self._lock: + cancelled = request.status == FetchStatus.CANCELLED + if not cancelled: + # Claim the commit in the same critical section that read + # the status, so a cancel cannot slip in between the check + # and the cache write below. The write itself stays outside + # the lock: it serialises a multi-megabyte payload to the + # SD card, and holding the service lock across that would + # stall every submit, status query and cancel behind it. + request.commit_claimed = True + if cancelled: + logger.info( + "Discarding response for cancelled request %s; %s may " + "already belong to a replacement fetch", + request.id, request.cache_key + ) + result = FetchResult( + request_id=request.id, + success=False, + error="cancelled", + fetch_time=time.time() - start_time, + retry_count=request.retry_count + ) + return result + # Cache the data self.cache_manager.set(request.cache_key, data) @@ -307,7 +416,12 @@ class BackgroundDataService: logger.error(f"Failed to fetch {request.sport} {request.year} data: {error_msg}") with self._lock: - request.status = FetchStatus.FAILED + # Don't relabel a cancelled request. The callback gate in the + # finally block only suppresses CANCELLED, so promoting it to + # FAILED here delivered an error callback for a fetch nobody + # was waiting on any more. + if request.status != FetchStatus.CANCELLED: + request.status = FetchStatus.FAILED request.error = error_msg result = FetchResult( @@ -321,9 +435,26 @@ class BackgroundDataService: finally: # Store result and clean up with self._lock: + result.final_status = request.status self.completed_requests[request.id] = result if request.id in self.active_requests: del self.active_requests[request.id] + # Stop accepting joiners and take the callback list in the same + # critical section. A submitter that arrives after this point + # finds no in-flight entry and either hits the cache (written + # above, before the result was built) or starts a fresh fetch -- + # what it must never do is join a fetch whose callbacks have + # already run and then never be called. + if self._inflight_by_cache_key.get(request.cache_key) == request.id: + del self._inflight_by_cache_key[request.cache_key] + # A cancelled request delivers nothing: its joiners were told + # about a fetch that has been abandoned, and a replacement will + # call them via its own request. + if request.status == FetchStatus.CANCELLED: + callbacks = [] + else: + callbacks = ([request.callback] if request.callback else []) + callbacks.extend(request.extra_callbacks) # Update statistics if result.success: @@ -340,10 +471,11 @@ class BackgroundDataService: # Periodic cleanup after storing result self._cleanup_completed_requests() - # Call callback if provided - if request.callback: + # Call every callback: the original submitter's and any that joined + # this fetch. One raising must not stop the others being delivered. + for cb in callbacks: try: - request.callback(result) + cb(result) except Exception as e: logger.error(f"Error in callback for request {request.id}: {e}") # Delivered. Drop both references -- they point at the same @@ -456,6 +588,8 @@ class BackgroundDataService: return self.active_requests[request_id].status elif request_id in self.completed_requests: result = self.completed_requests[request_id] + if result.final_status is not None: + return result.final_status return FetchStatus.COMPLETED if result.success else FetchStatus.FAILED return None @@ -472,8 +606,22 @@ class BackgroundDataService: with self._lock: if request_id in self.active_requests: request = self.active_requests[request_id] + if request.commit_claimed: + # Too late: the worker holds an authorised commit. Report + # the failure rather than half-cancelling a request whose + # data is about to land in the cache. + logger.debug( + "Not cancelling %s: its response is already being " + "committed", request_id + ) + return False request.status = FetchStatus.CANCELLED del self.active_requests[request_id] + # Cancelling is the other way a request leaves active_requests, + # so the in-flight index has to be released here too or the key + # stays pointed at a request that no longer exists. + if self._inflight_by_cache_key.get(request.cache_key) == request_id: + del self._inflight_by_cache_key[request.cache_key] logger.info(f"Cancelled request {request_id}") return True return False diff --git a/test/test_background_fetch_dedupe.py b/test/test_background_fetch_dedupe.py new file mode 100644 index 00000000..4398a474 --- /dev/null +++ b/test/test_background_fetch_dedupe.py @@ -0,0 +1,450 @@ +"""A second request for a key already being fetched must join, not duplicate. + +request_id embeds a millisecond timestamp and active_requests is keyed by it, +so every submit looked new and nothing compared what was actually being +fetched. On a real board the season-schedule cache_key is requested by both +the Recent and the Upcoming manager: they miss the cache in the same +millisecond and each start a full download and parse of the same payload. +Measured on a running board, 138 background fetches in 24 hours arriving in +pairs at identical timestamps -- half of them redundant. + +The cost of a duplicate is a second download, a second JSON parse (the +expensive part on a Pi), and a second parsed copy resident at the same time. +Schedules on that board run from 256KB to 20MB. It also consumes a second of +the three executor slots with identical work, which is what makes two large +parses peak simultaneously. +""" + +import threading +import time +from unittest.mock import MagicMock, Mock, patch + +import pytest +import requests + +from src.background_data_service import BackgroundDataService, FetchStatus + + +PAYLOAD = {"events": [{"id": f"g{i}"} for i in range(20)]} + + +@pytest.fixture +def cache(): + m = MagicMock() + m.get.return_value = None # always a miss: force the fetch path + m.set.return_value = None + return m + + +@pytest.fixture +def service(cache): + svc = BackgroundDataService(cache, max_workers=3, request_timeout=5) + yield svc + svc.shutdown(wait=False) + + +def _resp(): + r = Mock() + r.json.return_value = PAYLOAD + r.raise_for_status.return_value = None + return r + + +def _wait(service, req_id, timeout=5): + deadline = time.time() + timeout + while not service.is_request_complete(req_id) and time.time() < deadline: + time.sleep(0.02) + + +class _BlockingSession: + """Holds the first fetch open so a second can be submitted mid-flight.""" + + def __init__(self): + self.calls = 0 + self.release = threading.Event() + self.started = threading.Event() + + def get(self, *a, **k): + self.calls += 1 + self.started.set() + self.release.wait(timeout=5) + return _resp() + + +def test_a_second_submit_for_the_same_key_does_not_fetch_twice(service): + session = _BlockingSession() + with patch.object(service, "session", session): + first = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="nba_2026", + callback=lambda r: None, max_retries=0) + assert session.started.wait(timeout=5) + + second = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="nba_2026", + callback=lambda r: None, max_retries=0) + + assert second == first, "the joiner should share the in-flight request id" + session.release.set() + _wait(service, first) + + assert session.calls == 1, f"the payload was fetched {session.calls} times" + + +def test_the_joiner_still_gets_its_callback(service): + session = _BlockingSession() + seen = [] + with patch.object(service, "session", session): + first = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=lambda r: seen.append("first"), max_retries=0) + assert session.started.wait(timeout=5) + joined = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=lambda r: seen.append("second"), max_retries=0) + # Assert the coalescing happened, otherwise this passes trivially: + # two independent requests would each fire their own callback and the + # test would say nothing about the joined path. + assert joined == first + session.release.set() + _wait(service, first) + + deadline = time.time() + 5 + while len(seen) < 2 and time.time() < deadline: + time.sleep(0.02) + assert sorted(seen) == ["first", "second"], ( + f"both submitters must be called back, got {seen}") + + +def test_one_callback_raising_does_not_silence_the_other(service): + session = _BlockingSession() + seen = [] + + def boom(result): + raise RuntimeError("consumer blew up") + + with patch.object(service, "session", session): + first = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=boom, max_retries=0) + assert session.started.wait(timeout=5) + joined = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=lambda r: seen.append("survivor"), max_retries=0) + # Same reason: without coalescing these are separate requests and + # neither callback can affect the other. + assert joined == first + session.release.set() + _wait(service, first) + + deadline = time.time() + 5 + while not seen and time.time() < deadline: + time.sleep(0.02) + assert seen == ["survivor"] + + +def test_different_keys_are_not_coalesced(service): + session = _BlockingSession() + with patch.object(service, "session", session): + a = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/a", cache_key="key_a", + callback=lambda r: None, max_retries=0) + assert session.started.wait(timeout=5) + b = service.submit_fetch_request( + sport="nhl", year=2026, url="https://x/b", cache_key="key_b", + callback=lambda r: None, max_retries=0) + assert a != b, "different cache keys must not share a request" + session.release.set() + _wait(service, a) + _wait(service, b) + assert session.calls == 2 + + +def test_a_later_submit_after_completion_fetches_again(service): + """Dedupe is for concurrent requests only, not a second cache layer.""" + with patch.object(service.session, "get", side_effect=[_resp(), _resp()]) as get: + first = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=lambda r: None, max_retries=0) + _wait(service, first) + second = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=lambda r: None, max_retries=0) + _wait(service, second) + assert first != second + assert get.call_count == 2 + + +def test_cancelling_releases_the_key(service): + """A cancelled request must not wedge its key against future fetches.""" + session = _BlockingSession() + with patch.object(service, "session", session): + first = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=lambda r: None, max_retries=0) + assert session.started.wait(timeout=5) + service.cancel_request(first) + assert "k" not in service._inflight_by_cache_key + session.release.set() + + +def test_a_stranded_index_entry_cannot_wedge_a_key(service): + """Defensive: the request is looked up, not trusted from the id alone.""" + service._inflight_by_cache_key["ghost"] = "no_such_request" + with patch.object(service.session, "get", return_value=_resp()): + req = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="ghost", + callback=lambda r: None, max_retries=0) + _wait(service, req) + assert service.get_result(req).success is True + + +def test_the_deduplicated_count_is_reported(service): + session = _BlockingSession() + with patch.object(service, "session", session): + first = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=lambda r: None, max_retries=0) + assert session.started.wait(timeout=5) + service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + max_retries=0) + session.release.set() + _wait(service, first) + assert service.get_statistics().get("deduplicated_requests") == 1 + + +def test_a_cancelled_worker_cannot_overwrite_its_replacement(service, cache): + """Cancelling frees the key, so a replacement may already own it. + + The worker cannot abort an HTTP call in flight, so when the cancelled one + finally returns it must discard its response rather than write it. Without + that, the sequence is: cancel A, submit B for the same key, B fetches and + caches fresh data, A returns and overwrites it with the response nobody + wanted -- and calls A's callbacks too. + """ + slow = _BlockingSession() + stale = {"events": [{"id": "STALE"}]} + slow_resp = Mock() + slow_resp.json.return_value = stale + slow_resp.raise_for_status.return_value = None + + def blocked_get(*a, **k): + slow.calls += 1 + slow.started.set() + slow.release.wait(timeout=5) + return slow_resp + + called = [] + with patch.object(service.session, "get", side_effect=blocked_get): + first = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=lambda r: called.append("cancelled_one"), max_retries=0) + assert slow.started.wait(timeout=5) + + service.cancel_request(first) + assert "k" not in service._inflight_by_cache_key + + # The replacement writes the fresh value while the cancelled fetch is held. + fresh = {"events": [{"id": "FRESH"}]} + fresh_resp = Mock() + fresh_resp.json.return_value = fresh + fresh_resp.raise_for_status.return_value = None + with patch.object(service.session, "get", return_value=fresh_resp): + second = service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", cache_key="k", + callback=lambda r: called.append("replacement"), max_retries=0) + _wait(service, second) + + assert cache.set.call_args[0][1] == fresh, "replacement must own the cache" + + # Now let the cancelled fetch finish. It must write nothing and call nobody. + # Wait for the worker to actually finish rather than sleeping: a fixed + # sleep is a race under load, and a slow worker would make this pass for + # the wrong reason. A cancelled request is still filed in + # completed_requests, so that is the signal it has run to completion. + writes_before = cache.set.call_count + slow.release.set() + deadline = time.time() + 5 + while first not in service.completed_requests and time.time() < deadline: + time.sleep(0.02) + assert first in service.completed_requests, "cancelled worker never finished" + + assert cache.set.call_count == writes_before, ( + "the cancelled worker wrote to the cache after its replacement") + assert cache.set.call_args[0][1] == fresh, "stale data overwrote fresh" + assert "cancelled_one" not in called, ( + "a cancelled request must not deliver callbacks") + + +def test_request_ids_are_unique_within_a_millisecond(service): + """request_id was sport_year_milliseconds, which collides. + + Two submits inside the same millisecond produced the SAME id, so one + silently replaced the other in active_requests and completed_requests. + Dedupe hands this id back to every joiner as their handle for + get_result(), so uniqueness is now load-bearing rather than incidental. + """ + # Stub the executor rather than the session: this is about what submit + # hands back, and letting 50 workers loose would outlive the patch and + # make real network calls. + with patch.object(service.executor, "submit"): + ids = [ + service.submit_fetch_request( + sport="nba", year=2026, url="https://x/s", + cache_key=f"key_{i}", # distinct keys: no dedupe + callback=lambda r: None, max_retries=0) + for i in range(50) + ] + assert len(set(ids)) == len(ids), "request ids collided" + + +# --- cancellation must be terminal ------------------------------------------ +# +# Cancelling used to be advisory: three separate paths wrote request.status +# without checking whether the request had already been cancelled, so a cancel +# could be silently undone and the work it was meant to stop went ahead. + +URL = "http://example.invalid/scores" +KEY = "sched_nfl_2025" + + +class _CountingSession: + """Records whether an HTTP fetch was ever attempted.""" + + def __init__(self): + self.calls = 0 + + def get(self, *a, **k): + self.calls += 1 + return _resp() + + +class _BlockingFailingSession(_CountingSession): + """Holds the fetch open, then fails it. + + Cancelling while the worker is parked inside the HTTP call is the only way + to reach the exception handler as a cancelled request. Cancel it before + the call starts and the worker returns at the pre-start branch instead, + which would leave the except path untested. + """ + + def __init__(self): + super().__init__() + self.started = threading.Event() + self.release = threading.Event() + + def get(self, *a, **k): + self.calls += 1 + self.started.set() + self.release.wait(timeout=5) + raise requests.RequestException("connection reset") + + +def _fill_every_worker_slot(service, slots=3): + """Occupy the pool so the next submit is queued rather than started. + + This is what makes "cancel before the worker runs" deterministic instead + of a race the test would win only sometimes. Returns the gate that + releases the pool. + """ + gate = threading.Event() + for _ in range(slots): + service.executor.submit(gate.wait, 5) + return gate + + +def test_cancelling_before_the_worker_starts_stops_the_fetch(service, cache): + """The queued worker must honour a cancel, not overwrite it with IN_PROGRESS. + + Between submit and the worker picking the job up, the request sits in the + executor queue. Cancelling there is the cheapest possible cancel -- nothing + has been downloaded yet -- and it was the one that did not work. + """ + gate = _fill_every_worker_slot(service) + session = _CountingSession() + delivered = [] + + with patch.object(service, 'session', session): + rid = service.submit_fetch_request( + "nfl", 2025, URL, KEY, max_retries=0, callback=delivered.append + ) + assert service.cancel_request(rid) is True + gate.set() # let the queued worker run + _wait(service, rid) + + assert session.calls == 0, ( + "cancelled before it started, yet the worker still downloaded the payload" + ) + assert cache.set.call_count == 0, "a cancelled request wrote to the cache" + assert delivered == [], "a cancelled request invoked its callbacks" + + +def test_a_cancel_during_the_commit_is_refused(service, cache): + """Once the worker has claimed the commit, cancelling is too late. + + The claim and the cancelled-check happen in one critical section, so a + cancel arriving after it cannot retroactively abandon data already on its + way to the cache. Letting it through stranded every joiner: the payload + landed in the cache but the callbacks were suppressed, so a manager that + joined this fetch waited for a call that never came. + """ + gate = _fill_every_worker_slot(service) + late = {} + delivered = [] + + def cancel_mid_write(key, data, *a, **k): + late['returned'] = service.cancel_request(late['rid']) + + cache.set.side_effect = cancel_mid_write + + # Read the payload inside the callback. The service releases result.data + # once every callback has been delivered, so inspecting the FetchResult + # afterwards sees the released object, not what the caller was handed. + def record(result): + delivered.append((result.success, result.data)) + + with patch.object(service, 'session', _CountingSession()): + late['rid'] = service.submit_fetch_request( + "nfl", 2025, URL, KEY, max_retries=0, callback=record + ) + gate.set() # only now can the worker reach the commit + _wait(service, late['rid']) + + assert late.get('returned') is False, ( + "cancelled a request that had already committed" + ) + assert cache.set.call_count == 1, "the commit itself was lost" + assert delivered == [(True, PAYLOAD)], ( + "data reached the cache but the callbacks were suppressed -- " + "every joined submitter is left waiting forever" + ) + + +def test_a_failure_after_cancelling_stays_cancelled(service, cache): + """A cancelled request that then errors must not resurface as FAILED. + + The except path overwrote CANCELLED with FAILED, and the callback gate in + the finally block only suppresses callbacks for CANCELLED -- so cancelling + a request that was about to time out delivered a spurious error callback. + """ + session = _BlockingFailingSession() + delivered = [] + + with patch.object(service, 'session', session): + rid = service.submit_fetch_request( + "nfl", 2025, URL, KEY, max_retries=0, callback=delivered.append + ) + # Assert the worker is inside the HTTP call before cancelling, + # otherwise this silently degrades into the pre-start case and the + # exception handler is never exercised. + assert session.started.wait(timeout=5) + assert service.cancel_request(rid) is True + session.release.set() + _wait(service, rid) + + assert session.calls == 1, "the fetch never started, so nothing could fail" + + assert delivered == [], "a cancelled request delivered a failure callback" + assert service.get_request_status(rid) is FetchStatus.CANCELLED, ( + "a cancelled request that then errored was reported as FAILED" + ) From eae063700f916f713f618da340466868025cb890 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:25:10 -0400 Subject: [PATCH 49/53] test(sports): pin the four-case sunset matrix before B6 runs (#505) B5 and B6 make different promises about an adopted scoreboard, and only the second is obvious. The table: | bundled copy present | bundled copy removed pinned old core | loads (the fallback) | ERROR, naming the module current core | loads, using core code | loads, using core code The top-left cell is the one worth having. Nothing in the suite proved that an adopted plugin still runs on a core predating src.common.sports_scroll, and that claim is the entire basis for having shipped B5 ahead of B6's gate. The bottom-left cell is asserted through PluginManager.load_plugin rather than a bare import, deliberately. The manager catches the ModuleNotFoundError, so a test written around pytest.raises would pass against a core where the module is merely broken rather than absent, and would say nothing about what the user meets: a plugin parked in ERROR and one log line. The assertion is the ERROR state plus an error naming the module, which is what makes the log actionable. Modelling notes, both of which were wrong first time and matter: - The copy-removed shape is an UNGUARDED import, not the guarded one with the legacy file deleted. Keeping the guard while removing its fallback only mislabels the failure -- the plugin reports a missing scroll_display_legacy and never mentions the core module that is actually absent. - Only the leaf module is hidden. A pre-3.2.0 core still ships src/common/; hiding the package would be a harsher core than any that shipped, and would make the failure name the package instead of the module. Ablation: disabling the old-core simulation fails three cells, and dropping the error object from load_plugin's set_state fails the fourth, so none of it passes vacuously. The install gate -- the other half of the guarantee -- stays in test_plugin_compatibility_gate.py rather than being duplicated here. Refs docs/SPORTS_UNIFICATION.md, "B6 -- why the sunset needs more than a version floor". Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW Co-authored-by: Claude Opus 5 (1M context) --- test/test_sports_sunset_matrix.py | 310 ++++++++++++++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 test/test_sports_sunset_matrix.py diff --git a/test/test_sports_sunset_matrix.py b/test/test_sports_sunset_matrix.py new file mode 100644 index 00000000..db6da2fb --- /dev/null +++ b/test/test_sports_sunset_matrix.py @@ -0,0 +1,310 @@ +"""What happens to an adopted sports plugin on each core it can meet. + +B5 moved the eight scoreboards onto `src.common.sports_scroll` behind a guarded +import, keeping a bundled copy as the fallback. B6 deletes those copies. The +two phases make *different* promises, and only the second one is obvious: + +| | bundled copy present | bundled copy removed | +|--------------------|---------------------------------|-----------------------------| +| **pinned old core**| loads -- B5's whole guarantee | ERROR, naming the module | +| **current core** | loads, using core code | loads, using core code | + +The top-left cell is the one worth having: nothing else in this suite proves an +adopted plugin still runs on a core that predates the module, and that claim is +the entire basis for having shipped B5 ahead of B6's gate. + +The bottom-left cell is the B6 failure mode, and it is asserted through +`PluginManager.load_plugin` rather than a bare import on purpose. The manager +catches the `ModuleNotFoundError`, so nothing propagates to a caller: a test +that expected `pytest.raises` would pass against a core where the module is +merely *broken* rather than absent, and would say nothing about what the user +actually experiences. What they get is a plugin parked in ERROR and one log +line -- which is precisely why B6 needs the install gate rather than trusting +the failure to be noticed. + +This covers the load path. The install/update gate -- which is what should stop +a sunset plugin reaching an old core in the first place -- is the other half of +the guarantee and is tested in test_plugin_compatibility_gate.py. + +See docs/SPORTS_UNIFICATION.md, "B6 -- why the sunset needs more than a version +floor". +""" + +import itertools +import json +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +project_root = Path(__file__).parent.parent +if str(project_root) not in sys.path: + sys.path.insert(0, str(project_root)) + +from src.plugin_system.plugin_manager import PluginManager +from src.plugin_system.plugin_state import PluginState + + +_PLUGIN_IDS = itertools.count() + +CORE_MODULE = "src.common.sports_scroll" + +# Only the leaf. A pre-3.2.0 core still ships `src/common/` -- scroll_helper +# and friends live there -- and it is `sports_scroll.py` alone that is absent. +# Hiding the whole package would be a different, harsher core than any that +# shipped, and it would make the load failure name the package rather than the +# module, which is the thing a reader needs to see. +HIDDEN = (CORE_MODULE,) + + +class _PinnedOldCore: + """Make `src.common.sports_scroll` un-importable for the duration. + + A meta_path finder rather than a monkeypatched `__import__`: the plugin is + executed by the real loader through `exec_module`, so the block has to live + in the import system itself to be reached. + """ + + def __init__(self, *names): + self.names = set(names) + self._saved = {} + + def find_spec(self, fullname, path=None, target=None): + if fullname in self.names: + raise ModuleNotFoundError(f"No module named {fullname!r}", name=fullname) + return None + + def __enter__(self): + for name in list(sys.modules): + if name in self.names: + self._saved[name] = sys.modules.pop(name) + sys.meta_path.insert(0, self) + return self + + def __exit__(self, *exc): + sys.meta_path.remove(self) + sys.modules.update(self._saved) + return False + + +# The two source shapes, kept as literals rather than copied from a plugin at +# runtime: this file lives in the core repo and must not depend on a plugin +# checkout being present, and pinning the shapes here means a plugin that +# drifts away from one is a visible edit, not a silently weakened test. + +# B5, as the eight scoreboards ship today: prefer the core module, fall back to +# the bundled copy. The except clause is narrow on purpose -- a bare +# `except ImportError` would also swallow a failure raised *inside* a core +# module that is present, quietly loading the legacy copy and hiding a broken +# core install. +ADOPTED_WITH_FALLBACK = ''' +_USING_CORE_SCROLL = False +try: + from src.common.sports_scroll import SportsScrollDisplay as _Base + _USING_CORE_SCROLL = True +except ModuleNotFoundError as exc: + if exc.name not in {"src", "src.common", "src.common.sports_scroll"}: + raise + _Base = None + +if not _USING_CORE_SCROLL: + from scroll_display_legacy import LegacyScrollDisplay as _Base +''' + +# B6, once the copies are deleted: there is nothing to fall back to, so the +# guard goes with them. Keeping the try/except while removing the file it +# falls back to would only mislabel the failure -- the plugin would report a +# missing `scroll_display_legacy` and say nothing about the core module that +# is actually absent. +SUNSET_NO_FALLBACK = ''' +from src.common.sports_scroll import SportsScrollDisplay as _Base + +_USING_CORE_SCROLL = True +''' + +PLUGIN_BODY = ''' +from src.plugin_system.base_plugin import BasePlugin + + +class SunsetProbe(BasePlugin): + """Records which scroll implementation the guarded import selected.""" + + using_core_scroll = _USING_CORE_SCROLL + scroll_base = _Base + + def update(self): + pass + + def display(self, force_clear=False): + pass +''' + +LEGACY_COPY = ''' +class LegacyScrollDisplay: + """Stands in for the bundled pre-3.2.0 implementation.""" +''' + + +def _write_plugin(plugins_dir: Path, plugin_id: str, *, bundled_copy: bool) -> Path: + path = plugins_dir / plugin_id + path.mkdir(parents=True) + (path / "manifest.json").write_text( + json.dumps({ + "id": plugin_id, + "name": "Sunset Probe", + "version": "1.0.0", + "entry_point": "manager.py", + "class_name": "SunsetProbe", + "display_modes": ["sunset_probe"], + }), + encoding="utf-8", + ) + shape = ADOPTED_WITH_FALLBACK if bundled_copy else SUNSET_NO_FALLBACK + (path / "manager.py").write_text(shape + PLUGIN_BODY, encoding="utf-8") + if bundled_copy: + (path / "scroll_display_legacy.py").write_text(LEGACY_COPY, encoding="utf-8") + return path + + +@pytest.fixture +def load(tmp_path): + """Load a synthetic adopted plugin through the real PluginManager. + + Returns a callable taking the two axes of the matrix and handing back the + manager, so the caller can ask it for state and recorded error. + """ + plugins_dir = tmp_path / "plugin-repos" + plugins_dir.mkdir() + + def _load(*, core_has_module: bool, bundled_copy: bool, hide=HIDDEN): + # A distinct id per cell, from a counter that spans the whole session. + # The loader names plugin modules after the plugin id and sys.modules + # is process-global, so a per-test counter would hand the second test + # the first test's already-imported module -- which passes or fails on + # the wrong plugin's import. + plugin_id = f"sunset-probe-{next(_PLUGIN_IDS)}" + _write_plugin(plugins_dir, plugin_id, bundled_copy=bundled_copy) + + with patch('src.common.permission_utils.ensure_directory_permissions'): + manager = PluginManager( + plugins_dir=str(plugins_dir), + config_manager=MagicMock(), + display_manager=MagicMock(), + cache_manager=MagicMock(), + font_manager=MagicMock(), + ) + manager.discover_plugins() + if core_has_module: + ok = manager.load_plugin(plugin_id) + else: + with _PinnedOldCore(*hide): + ok = manager.load_plugin(plugin_id) + return manager, plugin_id, ok + + return _load + + +def _assert_loaded(manager, plugin_id, ok): + assert ok is True, ( + f"load_plugin returned False; state is " + f"{manager.state_manager.get_state(plugin_id)}, error " + f"{manager.state_manager.get_error_info(plugin_id)}" + ) + assert manager.state_manager.get_state(plugin_id) is not PluginState.ERROR + + +class TestBundledCopyPresent: + """B5's shape: the guarded import with the fallback still shipped.""" + + def test_old_core_falls_back_and_still_loads(self, load): + """The claim that made it safe to ship B5 before B6's gate.""" + manager, plugin_id, ok = load(core_has_module=False, bundled_copy=True) + + _assert_loaded(manager, plugin_id, ok) + plugin = manager.plugins[plugin_id] + assert plugin.using_core_scroll is False, ( + "the core module was hidden, so the plugin must be on its bundled copy" + ) + assert plugin.scroll_base.__name__ == "LegacyScrollDisplay" + + def test_current_core_prefers_the_core_module(self, load): + manager, plugin_id, ok = load(core_has_module=True, bundled_copy=True) + + _assert_loaded(manager, plugin_id, ok) + plugin = manager.plugins[plugin_id] + assert plugin.using_core_scroll is True, ( + "the bundled copy must not win while the core module is importable" + ) + assert plugin.scroll_base.__name__ == "SportsScrollDisplay" + + + def test_a_core_without_the_package_at_all_still_falls_back(self, load): + """The guard's other accepted names. + + Its except clause accepts `src` and `src.common` as well as the module + itself, so those branches exist in all eight shipped plugins. No core + that old is likely still running, but the code claiming to handle it is + real and nothing else exercises it -- an untested branch in a fallback + is exactly the kind that rots unnoticed until the fallback is needed. + """ + manager, plugin_id, ok = load( + core_has_module=False, bundled_copy=True, hide=(CORE_MODULE, "src.common") + ) + + _assert_loaded(manager, plugin_id, ok) + assert manager.plugins[plugin_id].using_core_scroll is False + + +class TestBundledCopyRemoved: + """B6's shape: the copies are gone and only the core module remains.""" + + def test_current_core_still_loads(self, load): + manager, plugin_id, ok = load(core_has_module=True, bundled_copy=False) + + _assert_loaded(manager, plugin_id, ok) + assert manager.plugins[plugin_id].using_core_scroll is True + + def test_old_core_errors_and_records_the_missing_module(self, load): + """The B6 failure mode, as the user meets it. + + Not `pytest.raises`: load_plugin catches it, so nothing reaches a + caller. The observable consequences are the ERROR state and the + recorded error -- and the error has to name the module, or whoever + reads the log cannot tell a missing core module from any other + import failure. + """ + manager, plugin_id, ok = load(core_has_module=False, bundled_copy=False) + + assert ok is False, "a plugin with no scroll implementation must not load" + assert manager.state_manager.get_state(plugin_id) is PluginState.ERROR + assert plugin_id not in manager.plugins, ( + "a plugin that failed to load must not be left registered" + ) + + info = manager.state_manager.get_error_info(plugin_id) + assert info is not None, "ERROR state recorded no error to explain it" + assert info['error_type'] == 'ModuleNotFoundError', info + assert CORE_MODULE in info['error'], ( + f"the recorded error must name the module that was missing, got {info['error']!r}" + ) + + +def test_the_matrix_has_one_failing_cell(load): + """Guards the shape of the table itself. + + Each cell above is asserted on its own, so a change that broke two of them + in compensating ways could leave every individual test passing. This says + the outcome depends on both axes and fails in exactly one combination. + """ + outcomes = { + (core, bundled): load(core_has_module=core, bundled_copy=bundled)[2] + for core in (True, False) + for bundled in (True, False) + } + assert outcomes == { + (True, True): True, + (True, False): True, + (False, True): True, + (False, False): False, + }, outcomes From 4aeb0033e0327a3d39d4a40e111e7ef736d43673 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sat, 29 Aug 2026 18:32:50 -0400 Subject: [PATCH 50/53] fix(install): don't abort when journald settings are absent (#507) --- first_time_install.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/first_time_install.sh b/first_time_install.sh index c7ec7cf1..198a8f2e 100755 --- a/first_time_install.sh +++ b/first_time_install.sh @@ -1740,12 +1740,15 @@ journald_effective() { systemd-analyze cat-config systemd/journald.conf >/dev/null 2>&1; then systemd-analyze cat-config systemd/journald.conf 2>/dev/null else - cat /etc/systemd/journald.conf /etc/systemd/journald.conf.d/*.conf 2>/dev/null + cat /etc/systemd/journald.conf /etc/systemd/journald.conf.d/*.conf 2>/dev/null || true fi } journald_conf="$(journald_effective)" -journald_storage="$(printf '%s\n' "$journald_conf" | grep -E '^[[:space:]]*Storage=' | tail -n1 | cut -d= -f2 | tr -d '[:space:]')" -journald_cap="$(printf '%s\n' "$journald_conf" | grep -E '^[[:space:]]*SystemMaxUse=' | tail -n1 | cut -d= -f2 | tr -d '[:space:]')" +# Both settings are optional, and stock images ship them commented out. grep +# exits 1 on no match, which pipefail turns fatal under set -e — an absent +# setting must read as empty, not abort the install. +journald_storage="$(printf '%s\n' "$journald_conf" | grep -E '^[[:space:]]*Storage=' | tail -n1 | cut -d= -f2 | tr -d '[:space:]' || true)" +journald_cap="$(printf '%s\n' "$journald_conf" | grep -E '^[[:space:]]*SystemMaxUse=' | tail -n1 | cut -d= -f2 | tr -d '[:space:]' || true)" if [ "$journald_storage" = "persistent" ] && [ -n "$journald_cap" ]; then echo "Persistent journald storage already configured (SystemMaxUse=$journald_cap)" @@ -1771,7 +1774,7 @@ else # after ledmatrix-persistent.conf (zz-local.conf and friends) still wins. # Writing the file is not evidence it took effect -- re-read and say so # plainly rather than reporting success we cannot confirm. - journald_now="$(journald_effective | grep -E '^[[:space:]]*Storage=' | tail -n1 | cut -d= -f2 | tr -d '[:space:]')" + journald_now="$(journald_effective | grep -E '^[[:space:]]*Storage=' | tail -n1 | cut -d= -f2 | tr -d '[:space:]' || true)" if [ "$journald_now" = "persistent" ]; then echo " Persistent journald storage active" else From cbc540a679b800405780a41c690a9acaf3b70d67 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sun, 30 Aug 2026 09:08:42 -0400 Subject: [PATCH 51/53] fix(assets): drop case-colliding duplicate league logos (#506) Four league fallback logos were each tracked at two paths differing only in case: assets/sports/mlb_logos/MLB.png + mlb.png assets/sports/nba_logos/NBA.png + nba.png assets/sports/nfl_logos/NFL.png + nfl.png assets/sports/nhl_logos/NHL.png + nhl.png On Windows and default macOS the filesystem is case-insensitive, so both index entries map to one physical file. Whichever git writes last wins and the other entry reports as permanently modified, so `git status` can never be clean and any `git pull` flips which one is dirty. For MLB/NBA/NFL the two entries pointed at the same blob, so the collision was only cosmetic. NHL was not: NHL.png is 1439x1621 (672455 B) and nhl.png is 768x768 (107184 B), so which resolution the league fallback logo loaded depended on checkout order rather than on the code. Keep the uppercase path in each pair. Every logo lookup uppercases the abbreviation before building a filename -- LogoDownloader .normalize_abbreviation and .get_logo_filename_variations (src/logo_downloader.py) and LogoHelper.normalize_abbreviation (src/common/logo_helper.py) all do -- and nothing in the tree requests a lowercase league logo, so the uppercase name is what the code actually asks for. For NHL that is also the higher-resolution asset. Removed with `git update-index --force-remove` so the literal index entry is dropped without the case-insensitive working tree deleting the survivor. --- assets/sports/mlb_logos/mlb.png | Bin 16415 -> 0 bytes assets/sports/nba_logos/nba.png | Bin 23678 -> 0 bytes assets/sports/nfl_logos/nfl.png | Bin 40064 -> 0 bytes assets/sports/nhl_logos/nhl.png | Bin 107184 -> 0 bytes 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 assets/sports/mlb_logos/mlb.png delete mode 100644 assets/sports/nba_logos/nba.png delete mode 100644 assets/sports/nfl_logos/nfl.png delete mode 100644 assets/sports/nhl_logos/nhl.png diff --git a/assets/sports/mlb_logos/mlb.png b/assets/sports/mlb_logos/mlb.png deleted file mode 100644 index 7196811cc0735e35c6351ad087c9764d3fbec733..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16415 zcmeHO^;eW#v<3m`mM-Zo5ou5X0VNcq+o5v+>F(}Sx=U$?hM|U%?i{*1qz1Sz-@53EQgCliG_rOgsUJgql$!tyz=ym@f`TXd$KJV2}x>8 zLFV08xB0^**Nm@I_m8JDjjr><2R6seb!fZta+Z01Mpg7iGZqKT5|E8_xMuWE`=5f8 z#7bXZNAOcNS6@Y-kzljEY^@#q_eS9As+S9Pv1Gb8e0SqUkhn6uDB-l-a=g9hG=fa{ z|3Cj1ED*m8E4W1#jRCbe{vtJUeV8a1JbuZ*a2g&qEEPX2wL1KWyF#8ge;Vf~ca;pi zM7nt^GK#;0{S~S3Eo~fbgIJfAjLa)uuLXaN9K2af8`L+*)?$y1wdcn)ps0MLhpSv` zq)>z zMvn5rSp&+zz~IS)dUb-kVw6LU%N-de|12NF0?7c)4m}HPk4T@8mFt?i*MYFL z*b}eHZ^wiKtuic(X~J$rVMPe4`&KS5K==rGR%9^)g&I8rABXjpEgpCaM;Ow+KOET= zX@Q8a(*f70*MZB34NPAZx8rR{NErWh^5ge}X?18bEDfyl99V&!5et}t?~6L{-31DQ z2u!fZNg98VpzX73&R9|!e7%p~k#T|NzSrPhGlQ{5$z$nv-B7`Az$_<>c_Zf$z*G4i zD;_IWIg-=#)U`-Qzr0YO_2J9sLlfq#3=EbPnAd_Q*ha2g(!weGY@YrT{@5?t7#PMs zq59cBgCir@HIZE<*FE*je;5!AqI{Iv9z)aYqBb)sBj<^(%#=4F z&4jC-MCJs2$Uk*c1tE*GzA(F@q8o3r8-)Lxh@UitIyP6=)`H2lR(XcIGReB}Zu#xV z9A+^~X|H{u$5H&Y)o$-Jah{D6=+EudXoc9z$%XwEv${T6pK3WhlH+)^0ofnr=s%t~XwXAAbJS4Hw2Emo7U#W@FG7vYRxrYrbU&!9U6-f*;2G z6xGOh#CDN*D3i>1y)LZ$eWTNF^L(Y8gWi5#DQ!r+7db#6{u5#_4XZA|}HvHRQ3}t6#*C3XD{PLXfd`eHJv^lto1fEy& zl9xHg^6Jy^=`MxlIK(#fHtRdy`tSjG6TM%`hy=x<9 z%=c*wwuiqO5gKony`*;DXjTgMM)5a9p0P=Ikdl~F%T7Z-{#{=BXMxH??;$0e6oqtz zzKEuHylWlw-q^s~Q$(tDHMXVcyT~C8O_`hQtExaQ_P^^*gY7+?U1mnCH$H%m*Lk&N z#i}$$!D2ParS?czDTIZ|Qmisk{l=%-GA8cg+`g9En7MEB@4`RP@Z}r6RK$hruZ0FBASkQ8O?gzb{8N5oJ9oM%(Na*&^>oMxiuFmM z%ZhRdzEOxNtD&AV6_wpJ@oLCcR@+adi#G=a!+KG^NGP-`T1MGfEymW?V)e`C!kSB9 zvSzD&=l5Y4Fmi^Nu8GQ)F2VLY;%Fsq1zckgPDKE9*{~R;tAo!OF9y?AwtY!k_~i_| z3zq(<*wFZ=INN~&%rVv@N<;)j@k6^Xx)!C5IT=-}JUgd4E~>~?&`e=CbItrF$M!e# zN}|U*R$PURTKV>L<3K229ApOK`b58oJyH)=vXM_Kn!MRVo! zqv@^ocZWic+Tz!RTj~)46%BU$Vpe+qUYzlY^(-k3ArLdJ=eOgr3BZwzeCiRxW99qR zE}5~cgE=EubEN6|vlqlHU&Ea^ z{&AMwr`nP#)zAsuTF6G}_Vf2?Co}xT?i`;Bz;ISO zV;G|N;knykX6$v3($(jg(WFJZc?f%!?@M>| zsQZNu1zxdL>b+&X;S@329?RpgpebNZ zF(xaz+IIBw_eycNj*y-C{*nrj-Qk#-9I;>7ks@qQ8A%f*u*p*<9dfY2tI{GP$M$Pa zQ>IBbt;%v=*~f8dZ3hw0^gMg>T!z&I_pp+(A=QzA&_&{sWr%a;azl?3{X=)lvxk9u zxE;P!`}OTBN&Pj>V!R_GQ}2Wq!Kciunarl@$Y{UAdN zbC;9p658nEquOY$?#_J`8qhao!Yj6Yf@nh5vS`v<-0kdc7{a z53jb3h)H|*fUHAb?#fctfhkwwZO2X{q0454v($jUIE6IKB8pht6c~1(eq%S+Q4=dU z1C9j|Bk?5-k|%R_GVbNR)PPxFkc*Cs&U3D> zaQUv9WC@Ypi120X7)vz0x8F&>rv@_;N?w}?a@!tKmJ!CMf&F474F^ePmOHj=n-fSo zQ%0n3cI5!`sL*}17aw_V{8Q3kBGEjc)_AC?XYP`hJiAVFkHKL zMcDU|mFQ+$%JAlHSu0Z*I>Fo8Rq5+1HNK8O zMgIJ{>5SLa8KS|nq~}x9SQw3~>-r&^kpS?WgH+8v*h&qTxPedy7u}q;B*LGo=9#vM z`3;824Y=9%eJ&qiFZ~NFHTdLv5J=#V`}gZVkg?Ilk4~ScR>5hbQm9U%G0GGm3+&+Q zu7qBSHw{v(JOKO-DkB_bNtvG8AC>1Em_yU*JN@~P4@5H806WRAfI^2z#uJ>+B@n%U5ZGolDAN4CB}`jso= zg*QA4FvxEU^&OcRb4Th>FqX7KO=dG*p_(5#sKnM{&0J`CCwA1FDbL+2N?*Vad2c+Qu6w~Afe0NHgsq%6vGcNjLzH-@i8{Qv!=sXa zY}mMD`We_{$LD-89`9+Mk!a?rPj*6=nK ztrTomNTd-YTatwNK*|_3H-_LcdgzyS{f~MntqyB@-|l9*3X}Ee%({A4FZbNIvcsFb z@lC{Dz3*CCsD*3D*0ysPiAMk<=;HRZ^^1U@o7 zpR#*7*jr*e#U*<1c8i+Nf3J>6DSd zIDc;ny`@mu&|#y?`62O}HPtMyRqhn;3lZGtAz^mXZXjRk-8bKni$tiu;JK;#iQ>Gpjwx><4@DUHV$aRj%;qo{zGS}lww-#% zc%iDc%o;&kaO1mSksFMoo1sU?e>1T;Ell2>SJ|RigLw^nv5Z@aoApwU$gc2<_6(1| z^hso!pqGMKA}-X+4@S(7SNqMSb4zh2pjWUsA=}r?*k<3j@=ZBpNvh=Y0%ssNpbrOK zvfJ5Edi29kslf&7GI|h>$efRvBJEH?wX}%)nBGy_qb05PmLOzilG)h;Fq<&-mVd9| zw^_;i>tTw>6K1z~oWbLKL>*&^_pbg=jfx4f_g|4CnK-<>X@#)2hgKlgyv7eRMZ_m} z9-E&xvQ~F36F)dyZf@6JhddfTELx%_wkW}~l&E~}L(#AI&+Q4rWr!Xg{ORvYu3?0; zl9gpfUyvf1-YdKK9cQ_}o-sVN1qgDi?d3ydDu0NdRHXJquhSqe?*^>At=A_%bgkxh zfn7?KtvR=*L4De0c7+1sbKPK($rzVoaZF4FY6%W0T9rJ41K!#4QtDy2U(f3S6Npl? zMO11K&W-!Km~^Q~F?$3*T>O&lBi4BWsU%*NR}Yz<^S!>!f_D1Jl`Gk{n)C2`@!&=_ ze_3QMqU?*phL@>>J}0=t5}Wb?a}QBj_f`v}Q{IC&{j$a9cZT}iNtle_ITuwH8CmMk z-fZ7Q!^c%*fcq1B;J{vO43U32{eu3EXu9-iVU?9$=Z-ZuWA(~wNgzCh>!9qf6(QaR z_@UIV!xt)Bg9Ma?lZ>F*Nd$J*$17K^Qm{Z@?;l{{>Z?HR?9_;f*lwwkSY<50V%@L2 z?j?EQA|S$I8ecYT$;zxdMCeH^S0pm9uw7#a? zEQnSV>Dw<3xAEkaa>O+G`?>k@KGu97GS) zF!)?7CGNtD^;Q>fA+`>qN!5Bzb3zo8?HswuCx90n-3|&Yp0Ct?4|F zzPN9Fxh~`UiWd1ytAMpgk-UE7sT?2F=i!qY^M=CYLmqBFCKL~yNc4(myF)yZ7 zYxn7WmlU535=k*_Orew93n;@AQakGwWL$vQEO6JBK_HB2VQqAnu>2H8qWw+X?FH`)ZdKCaKO$b^35D@_u$CadNN{^U$KX|X}u=r0) zVmX!z@2f@phx2^in(Maq@#Xl2yN;~!XJy0sLAw8)+W4QndH^3LVzTA<;tHRK8x#C1 zWm`?K&y;s}c-MK1F7&6)@$k3rRw-Euk@D!h5l9>GM7qp4)6zS2q{u$s!(Zk~bRz}) zU9Terr)lC$)806|D5iD7I6~!}vxqVt{6|^QV#lBTOE0|&8%c~HGcH+Ozm!7@nTz33 z!Hy5=AUDjka#hN|2Hn3riC*90ce&09LQeqL{$syYMT-mng3Bb!#(gyZbtrDmS{-9m z-TRbhS~3r<{EW}5;ackLr<3(HNdhY3)V={ImVj%ih+na^H?F_a(On=uZ{tZ%$Tv@H zew4=%tpu!mrlQA_-esmz^paODnotg6%WwbBZSx6a+u zJV%)jlJ=@y7DG)#djHB7v|H_*7Gt}#NYMmXRli?<{~cF|MA~n|GstJQA=yCG@Wl`D z`&p*28QXpN$3ze6nDvueF}l-1I9F~(8~YKx&5ifhtUpMG?XT23u#DpWCMR#2p5(O6 zyU{BQe8T#{XJ(X{bPxp0j1 z7rVGN5@xF?rc)1b#Y|x=*MvX|z-MNbJm^N+&UoU5?dOg3-W2vyD-H*(iedsD^P@C} zFV<`6QOTg!gaQ^tF7xG`!FX&BdDqsVpq|4ejo0l>Q-c8kkjj(DzBoTHC_Np4i3z46 zJc0L5}5DM1|kDvh*X(G5ZTAZ8b}p@ij-)I7b41nNBy zapF$CjlUmh-{tHlA*06d7Z zd#z6@8|-dzAH9LT97d0F;x57NtO%6X4QumaH!F$i4y|(5jrvKn0EC5isiOhT{EErX zHa%c5M+5p1nWQjyW7DK~F}Q;@yK|S9 z1yM7f0byQtww5IR288p+IT~>QYj$5>iVc^^XlPY2+IY*METTyKWb;-5F9{wpr=r;H zqHOQ%+`!wiFRFgF+5iR@o&D${qk^0yT9ekgtWRGDdvB)%ix7w8&iOwrQ0-C_Z&UV0 z&+W)z@wPlNtqtBbA4IP%$~O2}Locqz+K~zbrViCi@#(AdsSRqng*D(1A8XT>pJfEk ziXh|40=68N^3qS~*{JTiS>okg`44d|b3?M!6n*RnV#S<~`>8+}!ChSLJK9H0kCs5m z@8FpoFqg=7JO3gxA6P@3vf)_(^04F$hF$YfDD^8R-x@CiXB&35w9Mdz%y~v47&+xn z-NxJct22c4;2Oj-F!QkZ>>${!{NvZBvmCG76NH|4-#QGLZaZ!EnDbTnqNM}~nVR0! zf{XjCv>v&GgzT33pNh|ILz}3D&)@&G1+qz5(x&LvD*(2$WQyBdPwT>88>V^)B_^X* z<##!+N?CgwljKv z7g1POm2wXc3Rtt*h2t-9QstRC6M0*QaO)-BezMj*5LiCUyJ%=c?Zq7nPY8L9`N>1+ zWFE$61NP++-WJ~SwG~BfS5dd;y|H1m)LA;5t6ROg5{Hex^A=exnQO2SE2=#H&5;YZ z;OpO7pGpjfYn}*rLm-};1hx09KUdp>9JT8tR}G5xUYN(=cBC}ZTEXGAWmK+>AF*4#d2S6ghDe+CD#( z*viLFB-aO#(TI(oVnEfV`&W8)c&G?i!x*Nvs43t^r+9|33q2j^iYqbuzTI=he_0Jj z{{8!CYfIzcp2Rw)4v4ia*SB|io`czzHdf&``{=zmMTly0@^>`VpVSUOVZju8z5u~^UNdf` zUbLK3T^bzoxn$<)3#V=8VW#a(nLt^MLp#fAMivh^W*Xp_p^Cg9k~^UT#oLw%5OB)rc4$;)<{MZt<3lXtfnz zD!;W=8kdMw*YzN;`cE_;!L!b>g1uIf41Sv>?Ln1_*1%ZJis-rX4j8$Y_s3FkS9%wh zl<9CFJruN)l+UO&H^0LrWc?#}8RRhscGB!T>r$BkZ(h1q*B0@(R9b7l7|v8&?YP_R z26A{oJ6Kq;giJi~X9RU`ZgnX$WBDI2QzvMowZooN>czy5`-+yA>f1b>y4t^<=xjpU z>Pihao|qe!DKnZ?oN$;7!=&JAvyDUn82SAC{LP_$?8%NmC1H%rw|`9?4$|ymc03y}j%e>2SKgE{%4^v)SCzLR?m9O@TMCDv&0PrhzZnF$_HMeBD9KXCN05M6921@}eR)fU0Y<C4HJ_O1 zc*&>l)EpnNMgE2T;P=!}dyxJ77Iti68b!Y*EPl7wtml2+`nr233(%n=@G?JiWZXL= zmm-lWR4O|hMTJLE>->g=nVXwXAS8L8C!TKW0;<0Ll~hkFz9dzxbJbx6QnEQOR98Fi zzXj4x**m=B>7^4)=gOADC7MBk^oL0Wn+9=mJ{V80iA1-zx7aib<{BtA&TBO0}MXG$( z43x*3fNPjsQuUoV06D84*R1>c!TDJh$W(*)om6|)S-C@YFG?_PhIDpEGFuY8#Y#1?vXM2 z&UN;cxwiW}@13K+AIvHh6sli^0~-u4_mSX9U*8(ZYpMH;n^3^fQ1dG++rW{9l_zb8 zY`=bVnf7BNk_)wkg%$8Z30>F8D$7AU$U%wmK1Bb(&LZ#6pTpxU_|n2>TJaqG``e_R z*Nf4%@xCi=XJ%1dyBLI)wyfjhAA65JxJV+1_A46G!D^8o`~g%8=dWwRKQ~i>X0;T@slu&hkQA~3A%y~y|x!c`63DEvQ5py{z`pt-1 zWd6>EqU+ilJ(237_jhY_Z}g^vv@(*b0y2p?jqKq3kWK40dzb~bG9bTXG;&AgZ_&I3 zB3J*i_%nI&S+zD2B)WHwY5KlhoV)|Vdq^R2tLFP$ z@#5nSw)oYaCjhFSx*Z-a-M68Axl>8Wq)cs{^?*DUox7LbTeVfA3!WFdFy`=+{bqO% zI8>U5Gc~QP2;1qeDDKF$PVudw`D}{B-V-3tG~W4}yB*94&DMC&NTF4mQv9P6qHgCE z8O+tOBWt{i?H>L|Tvj@PUV=m#-um`sWL~G9dV50J-Z(ixMignK0;(CWRyXpu9q(FI zPWmP)t7{<-gWKTqKxCE}iU`vmCbYlW$ewIwX=i78*)()QzRcEGeVH_m|YAbvN{+;abbWu`(`Y4!Ho0E9aP=OCS^ zcWoG3)diuK0m5flocuUQmagTzihJO5j}o=c7UU@}R4Mb1;g_q>>(g$5N@A?wsh8t- zBkqHpBo~PijxXC@|9p)nMNtwo+k2eWW2Ew zz~z`fRP{Ahhb)3YCK_%H`bk-QscSa^d?|$lKz?OvAmR(Tg_)&PYUSg}zl!1Hm4_c^ z%Nw}~Heo-3dIOL;S+jpncpLTiVUPYoCNVh{lP9hQ!nVKK>hQz7Xq|F}i3t{{&bb&^ zi?%xSpT6_!H;@v389-P}kdIaxdi`3koSsP_uzS1X%tKUjAVQKcc1s4KsS_s6^jv3Q z-qO}uQyOT)1BIUxX@Y>U@KE;eeA6KoO3CY}=|8%Y!cXpj9m*KgMF{M`0EsHlS6&Js zeY+|{&G6gyb;~0~wjr1%dQj8*=D4q&T;wxx*nz}CO_d;_>??^V?!d#VctA#aet+SG zm{PZT0bqa^8bb1!KbK$-##nOASy626?SRvXda}zhu|_kX$$k6lKn>LX?0-kTfWi(i zjTef*M<&k^#}sZ6=U<)ykp7LdNf?i3Ft4V-k9cc@p#)27;}`|DUS!ho?uCzS1m(Kn zC=*w_-SbLQ9NC-3D4IX`PG*WG13~dolp4{6o=}EIJn)So9C}QZ{6CQZr}f-lX{jtlMf>#1h&~io|9J>cCL~+A{3J@_vKv$hXLV}Zd`lz zC(3xSq(jf6LnuwqmYTL+7nkwrMQ$OLgEeVRP}GXsRqZJ9aYKQ0*(Z7U_~f#|hLpzR zWvZbYB>-)fecCcl{2$+tte4t>KvUn51&|c54CBdPL4JR@CVSn+LP7V`Hd)OTXxK{- z0Gj^%Du!&LUZMbf61&2lPDsKsB7{P?frJ*$$s?K>G0qTNDgWzyhTM?_&%Cdax4>vx zCQ28j68JZxwKQYIMS{Strh3^4Nw_A4E~$S!3We7_T9WVmXRdaYWI!W&rTtj;s-orO zbqMZW`3Z@_Tjm3xUb?NRB|qu1UnymKzHJ}53>@t%zja|$6m325QMF=vT_vp`k%AFM z^I5>xtme2}A$^%)A4WbNF_5%Y;Aw)xs>Ze)(St5fQ_r;Bo3>gh) z5-Q(2q^F*J@6mI`*M;iHkrxA;V55ssWUs4V`Iu&RQ!g3%Z+Uc{!~L+zcqI}$W(?l~ z%2b^3R27F&f61N@@5;8PIo+TnO4)a~Ces|f4UWspNFXdRVrc%R^u*RW_p8;}&LQKq zK8TfJX}x0c;$G=%RQn<1S#xw3QSepJCSEWn;Xz6~nXumQ<8G%mAnG|#7i(&Em-#V+ zE-E?J7wxHKV?wfIRcje8JoAg53|AnGIvxf?z`!|f{Z?p~KpHgML#V@QS@t;!^bFc2M#2$*XS z+}CFE4rq9S6!ZiskhtJT>(H(BaqjN=RBQ3tOfOe!>wl`e<6n9J+bi}onTFoaYLOmw z>U|^~GU-zyix;qz*-q6(UbGfFln{F09$cxh6}J;HR{jhZx zLabm*3oBzl(eygI(b4#&Gg8DYW?^d`YY;^dxK}<|?h^|%ga*#OskbwEw3+1e+2C1#oI?uhq2Dl7-fJiV*o_CF#x5#9bi&E;En)QVSQZWcUGjnr&BN+YsP#~tL z$xt@-h>#N0H`s5s{HtG)Tq0@V*gGKo$Sv--m^DeKg2oq(ZsS_k^%}u*?E=^2j*!mA z#5u18#@D`WlD2pAe{qa@bD5+L7Chz1)&vRF+4%}_7(f@!3 zC=94IUS7w)KsUEzPW@-@>jJ-4YHsG=-8pOq7prAtr$aQ&cWt`kkPn1h(#j0u4fN2= zV<{B>mAtdm-7!%_5{n5m)&47GLTR4$U0Bl4i<5ByB77ESnc=Wz+_2;^&zZ~m$5a@Y z7JNavQN}Oh!SCu5mshFOQLrl>8&&3qep$KEY)-56&VcVIIi5d+D?7DTSu}%vJ;a(# z)SpZT&0XYghNyBZZTQ7Qxh=o}I%ilV?5|7}@i!fsg)LV*q9}DTM~(Y|uGg^mE;vA2 zG8h~Hd^Q9qKY|FZ@j(huAElNoP|uB@-DMFX9|+@3-E(Mt-@(ztG>C%NaG5DnjdYJm z|M{r#tidMzhgxp}6%LSkU(p9i*nK1$f~o1mxyUL5)2aMpa;ku2$GmxueeRSd85JDfg=xU2&dlL)#Kc|@Tdq0l7AltAa z7mcCrH3mB;XWf(Z^8qt0Jrw8tw1V;^;4*gh0noMNed0X()o(6SqsIY*>R=6ybYN1# zU1ZJ*BOa$bptg*2_xx}jylGafl$%J1yja{4l*c5MkU)&XS7M0vaphXQ9XUJo9)_!O z?d?OT;7*|HjMx-Ek`$N{d%WH%TYA%7bLHo8VCs}x3m zTxlyfJI@BJWP5lhli(!97*GGFpUQp><27>hsX@! zHAxb~Q5Y*tw=DAuVhJ8NguTYSF*==CD-C*!NiQ~DX~oio_hk`4@&~`-UBd;yWZQ)s z1;2nN0BjkZolKYnS=i%O8&9~r#unwlRCIp)2$YnQqzq^EeooD7l4oykeXdXsYghVBBH=Yu8kbHP*DH;nb*<6Y^gN?c}tfKGU|sAiTMY1IAgTHu*lrW@S1Gp2MUY^3nLP7Rp_!!~ZE?KH7d^XSiJvEOVV2V4OlJTDCFvKT-YL)e+OR^N2$(sMQ zldt5_A3jr=VM6TnI#6r;Ug}Qdmf zjUeKbcVu*YK5KTbxi#>P);&vbzkY{`fR%*#sYOvrFS7wBlcu59nX&5#x4;lx?Tp9p zbzd_za66P>h?nmygBKoGt_1b1{Jl97AX5cl1@9i7 z0R(ic=FOiE3<4~ESKV-`WA9E=O?XGjp`Zh{aj!$koc_q;(urJ_uebKN=IY+iq8%9! zk6Jn)U6}{{?8W7GLjv}Mc2;8&D<)W>jN;cbsW<&ou2rTB6=|UW&K;Ef-Ug(QE`5A) zB{{Z&J)xLqlDq&H`)|`$OAVRxsmn@@>jMoAVj!GD8hTRrBa`!GG~yUv+4KY|$v;=f zMil-8&QEifwHvG37M9kU*Rb=lX$N5Ti;aCi6BO3he#FXv_h+P4X}{?=`wX4=MzjD3 zhbEWL_hx?(m|;ym&UWH*!po51i<`>F?*n{ZSI7PI^6F-D=<00RGkcT}kJF3zW&6ui zo8NxS438fyTv5bm-6@JCk{rbtO%MK(fN%(nc0)NuuGY$b2wD3o>j{Sfb3qA-G|-yA zzBr$I9m0h1E^(E-uKTTyZqL=k{~b^r%!IeKfz&$K_8DRYr4^s z7(a8(&t~T72v88abc*WG`TOz-OjHH8T4p5J(~)_~L>W4m)db;=`on*knLz z*ua#i>6~2l`N81v$q_o|+!tNBZU;VC&W=(>LV$Qp3$0*K7$OlYEwJ!#@p=`OlB5@E zIAZrbqzdKW*YEA{ZTQ-5>M{#2YkESbS46}akWgXUm!r$e?`-uM#ayRqY`5+9_NaC- zLfgG?)mt$iSJ=1)ddDe88P4p>;6TUd(;Vn4(xWs1N>C>uFu+z@3!&Bc9r-}>=)w3d z*zOEKHg^T&6o9Gs!Lg?VYc7X-F%IoM1NofbapJMENa1(l%v07i(!Mp3+2EYAXL^5F z`>E91okDs0cgNR)@pt3r*jvK05G6al92h}-`vkSkKW1C^7TOxm z>z%Z)3+4YEJ-rvpEje_b;FckQM`c2|IkPBzI5?QMN>#o$<8?!~B>77V9b6#B+fq0V!rF!a!hZpib8RQYcwO?uYzM+(P>&lxuma6=Ys#W-{8Zf0<-l~aBus8h0ZIQ>( zsn$79eo+QRD-s-LqPMv~_^nr(_pB$mp0KPmcCdg`Y$23$_OO?l#t3^97~-=-ns0V` z0i@Mr=HMK9i||jFHbCFD3@A#bcZ5G=w|; z;<9)3BF*-{M!zs*a@*!1a`xas}mv|8>@+sX1QdiHV*b-(7i zL#Q5}4lsOB0$%wlMHZk*T5WddVB)GDJ1wPYXkeN#heIFz2Q(a!HtFem|+Zq+aBv{5)PaHvKT>EIZXGAp8nA-KvU>b8)sdUKc+ z%n(0$YX?XUR(lpV-vn&aA;fl;cQqxS-u6fHhaga3t2_hy{hTc-7p1vr8c?KTLK^t+ z?jHUuq$dM~)nQ*Ew%X_1#o3N%qI{Cfz%w>=Sqm~#X{!4MF5*P^c-N|S;(mG1WD=6X z`$mqe3L{kQco26iz7+Lq%j5NQ)|waAwo*@n{?h%ah7pCn7b<4cHMy#}NGs9o=n^9( zM8$AxIIU!^j7aTyAW9J9u)(9{{*F<9;K;a|G)a}}ajj5_x* z%)%bP-GRq9D_utjzo{WI-|}X;q|I~3@jMaW#zU!7;)}yjoOH<1M$rPDiK}=1uv$FM zhA`gIn%6*^51U4g+(HHq%Czg{gS{(C1Fl<^U6lRt?jkTA)0x+W-(C27oke?Delv5wuF&CGZ#N3d}cALo-b;VZ%ws z|LPj|d`R@ZADzC~+AeWQOuAeK5WV*)-6)k>3O!eg7jSGObPQwP-CAx9}A3L1@0Hw zqyr<5KfIx+0}EzYdX`{@|26)I(y`OH7)^_rWJ2Wy1EXXJ}Ksg$xF(ZI!_+VfM* z_Fp)jh8_DxeB`l_2dgMM=S%CmzGLjDAHISb4_f<4(cSPGCdq3xb?<+)BG7R8{+xd_ z8eM*)Ax!X*YbZd_Y@5?an%z$c{{P2bAs zcO3p4Ja8B?xkMn~S6$TtTw}`yB-@QBx)00#>wOeLFI>M!!XV!+U7UeyY;;llo~=46 ziagsiz5UO9aJfnM<%_PTqkK1^SW6y*`!iw4|VKF1i30dQ8}`a!(BOmY=d)W|xl{63&C75v_q zfyT0BqYN3OO}-FvZErY>x45Kzf3%n$u6c@Ogz^=s9hJ04AV^k5hKLCjf!07UOHh6i zoF#>h7*U{=C>eq>L$kvwsJIyz5-R4yE6%+fesu{IAtRCrmPaW%BX>L2CV@Mj>^Z?Z z#FAu4``>8V2)gKZD)*>kc#wwC+c013*S;W(e_f5;5WZ7TKJOM}zX;r>q(4B3dFF$P z6Y1Ood^6GD&NB@pO=NuJp;Ih)!}Al(fLWF%o#`HE?%U7V)xfpNG?u({oO>)Sv#9Vp zfeK9R>Ec?-VRTX=niN`O3w$~9fts<)q-AIPr_>l^TA z)sqTV(ZL&+m0GP|cF8rs`mJWMqflc|@X(*TGnaQ?PV-$ZcGgq4+0^c=}vWfJm%Xrk^3l##J;sA+CPotZl+=m^B z=}l|X+>J8v5XLLMc>YtY%bB>UNWhEAe|or`b?UvQ%>V!VU$VeB{1Khw#%|{(tBDP` Q_>ZI@t1MIT-YDRI05vXob^rhX diff --git a/assets/sports/nba_logos/nba.png b/assets/sports/nba_logos/nba.png deleted file mode 100644 index 6738f81803de2be47897924731173a6f340c665d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23678 zcmd2@^;;D0*QQZYrCU%y>F)RlC>_!rlG5GXND2bd0s_+Av4oVw(j7}kvn;v90`IKf zf8zDJ@CRn*nKS1(ai9B~hZr>#S$rHy92683e0ez;brcj-2=eC%7Vw*ZnXXh66vOxO zGVe6KmyTDxys75{prx$N!EQRqz8efBev9_{544~&Z#SR)1+FRl=&0jd)D0*4 z|ImO9Wc{EKJ-y#ehmxbprnV1hhFizo(7*TO%l67|#lJ3mTirW$Hp3{BMMFIP-Z z%Wr9d5;Q@de_-!n3ZR404sRciP?HD9jf9ZT#7WQZS6f)4%%i2FvIYy|EXtFOi{ABJ zyFj21AULJD7=PlM;P)(K9Vjl4Pmm7~u^*Vx2j6p7oWzqbJW=83UMPeV=Dst01t_F% z3(Lw_L_|fif_2aZQLE8xXLvLO;@X2tf}KzjKXzDGKtF#(iTzfy$^W0YGbU{CKDF9} z{nfpC_jGr2@Nx(TW-ah01)?r%G&;;GbV`qvKQBpKSe;p6Mi7csI#NIvLw(K9y;)(q z-TpMyJQ%e7sQM;YffFY9LQgLB&FC!audYn*BkM#C{;sHIIAyOCGrF zdIq~RZ++Hh9enxsp1=RUD}H(fiC`fdA| zb#6YPixqG4E@~Wn%z3Cbxcf7Ib!JBm?HqbVC^$eY`RhK&D-IPMD{PAx1i6rW3Gw>Q zTNt@i_C)%|!`LZ#CXL3s6mPI|!Q3RZF-%O-I;lRUM`-7oQ|LbE7Gz~eMjhr7Kn@~g zrtS?#Yngna!u8)HE^CoC#I9Q>o;&AuX4v}&RGfm4q1!JQ^hWDi|3@?DNScv@{qpMg zRVPhHOJgc}l;57b)v(2RKScaR@zwRC1h*jtUvxs`S#8k9@0YYUx<9tium7U($vzI( zrxmx6&l|S{dv!rC=uPyOr9!$IsrFjVQE%rQgF^)~ne`E!JK@Z}h-vQPYo~;sYhwaa z6C?b|>9l8hwcjBtdg1A#gYoYbJmO|Gs7L3mEAKP8nF%5a1*ONdWp&VVj`U!-?_F&%M_3)0N-!VUqWnan#X7NDe+l6P_Rj}djfcggC0 zzUWtz)lRqEJLV+@-Z666cE#-GS}X|^N$3+bcB7>8KNKF#{b)3Wi_e)+a6Bivc&Tc_ zw7w>MwTf|{T2J(pNRKBk(hHQaG;boF`DH;pn-vih#|=N2euHD0VT= zRP0vaYr$&*`ukSk*#1=>Cl?8+VNLL%kUwP1`gHfCHCp$zTJCxZc=|TQq<(k$QNF4% z0C6$vN6Q3Z4t&Vq>w;I#7zPQe@QF?3HDP%(*T`#pvtY9s5h(rj-^>3b^763!pSDnw z&pGRW>pNPGxlC?13jP1eCe}YX6Y73vDW|zHjA}M+&j#HccBcpJmJhpbTMk9 zGeVE(y?$b5-d+FG3!3jXD{VnY&Cua3;i!eji}*S{9V`NgQ4KsxS^6l?GX9ZBj?t2b z0{a>rQnq1?qK~U82Fe0GY_bH_9$?nB_WtY~6ovFhHCE_O3}LUidg}GT_lD_-e?A;n zm+ah*+=e+J*edO*v9rcpuY-d65)$um#R(8=uGpOqfDR4^qe?62!? z*h8$o*ohMAvE*!3$u`4Uog{|3=Ads$Iym99E&dOj)w>?)oC6-LUdP!^hd;Kw;BGKc z$-8iBNu~Q`0MUDfgq@_2!=}{Y(vQaZuXC@U3G8~&=fvfgL zciccYtn8ZnSkVkGKf=xCVaU&=cOwP7PgA(7v!_^hGO}KDiQT+u87bbrhXW~ln7m;C zOzAbYUafv6mVeUb7a8f~@tvFgvP=IBzr=~GQLfbHR9x%;L+_oV+T%K&$g*xzhF zFWQ0us$;Zti)_XXrTyJ8|CGY!v|Ff_`BlbE_*xMrlHnO8^Y!}@Ii_#B&eVJ-g?bpQ8+MH%}FtLDSKYe1+wk#PyE)cx;`MZjaQi z>Njh!J8FiFQ(mSaE4OU|+FV`yeING<-{^~p6HHa~@tNny80X9ZLb%?lnwE$S@GPQ% zf`nk~o@+PGlF@#BFhPR04z->HCB?&XH-w#DMHVCO3a{ z{JW>}#hTg$$B4kk1LyAN@ZUHKpW=Ehx&z}Q1;*^9j_{65mCW1(^EXr0DnE% z%XN0JHB9enJe2QjiP)nUi9BO>3)cUkDEkw+cbX2w*rr-Jx3#d*$N=}t;#ueWP*gg) zfO*W0jXy2I?#TSJ?fu!Wye%)W^<{==eI%7hXR_XO*%K`PtupJEoepHfyT;iq z$64TM_z|vmDZLS_f3j@?$Wu=TWL*rJHLkK~p>lO*BlKd*512n7K5X`-h51Y(54qX9 zlP-4w+ahh>hur#o=R5w(cat;P3;ZE3TLgSh0Va$uvgU4bBjI%=O#r$3W3z6-JH8j> z={ISMt?5@L>tQa;u+M$ut`0QYOCnLXG-bi>QQ3%&TIya$y81VIFv?q>>HBXj#E~b? zW42No!NosV&38RDy8nbHkkWs+Pj%pB!_Nk|K1O*kexvkovZjdaJ9toH${5MOs@oqL zvC)kzr@T`BT?cG`Lt(YE`aFgUI*B zp8}4r>g_m*0*BffA#aiT(#X~#yojMw7=%F%juCs!cEUm`p3#52E+=lMO;S$2V~j8u z)L_zh187~V+cq5af~g)Ewqdh$k`f#&1LN&<1zT5l->He3*oie!V*yCyi&=p}n}q7> z<76234BXf>eS2>ZJ9}dB#1vQC4Gg>z$CMB2@U=X5Wnf%$;U$}cBq$6zm}^$Yreo#N z4TLWlqY37FXUeG0m#5sFA_@*Ge?pw@-KUHg^^}8va1o7XSQG-@olxWT#O3xG=8Fot z66FzvbC;4HHYr`Vj`9ctZpBl)E2Y&Prk(dyE;w>&?>hqOFH2S_*%~h`+0y8pXIgih zmcOM!+|9tF`&Cq?~WO98#TLaV>ny7h7S`0Em z+tqs~qtK<1hw8vQ0MT{}DJ?l<@}n9NcNd^&i}O4S%dcfp_!)|7@8+W)@hJc^y8f_kIHzo>k9TXd9fhV6D$_5`0SOMjJYf)x^b0vMAyBlgW15#EKsWK zzE6@5daz7m+vCzYG_T2fuJ_1$gLRNa3ECJst*#iUMSnrDLTcDWrB7~O^Gbq?yTUV| zR8pFaeF3ifQIrtD?m==B&^jk@^*En6Ok3iQY2kM|4q3_=nm$kGOnGUy?`IVEXiwVf-Pbe^uD} z$tJutJWN9Bog^yApw{JG6XfesOX|JJEx|FUwOMG_t1Mn#{R;49i%qs@=hvpOMgv%R zVER4ybAp@FFPNqB;iW3!!2fbGVKdsYzvRR9BtRjlu_YUEVPBh@Ad~mqTlS9&CBsJx z8r1GZs6EK+U-ne&A*>nVBh!bfpINTEpbqX8_TUrzU0R)>WH1waduL4=nvyaVcGWUju z5uKZF6;1sVa2T|qWt*F}!UJ5Aq2SMA(GQ7P9>v3g^7U0?=+)Az9HZT;m0?Lm!0{CRFp9Mg}m z_}E=8KVN)0Kk)kw=oiN8fE2F~AtD$N2*Ori&B1)RNlI`J(MCF$p@DU+=qDD8+D20k zLR}#0&7yn9DL%K$7kR$Jx$ebX-Q~UUlVE+TY(p(+FJt6$#=O+!ESbyo&iYjCftkI4 z=*@h%09Ks#27m&GzDGI$K17f_MBw(HaZSzOM}-^+Ngp!F|o) zM>61|iI0pc3}wH?P3QSLP7f@zJN?@zJBXR%O@ps7t!rU^k3AQXk0>FZ<8l5i8Z+yp z=*I8V=1b8NOuF)h(Zq;W*wBEW*5oxHdqo`2w2T0_!ZtV~G_>T!|5#Di2 zf<_~uPLHgUyy@ldfGuI@haMN5Q+l7zjy%C?xba^kmp)pV{th>JM4D8&IC9tMzn#wI z_rM~{)21}Zv_y6Ov)K2yx<8kV$?eiyodkA*0)AsqjB-_BdAj0j>~-@ZZZ4~0V(~Y9 zo&R$;iAS&SokkneG+-l&3!Epw%JbD0zS4&AAI)m(wRsLxf>ca>Q>1hOc{O6J*<*KR z-2UF2J5y_cQYFjD%c?q(2H;(dEs+h;fP!pctprZ3R>JIKTDEcAK;gm{0Qs;R7e=14S1G zGyX`VF=oBeNaPT!f8R_sNM#-kI#8-iVgy8z_jyUxho8;{1bL-+ZrXB)O!k2tM~|%; z0boOMghe8BgN5(LjU{q|O*9e*%74GN-YQqH@nAz76 z+x*O!PshCWpVvuIx11q-WTXQ52ZVOYOBp<&pUHJfXn;qRzTmV~v5dB|S^=U8Z?(yQ z^+SjV;!la*;f4Xi{v@`ZcGDUE6d1enpn~z$5*Ic*pKuyr8c_}C;6o(~?Pk0VJ0s=X z%6d%qx~{6THt*bYyeV>B-CS!uAWylasp^<3eIrs4lCfE1;}1tFpw6^=P`TRpowPnmQY~+tFbd7k1L>+kpWgCj zy|C?$A@81j>OXs08}Opd6+lWuTlg(|Cl~p+GVUIL6mhkr~h4fim8PfD}il`mhr@J ze&be=tk&5xhtixU+q>ae6rihtjwTwZOWb%5Ze*eh%!A%*Ume%}rKqfcf5gbk{BPhE z5qJ>YWEjxnuVTQbjIT{^%<~>hWcL~5Kr&gcGj*NY{O#(`dWlA2v;-y(bXM!V_u=eF z%=;M!y(`F=1yCiWhOJ*eJqF#2w_b4^h_>1I&u4YNHH*q1?FHb{$h9_%&6+)HJlO}| zHjZbc*~1mcG0uCkAcn@zqTaMB-Kw4^?{ET!H)rzIC$T7zJ_ura25arOY$K8B4dj<= zEnBn5Tq~KYJ5gNE2a2i%seIl1E1l9s^vZ1lY7PgKM1B$opfF({w7lAe41fSEZF-}4 zrVrrNx<^Tn>N2MeP}Z?|@`4J0GJH+mcaT;JcULJB#(#iD25cY%5g5e^&XaXWC3Sdn z)2J+=x}Hh4aNhcuf45QcMeF1x+`H30;VAchK?-%t_M#4Hf3nXOdcN?E6f9mE1`SJ5 zYq2tF?Hz1Dnw@rlANux_XF@yMVFhQHQ(0c0c=h76Yk-V2vb{sz*GR^0YNe4%Ss31- znJJ!k=S1MYRnlLcM_<8W6>@Q9UO{v(OO{NC=Fl1m!4Yd}w?#D~xw3%|BeDTN2 z-R-&r#I!!i-{I4eK=1b}essK((T}4ijUs%fc~2Rz^C52lCa$^9@;C_VehL6r6dpbd z$~OFnoL`o;x`R%24)SXOd1)e)8kyTT#q2WpE5DG$X3mf^TDbX$oCH+p-Y3BBc-!IW z6(b86h;Xv;9vfvFAPdJ4R&6=o__$?E{t^#Td|PivN8`0(lI_TsjB+KGd?@&FPciLC z_&r!xpp_JkG#43d8bX&vVK>;FyG@$R>=KGo02qH}+jZ+iuCcn&|gQO-GCf(_%q^l~!q`aJlbJMe;H=Qb|R?jJKaHm4( z0hcoe$Lv;SzjKg^75^63cu}b*dvdVLUAQC4*jzgm(g%OCzW& zx>V*HN4_M?jNcaP^Qxk93tk&lVW}HXu>=>r$@kDWTmJz&LZIa+DGZusaoyuI9R^LX6p zonard_FCHW1l1K=U-60>eVP)+KVvJefZTUSZ~I4rJHf^2{caN{01Ct*PfAV-acJLlt`olgxr8 zkEi2ysF=v2Z2Q{{2?Z;9JI9qgEA9Axinrjxb7)koM);q_9J`l6 zEjj+O`;HVC*5KCw#=Q&%!jJ_Uz!X#%gA(nlehiR@01TS@>Z4Hc%5-ZS{raqip?RB( zKmXGMCtr3)8MhvQ6;QuH8`n2pa81%=DFGgsi5#=IqWv{w%9J%SeZIZ)A~%K_Ktjq# z6(ss;08_${YK3>SvMeu~2t_B9e9j(`EAd9N7~b7N02JM)QyEiQJz6dYYY5x857GcD z@v|4Zko#z5ez*{PG}YPt9HVAz=ZAWK;a1XGQx*_E89UL+11~LQR87iybdg$Zun-ya z3TllU%v)Aw$(i=?7NXm6e!@&zrR)8dpm7p@6q)Qvft^WJ3fgZJ+DKYc%ee5jMWaxS z3t?|xx-JJ-Z$>JTx!i7kp3#zx{(y$z&w;G?id$@^SbbUS%6nfIIPgG16=|UNH)*i597x&@1{=aCLM`d_d#_6ZSWzwN`0oTW;G{av zwcb~>oOMp1=L-^G&$}fz7U|nCaO7P7Wn&K1_3sCd;V+DvYAmMnl)6>N@O7N`+M9uo zd9SGXXZyaT?5Ty)KCP2piZWFS(G|fo-T^sq3twJ(jXMa4= zIJJbx!wMDlh4TB)Ki?KMZ^`@-`Y}v?YrG)0yGodPJ-V%SS8VrG$!;e4Ul`ZP{nxBv zv%-H;FGNAY1>%zK9C>FxLA@!2!*^h%ixTlPtEPciS z81VI@X4eQA-)S{ndxGM4Zdf7v4%QQH$ypr}r5~mF6`+zN6*mAzylw_H{*)99e4vnN z6azTT_X9fD9QLJK^+BHaWVzgm8-6eHG2On1;-|V~C~OH`NZgAZgF2gk!tDbZXEjaS zxCcBgKOkv}3O(FYxgHHS!)YjbkC!t}$T-%z8+w}#L}0B-y9 zYx%wcHjrlypjNy<8%oB0H=hh6PLf`8h!dJPc=*T?Q%QrT!IK`~S4EW&vT^zjQ%@i4 zWF+iY`Dp2r#+$}>K0cqT(p~L1OmEUKYQxZB5qJ~yQ39NQvaawiGEBw4mp62G^wGIQ zVh2s$72i=8Je_RG#NylUluknQ+D7k>I;KZJO?6vIhVU2SiQ*bl-Ep~%)Xb21DrA65 z#)NUT%WO@)B+HhF{ZY1V#$6?iK=3+mP1ZeDNtq-|OLnCC+ZhZ8TG<5RI(kUOx*|i0 zeJ`bzfEKrxRXn9R1_LW8QKony0WT1Uuc17yv3h`Op$(=+4zhAQj*JzHOw1bvzJa;Cc{~I}=a%E|1a_#C&naf-I zkl*e#Bgs0k5=}IBAB2=fAeiR6V0oI25D;*S%0|GPDzat2ztNiSb?iR(&C`=Sp!=IQ z7p$uB6`4!*BnHhsm`fc>r2l7ng@-ns6$!%m^8{UJed)uXQ&8fVm8=)ujOO$2Yrjwc z%1w^3zu;ZkxSg8xw_s3y0?>M3X=1Ko_jX4>1J$XT6QLF)1TV71-4*Dj~6LisozyHwIv>+hBV8*MdXgN1rt=KIuhWqx@L7{#9& z(u1;BzZu#d4OqUC#C~air4{W$BXZ)Kbo3zo>X9Olu1A=cTOvLjDUc-^IkFvwxA)tC zrKgoQrb1VKKIPKEr23j%BEgw@0S!u!N$r_o}gU z$z=b2zpLFKIzO@_gs2xCqJxs(MD!i)hwqG|IT6D{@W~4`@{m43Ls|v=diFX12a;?w zbA2zP&G!2#ChOUqeOE9yq-bip-S-2+ZN(;hLAb=sO}I0L=M?M!q=H9O`(Nb=HaP0? zMC1%3WB`x}+g3Cf;bmAl=aQfcp_q=&X-_^V=V0&z(yX-#RsXpFU)SS@vtuCA2j-5h zg?SHay8H;nvIZ#epACpW9rT(%lU2%>GAbqt>$l29Gz%8&DsSYxHw+KfPkqa(TYm<@ zG(Y=tNu3*fziU)Y7M)Ny=x2AA?8roh`kN{34o!Wh3gC+rbWlYpgSFCF7M_U*ysebH z_t>v%R(r{3O&WjVN5x;=8ak1kVYwJE+`<7!`9vx21ESZ>2GJegu)C=u6K5SwIL8LwNk0TwcOV1H1Iu1tkVr8pdwaBwzDs`3vq~0>7U=ox`C}e zscS{+9Il`&P%5xyd&SlHb!LaAuME%5#EakUVaw@Ma4r$_hk^X1Cv+b`7$1P?jw8hH z-P7CTGXM7^HA6KlkPcZpS-RY|{E^;uHAS0Au_LwRdHqk;m8Nsd5_}we@Rlz^kdmwT z`Q#Ruedi<+WqYEKi5agfDTrk0n12h&ivH-acq1g8QF7=bqiXQy2ppI_QP4`Xd3jSt zq-o@X%Jidk8~GS1uKn8L2bRjaZp+da0TKlzoc%Gfs(LrKS(M)|!E}ti;zH4)fj23T4$@HGEL93M{1bol7{&2Er5c|_z=?kaE!!zH zQKkcduJ}WQ72ch`pZ8V&Jy2Hz_8{eEZHArIG##|wypJwuh9Gh}DkdVB1nT!^L9=?P z%b_2En6$O1CW@i4Ppw+HEX`q}R+y@r-=024cb^QE&82OqyPVaYM&x@Dk&k`AF(1`2 z0s6pjjuj1$Mt~@J|J>gm)k?2<|K+jKK_o{b(3bji66BxaZw>FBb{9)5Qv|;X9QGl4 zXdK`ai~Y^GYb$~T8x!ayPg?>wTa;{;@mg=?h;Ygs>=sEyWWT7FlCVysv)hiL(Sw3jYR6 zN;`ERU-8F-n%1N}HlR}6HtvPUz7+Lba?6pHdDnoOysB^h%dREc&Qs*ZxRThL&OdO_ z8rO&Uzk8*p=xN5g0Nnh-=2zkEK`hg5&uipeZH86d4)88HJHI^2LQTEx^XI}2PgU4l zd*^#cNMC%Pt@wb<$loMQY5IWqBk$XYPT;LLV>$1g2O74Z=v4EJzPIYi%g=#)LF{q) z^Jex$aALZICRw3>U03!Z&T^r5@hk!1CSpZzh%T!jvaAHcQro?3GcHdN7nhLOQI<&uDc>_e$%#b>V`8w`q2{s z1@+*^2I->%a3m7>zK-OLklAG)$1PO;_it6OLQsJ2C=gCfoqOq>J-#4&U>!Vx_|#%6 zBZ-0SX=^_}$fQGjBZ1FxCgSeCN(`I}B|+jhR9##`vBsRid8oSixBr}>6`O4nx&1rm zdsyvAq%pxFl<~*qP@GE6^2|2rqwChk(E8%AMxfS9$RcAyF0dut$1huMs!FoFr}E@0 z7cjXNyp^;X^d#%!d>637_G@1aia^A2BekPy8TSV_sV7)p4yB?(x0glVUf7N~AisB* zz{1?ycGZ|R$g!EyoG*@^tMa)ZCBTI!L)F@skgW$-S0nMyf&O>;OYQ2maUY-X2|~)N z^CzYieGduvqmi%q)=yPFf4uR)++522p-0C=(FV%a(M8;;Dw#O;cjgTsX@zF9lENH2^0!I}DZM>MyjQu50KNFnA51d#!HZf7nTn+%%L@@gu5`=LAj z{i&c}F!zE|BoHgU9OcHW7bb~%e+L@9sZu(LYZib>d<)}_On?%@C5zt5zY^gN55=H7 zL_trGKB+(42Wod%$J6^ZkrSWrwOQ~XJ`!E%5jvv7CAze5-Qu>`4eGivrS-HOFR z*OgiFoedtnUBv*qMSXpnSlcSLWc3lqg_c#Jg2(peW4E2U_$?HUTN_SgC+kEl*YKG+ zqocbiueTS<9g}}=5;zDD@I*Psg#tei;LxSvnoNV4mv?g3jg@Q8m{v-CPj5}eE#~vL z#_v8qUWyahQ6}zA`pkzl+@Ep8`~QPt;%%N}QKAjyU}sE&Oc(RXEvfQ}EDNx8*M~c& zAW5?;9H`HM@RdIlG)x0kyJQ87^$QJTpvFT-6zn7hU_&MoWqwC7UxXy3v?mzVGhp-F zef5sqQ;hJ>nOHIy^gAIjDB)tf6Xzdwq0f{0x3;l;A3FAnJJQWHI?{#cl&IZ-CJMl0 zM-SUNauW5l2IP74*@mDPqsI%&WKR3vLL#%nLLEa|L_-$4dLx|Bz>r-5s5L-06$=i0 z&d~0UE5^9we_pm__G0qoMiOBm%l_YN{Oi+MO|xSZ^GKSgZCt9wtBs&fJL^Hw2ZtO$ zOKR&cehGYkcj4g_oNU{qZoHQ5YMJ(3miugJUV|2Q} z-Ev*A665$;@U{Q-K^cYn2+JUSs~PXIc?UI)y zQ=(VD&X7&TmM6sKuvX4_*PBpTS)Jsos@Dcy%|L&-e{jw1xr7e0!y8A%z3OUVpK8GC zMOVh5K{u)ZaGL>b;bUY{4Q}7;=r0Rj$LCGmZkc=)pD2T+x~R(jQo2#|jHA!vgvGYc z+wK1`UuJstQpk@fn`H>#MAn9LgYM~?I^dl99f$NC7Jo@4-(Flz20U)r`>YQx!uy?4 zfQ3+z3|ofWyAHUipk(L0@R^S)zLd*aHK-Q)w_ix*d9Q%UkKGvSMZ`m5{XJMxdDn;# znD^lTuNZ&}C#Gcq&a(bq`tf(MS5O>)3RyT>3Sg;ErHm#R?hOZP4kpJ|-0weE=q~)i z;>HrB?Yj<;&O1L8@*XCR3VIN2gZg~~dN%o2%Ja`f?S#9Q4wA#oyEA z=TpDkaVe_@GP+$if*PYv9&PdSuD3>jUb@G}yi5Phxy`Q_m1C5I_bv>Kdr!Hrw$9DYG3&4mhH34KyvEu#TL=cYgL{09 zU0f2SP!CDI)c;^J?Y=go`N8vf?}fTWaPRRx*_G?z>fvceX{R@c^Ld|+f!0!p=Ab); z0l06+)q5W=!Xi_OCn9~^)UEZwlH-o<@K=@|5Mwk#8)5g$xt4uRb8kCC5{U4LfSRk< zzT+iQ9T-}$wXoF{MEmZ(vQ@FWG)A#tXo&*?*#srQ+gJvM>e^`+;{+o{AQK&=69=Zq zET(X9B!4Q|EY?5XnWG^X3{d0v*o_&TKxX+FO83lsp`$Q9Z~Y!EqIbrSo)t(>CJtji z{CJR==cgDR;^Jx-i7+(kSeMpU{<|2pL_d;8=8~^xJDgFXDTw(;iD5t_4d`Y2R4U{-EN2NdL@;5e7m0y)|g2m{DfWJqjqXBvb%`9(HT!Bt!5Y4wGo z0QjbXf=oaxVDi46CgZM)utA#>1?AFt=s@92*Nw=Ej3IU?Ae9AP=n%Kq4Z;!X?@iQ- zAl9e(Lym;xb?#lHe~}daHrJ`jPKJGm)5;kiFs7KLe)6rl_{(iG$k>ht`GSE8a*Mk) zf-XegpwP^kV3D!kWn=may;?bVD6%PTPSy*c)4mhh6`>;I<{*7>pjtxPf*-y_0*|C3-#^S{i}y1qrYsn;2+Nq6rXZtmir;wzh7%p#<#Dbob#~jrB#g` z{<|!yj4N5E!Ip%7-DRE+B>CJ8UPadkIIY=9e(5xc(Ro|BoS|zj0SeL|7hGui&a()d zM{*A3irq9e`R{DYZ2Z9nu-VyPIHXANb3(0edaCpOjz}!ZX$1-vI%bJJitY{(YVSah}tdpx0XZ1+N!1o??Ct`L9E+!mTyz19cH53MKv^sdd6wQDCC5Q?3c#$lg59 z+*Ahj`a{3%oc)5}zDapCi^|w#x;Y=IBJ$!+YCgC9;_SF}*XhgUD|e+UGR4bs&R1M+ znIB~SN&lYXbVc#*rWa0|U(YOFYGUVnh4&&oyL=Yepk~fKmys#x9Er|L{X9~kOuHg%j$Y-9V9t5Duf`rl`p<=gxa%-C25opY_<&GM$$Ce#3xe&&2AJK#!4 z*?7Y7J4@LfPgkREjs)qpO$|;ge9Xer$LFd3O`;6yVOrE-26i1_c3t!h?(OALqu2N1 z?rvb9`-c5{zRmQf8ti&~T^zg~a6{X4WNTgkF2Emm>e11`&-@oHC{8LG5zwA^L_cV~ zKaxF)>Ee@mSD8@AXMA4`?`UKe~_+RxZ9X%Z{oc!g;1IgkAV`0vRucpYS1l^ylA`akGf7-%} zh~l4#S~CvJmdeWHp*`;*?@Q>+LEP&Efd?Os3FaImbkib4vrU76={kAmrnv~@4$B2S zkp|)y0L0FFf;!?)*01dK=8_3GGr$*#9fU+MJ1t+DX<8ii1PqpjL zX0>?FPaeLl$|4Zsg%A69^5EnBb~bF?rVZHqM#nx!TaAVwE6~7d!}hYhw_boNZxv9u0@-B zmM}ReuUOI-8yF}n#R1B%=%HX{+(G1oy;!58s=Vw6Y-|4Mi=O5!rcbuWQZpLwClC?g ziB9aFN#eLQjXda2&&N@~8m~sa<$s*uDGiqn{PeXuiFSE5h1U5MTHc{r>XVI*O{_-l zq!B-N&ys{gcK!i165JP0T9}XDH_mP9)#Ucqr}?kF${9-;A>;b~e4Vg5Axh<@-GpF* zt@LAfNe}F9u=y;KHQg2Zg@Vy<9ZmTX7#m}1qw~Y!Oq~ir_N!xlllR$i8jR*4dVI1E zc3ivEASR&8XK?Y>f1&2`8Qmj-B^fxwl>c}NH|dSAZo+#J zN@~|rPYPyz7$y@_->1GKxUBKLH%5|GMtAKnG12XpO{Afy>8e2&l)z|*m?es}8)nzt z2>*)z;JMJN>??CTYN3_B)kMdQ(W0?yg~p#hx1NEnsXy&j{5ne@U7f*U=tl0Z3-XgViXkoNJf-BVH=S|C*R0Oz^{rVkXo z*5~47OIx8Y+GxEEqw18NwvB#!Yrqb6tQt5GeRyRmG^J&j@w_#`Jo-Iac9%D0-pwAr z|4!k&x(N-%)JtYuVBV3z>~R8Z+{30UcB9RzA%1Wt#{oFJrZ&9%DsW@*y6gqzb0mX0 zY!BEk!$x`2q>+8e>+5$IlqLV8yPqYnz3BZwZ z+C}v<+|=$1CSdl!HXhmQRI=jDECWgq0`ewvtzZgs@3ErTmc7wM>F*2G{1 zf4UR?X=G{2^POKVW9sF&#wCP1mjeEHGwGCDRL>mGjWtN_U)A^McX#7|BuW=!Ltj~LHGp^>C{cpQ$?pi~Rlxp44G7)9r zxFm(22##eOy0`t9eOZA>pg)P;*~bYi>cJilyebXD(=v$9$^+!XQk!}hMmTUj0ice_ zHj?t%4u*iwTg|c>TuVUJxj|FWHMMBL$+bKm7nsWM|GbW8w{O&rYVbzs0bLbvDGehB zt?x@^Fqe>REgM>ArYzCoGuj`$m#r;fnxS)2d`BfLv|<}}0L=(wHwOM8FzuU$D5`H@ zhb2`zSrhr?K1EBps^w@G4}Ii7)6ev1tcO4nXbEqkV%>`kE}DKCXzzjevM`UFjIf~D1V8ncqxER{npGX-^9^c+SkSDzE1g6aampUkILZ0hG-0FGKkq68kFlt>)~Bo5?@c?U^2x~d6k__MJ$0(#M_KNUx5({ za*J>ep*=~nZFuz3EO$k~XGuwuOSBurUg-3pa`IwV`!=rMzm9C>dlt9eF)LjA^<{3z z{HL!dm;9WGcpTnqztD?bPud^3|?ghaSB8PpAj!>Ao z0&=3s`W*vM{=`A=g%=c{&h1#H$kya1RKuRo-H0)Cr#!MmtPII=sLgj09M1{ihE#k= z4+sbqze+`1L_T(z0wkg-dzdz&C-|fVH(|0fD3az*RMtl8V>HkC|7$>RsxHq^(9^i z91FowpIL^LV=iV`Wsc;#&yHg6WIKX^<1_Edid|fEM=msl2}(UknJL zg8UlJM&rYP0dJZboTetBWMP4BYKW46s;n1|QY{*-4KKF&mvXaKMnhOZC2N1BS2dmc zkL6N@dw*G-2tWr&UV;VU_4!1h77poN&1Ro2kAY+B_na%VjTV?$q!@ou{6t_#G0)U1 zBgH(9s#4_U?q1@| z^E+za6g^MqA$3hwL#xCz6hg#tKPA>B2{JdEOH%W1ZE|}KXK9CqAP(pK1`j5rrb$_08>zPohPhvk-~C#%1`JLUYPfga>OEJtYYGn@ zk^Jxqbw{Zhx=tAAkJtNWw-Z%oyKYWhhlUpB=X!&-uLO(}tOt2R?#gCQCw}SliJZpt zXdGU_07XX`Nl<~RL*bV3P~@K~QgggHDFK~c4jW!r z#K5q2Zyjm!{!#FVb*PzsfPeSN$PSLhZJwV<4%J1jJp*Y03n|;@+0S^dr-cJu@7%&1 zF}NY{R*tUBa496?PEr}n>OOVZN&uXf0dOjMByjMD$IA*oa2{20m=&Var2^J3Fq7Os zg>KP4ojUdt!OBjghP97o=>fKK$y9GqkJ+6xzJrYqah{Mh@(5Qy5e7gifWAMC-3Cb0 zJ$Jwh*`MaBDkls_?r?f{>J2|I!vz;UvO6vg2qr0@{&yf3@3v+knd7&M#pAkR&nuG} zZ_fP=V7C|K%5Y$-oh)XP3*!v^%_5b+4_7IDMSjhDs_p0qPz!Yd=IL1~5AL5T9$~ z=NA(5`S}7OWdR}o*UYs)GX4KwBUWxpDVJQ*TcJ=$t|Jl2H4-7WQHtDUuG_4X+`715 zRw5*Kxf_#vVs3N4%#b@P_uG7*`}`H({l2~Sx;-A}aUSO!qb=O3(oiSf$kl1~OO{Ak zddeFvXE)I*d$txc-^-@h>d#Kk1m?3UtAf+X#cSZn#ijtSYu~wPB<^BbC#>?#?JB|5 zJOAQ@)w&`^EunaIn=>#jAxGAGR!{ypSCl=G0v(T)DUUX*+`Pg*E}M@}B?baOe{~$M zIcn2=;@EM0_xt{r>3xI+@iim{aNq+ao^$pAPz$aO1^_$H>YUH^Is+qHPasGcDOJfBC5r^BV-4iV>yugAFnLCy63#>n%v`(Zy?G00w)dQ%U+so&QbX$ ztgL(ZV~fDyeJkY}Q>@zuV?kJ4kW6#}lTdX)pQ>DwQUi!ws0H%%YZghG8rUK_3{VTX zQnhsp#p1X5cwR^polR;5C^WOT>fXw#oQZt+HR0VxHVgXm@>De%kyFUGGx5BR+>`QK zptSRIdiGV7`6Yna1RjNyN=mv858v2edlzPz>QQu|HS|FvKnCLIE3t$u_1Z|8jf0Ad5&|^jWsvzdS`NJn1vUa_w}-Y1lqz3(ju+kC4He2HN~1fU)!Jy_BdF%8q! z=-}4fo4(S_b3=Fr*c8;J?16Lbp@G>C-;3=$cT0I&Cp5={W=iWbzKcao&Ye8IlPE9M z@Cm8m~-qv>>#O7!;^*pP&<5zJgimYn~d$#6y?iFe-MaT|L{ z&K>cY?DETzPo&(!ufB9WvYrF7Ke;?s#p~{8fg4`NEy;z+S0m!G-92Vi7APK*Uo#4| zexh{hpB#Xc)&<7B&5CFWNU4bd;UpYZR=ZvKF7u!Pea&6eN@OFtDv6{vHaIwQ&a{mX z|0I5ouDc)Y_>4RW%29GI_02XFPI5^bjd3bC1d z*VD+KCWiRY-uZK|JXEi!Z9n4 zIORCXC3J#pZmoGOP+LBoK=ww|w%(e;xwe$I`oO zAHGj9fZNt{Y(HQfV>`=dRXu#c^a8r`N{D$wDPW4VFQv3@zm*=p(?(q8lNaMCIMqV! zw#Ax`6}QiMLvs}E+s9aF6gTKQovslNK-g3kA)tjd&NO=$GHzID<$An$IMI z6DGq4L*<<^M{La)t4$Kun8uAOKp7Zz6gXYg%8@JEl3K*oSo~8%7-&iXa`u0rEnBp0 z#{*NtW4~WbRL?BSHW1fxY<7qwHpjat83U`)Yep@D^N0Y32hP_bWmwpB8rx-x0#Q7I z<6@k~|8@l)%bUHw@>hKRy*8CA55?cPuQkPgW7`{o_|xxyHI&Soc05W(z;(%K6t@MbpSUl4&Tl-n0UeWz%*j-j zV^BO{d^M6_C8ufb8 z0*5DHTPmOR-)S@Q&HDinI9`dBUcOh&5QcW2U&r9@CtJH%?Mkz<^UBSb+I~{&0zVTj z3Kifdp1H=IJ-<9MDNnxt8I0!!zQuR@DVjB3ikl&(0A_ulz4)}$YjN(%em;6ivTf@7 z+tUxri)y_wuW4TEdy0Bh&L`tzH6eOen@0=8;3Or41tbvX)EC_d>Z8~)>&$iy7+|A3 z6+&n4z;fuojqAb`X0A10ptmnUBw;-|WUiP;iS+m6Dg(AS67wgFS8Y%v;VKdC-YSEY z7Lp}TPu~NIzr=Fef}ejn-lJH;>ef6gg#n)HXkxX~v_fr?g|9-E>nMH?C2L?lJ>w}# zB4`8D8S8~_DM}-km;R;#auIe{~;iaOcK|~WT>XG~~_I1aH`p5H-vHX});8k;A_9$jk_RD41 z+8;w~6-vpi|ElNb^b=iJ)|JaxX!5x1w_Qn8+T|7@u180Yln?Y;C?L1jSDD7gHs>F6 zmTw8w2gH{(WoRe|%Idod>;x`-&v=QYm2?JjvIK!67cQwP`ke)G_4Z|zH=g6N0&JQA z?o!?$-Uuh=DnvVI-ZfV*xDH2F;zFePJNW!yTJpO$?&kq)dZIsThYvwY!h z?2nzXKYIy~;8aj?Plw60Y?%|X*~yDLa=3k8aCCFhn=*uz=!fH?IpIo71LyFM24> zi17>2M!JAXnfyTPt~zfAZzSptF`LbqBxd>R_M>hSX{f}D$+FaC4GwuOK)C6bU%TY=M15}G!~MOtTjLXFYarntsVe~J+TD<_ce7{8xArV#g;D~XZ}kvgUzNx z(T2gX^lM;nu1H96I`fRj1uaQ_(T^(#$s4+Uf{m+(k2WrOH?LAIimfER9r06pv@ zaM`q)?dJpW5l=-qMX|Zcy2k@;XGdL{byW5nU4(8^D3c+E z|7VKkL9HBhkQ7^RZp8AyU~?z;basGaxjI^⋘bq#i~1GECZ22Y`m|q{PORA%u$Hy z^E&44$=KI{tp=ll&(@})^&#;E6A8pa`S6od-Jp%s1V9VX4IY&UDsx&;odff-@~omD zFTYDhqveSui>Zt2;i+{$GE-!mx@^Kz6rGwU+45F|iC~!@AC5Sp9Wq?qGUpP@uZ!<= zZ(2K7qbU>DRSecG%Vo_+#{byt6uABaQf%Ie>%`7u2jI|{)5^0SaHHt5Jy9^W zUCMo6$CPcs6R;bqolBXL3H$&Oc|fk1V=HpMVReF?E;NGWT5^-ZQ37ut~9W<3GG zRVR6~@P9b*yc8NKYMjOJBBwAs5}CRGp5Qb6^Mynu5XX4=DVI`PGip4e-QhVjm1;u&UN;8eCA#<@y7R6#&HZ2V3#5Y7uQT zkUe9j@E<&t!q1z|J7wMLHo4Kln^y*dq+qqykT(eWw} z2%}>jK=@|Z!p(#PGaLVzy;C=L#-vA~C1{cby?XYS52sO0(R8iIY;8VI6h`Z*scM+$H(+M zEjSNsV{$F#XLsw$kw=xr)SARbz8mZzabOs zs?Q?-Zb-zJ;Qk{WGnrr|YGaHTP{Aca2?i>oJB{^wN?;U;T-?5WE5xCP{8wZ|I4lhe zK3Jc|YQQmaH_3~gAUpW^>i8{Ia<#T3I`9-vHm^$#oOQW3dL)c1v)DTtw*-T$R~`9% zN8rppPq*U3_q;JICte>gsVNXkeO`POPjzGp9Q4fj#wsjm7W`uan)jsr3@aeVSlD-Z zoO5G5JLcRSCh|t^9j4-sXdEw4i{E+c-7}#V8sZlucV`DrNF{pb>Y(^`*qDnzx?gw! zY=?s?WVdGrOI9xp18{^0a|=8>D7fouFPrvXQWwYmJ}YxfdDr_*MN5~bOZs&|lLPE) zzg~#rOX0zM0eiClxg~_orTK{ZJd~~jkK&yRYJ2d=M!i2U-`zIM=#cL>{}nT^2U8?z z!kE-eckW)C7b=Rai1?Z)O*Ex64g;5ES0Y)Bh^?-)e4-8wDq;D zz+-wRn1(FDJap_JW3sa?O8YHD_2-ogkXE3BoRWarf*6!A7F)okU5a=V;f2xlCM|jHndFvuU7E%&G^A3MD>j9BW9p;Y=p6nX#TpsF!k% z?zC^_?c)eaqcB3TmN)9(!|>BV4pV6U=cmLUGSs*o!O8ZH8D)wM44uOMbVG&F=DZ+5 zq2~`9{i};l$#aCy8JU3TCh`2918bG&;aWS1w6^2rB9c>P!lXf)yPLiVxDpF!2qqp! z55RJOADU5fXu4-^jqthf8sbcWSYvbRguE!c?0&*WY~dQUHq)W9?=M)%Kj~~3Gn8aa~gLUOleiA0icoA=*T__O5l4NgSfxm3R_{dFJ?%zd<#!JvfZz38 zt;%reu_xQF*Eq~mF9$&eOa)$l-me39UBZgliY!7VN=P3mIF|qtl?T`Jb5QNCWkC&6 z1XBN9df~)L0ub94Bi!fSptON^S#z>JE=Xq^J>_csdSviXd*~WX1tgO^6Hk$^JsuQG z3AWAO27AY(UJ+}CRR$V25@(wCqsZA1ZoBi{d>w}JVwUQ2d0WHzjjuI-tAHD(icbST z=73UR{P{59JIDzQQ7qrDI{FFRO>MvH_f7um$b!2>k!=|@QgFBn)0hPca=bq!DrLI>zy8GgPArOY__E6**>h__1&@Wn3 z7w)=|T78P};bSQv1M5T8_c(tZ*jbz~5no5R&d$Ou{ zx2yChI#|I(18u;~O9>Xye2+gysa59Sufg2kDZje6^ST4O<-3gVn0$NUlcNY+>!${= z=acWh=i6S@kDm?r@ zSO13sHjbq>s=Va^>mu zel4RQ{Tn*1X|R%Qw#m$HkhwX0>r?_x2Y{&KQi<^Xx7%+XanB_maVt9da9slDXm8dC zAxo0oF)o$Pb$NnRm#z6Dfo)+i6iv)OCx4@zkh-X4&l2?eW;zeG${P8Y4MoF6 zIOnRUz4V5{`Zfutk0mdj=)z%uGhgh~&6;Tvxsx6suE48wGQ2u9q&03^Q6Mo-rp%}J=iZsp`5OBZmex3+6n#p z{Y~R%kcB#;QjAG%s6NFslFpBtIA)_BZVq{~1 zu3}%vovI@2?=>PXC2NZ{FWk05PeLY{ZA?h?_0-H!CVp z;JC>~er)#|WZ^!h|3nh0zrTB8LJdp7Peov^ptaBE{X+IWsMdW8HYHywksSzv+lnLI zTPM?9kJb^eV6|pVg{DG&8#}2=WTFoVmSirn#M=(t7tbVg0ml>h>DrB(u@qIxFv06> zM$R0Q+0B;}h5`mMV@TP^P@NMu&o6Of0j;}=(F6r)ucX+;k!+VLp z>~;3+IWu$5{izB2q#%ujL5cwf2Ztr|;r(YgIQZqK|7a-S6QA+s?{IMba5C>DRNQCx z(%sy(Cq_e-8srNX3nxruY#k4q3|jpQvKXDwcweG}e_sOMpamj5_e5_Zdsb3+xmd_D zqk7o1Z@GK_pq`9lq@}9y{rjd5rFb80m73RPhL^9|#y~Q>1TND5*WXr7K5Qok21`_b ziz|X3mGlNzSTm{gs*$T&oi~wyom7>(4=rRVP(MgZB05BNz;wHRDi*uMkH1G64_i*dJM9 zOXwKlPo^od6!#2~A>8G^o9-eBe1c41?QFArhD<23&^>+mfJjdTa{mhklQ+X%Mu9-= z2$|8r$6s%e1ymsu={qymnh*$8c|7=tx=zuX7Y17_fWdyffG<&ooFhOWUHVT~FEhX^ zs6fvBJ#7>6VWbR%_)oug>E6-;erOa0z9t{$neN{9bmfQ#{KIkml2iF72*j^E?di78 z+TKkNh>tu3q9u$}rwf_TQh|JTdid{Z3HXeW&!_cGAEL}9w$^(wG;lpjuTT&!5hz}9 zCu7eS!(a+5ub$q+hDc8mGJ&cBu~8LP$b!L?yHftU0gC1<7DFM-^TWOfCf!Tfr+J>; zxc4lC-RMCe`?k2tQs6me|91X2{ehLC>@b)v3}zX%BNkl%OL;l@0(>Em`^TucLdLaD z>2IPC80=8CmhS0tfy&F>V%UzBEr#f05(BsmYnJ(cpF_M~Kp>@-7WwuHJEVgn88Tkj-hrO>Gb)NYRA%&Mt4F$edFD=<^Z9V*TW=zYZDCS88(9~fuxQ*WOg`3 z=8l$xmFDC(d0>CiV#im>ScsbkJDGqr70o8d2>+(C@E86P0f9{X!N*0y)0YsS>QaNx zMZAQ!Ir&h#Cp_eZS*u?y43}SIw`ym{`m*%rZ&5!X1b1{|T15%m_x{K$jc=6*{54t& z->7SeDeEq@#@VqjlvZuTXCD4xSr*SMYO}1Y-DK{BAI^uR&?vt9?>l$y+dQOc3TCg5 zP=Cl%M5GkEK3oo(^6iTbK?4R|^WvZhwq|W9P5bDQh;oR&U@?}8ypX_!CkzyWg33Uq zw8Cq3%7(oztfg29!cjA{Ko`y2pqS!284&L89b&gjq{W^`xl@g;KzbU48y*F=g!qPz~J;47F&cfzi z+eHW8&&vsY_mZ+>Mh0hmMdjLXolW=7oj2Fi%1d-brnMgi!;Z~+_4HcHoU2x~bIJ~o z`V<*5OA*ARlwy(li_c=3HHZxz+X;t%$F0}&s7n5Ibj{6r$^Tk!XaQvg6M_6Un7UCg zSonf%3)&SGM}6_?oOSu}P2#S{>8?Ca?IGHzU;($rRe!4NmGewApPBOGn?OcHyD&zU z|AsTQOAS*SxiOeIa4z?<-_-4>uY%%kf8T^BiB;!hA)I;l>0LD%XbYQYir{nM*g;vd zg=u`HMqoi}Qu5wJU?1zg8CtevzdGM(x;y@ES76KGvh6m&TDyw^-xR9@h9yt~j48Y;=g6?U|)973`D~ z`MGPdu-{(@F<{F|SUk#oLb=0t*H9vei$wcxh}kw#dm?qaA%Y*LOpjdzG^$qMTCS(O z6$l{N7f1c#Gz;|YYuvG;=Ch&{N%oLIPbH;zLlTnzhDk?Cw!hh;&XNI|L*YSC5?CmD zchOba%Urxof$xuixJu)Gjl<7nqq5VtqqEYVrlLCq4e@|}X$N3=Me~qneuo-%xOWX1@#q9Bt;!&$A$h2_5=s>i0IJKcP zWuVkgDD_DHF6Z3bR_7up+1&M_ODctTI#RBmTl%lm zUGT7HQD_XYW->Cjq%GawG85Veg?4OvXl0|MCu9y%^c%Si(Z6J<%1Pve&xB`(3o*8v z&RK-FqsBFtZU3sHL{O+Mk5t!>j)|F*oxLjFx(|MUZ5i4sh2C?1BZEr)>`Ks{2zP1G zN+A6;a}C8x`6HXYovKov7RCjgW>kA;R-ee@(L^L^ynn8d*YDTg5bwy`m6hU6Ni3Qk zC_cbZqDmpygz0)0GW|I<6vbO+V-yidJ2^R-`qD6WT<)#y;=Gh_8r$UAC61{EDLj7D z!2c$7Kvhh;@w;DRZQ6XS<;~ZIJSGf8e46W1(L@fr*|KN`Gx%`?y=pXvglr~45|^e6 ziYrzQJaeo1QGNo+dTACWgn2DwV(m?OH#jJ~mNCK(Z(KC?2h;;ZTcoc-j_H5@c z_6DggV5<{pA-LHDa{cNk7Y4IK^x%R&f5tjdbOT>o#ge5oe(r zd6(s3>aQ5)os_HFh9!rq_vp|C56*WtJPjF@(}s1rWH_@O@k1bLufspHn{L;jqf(RCp{SwelvDWNkjfJ_YcwS0Eoy#L z^%eB-p`Wm>Jne!lq%pz%xz+0RO5&rso#n?;_QL5RY!tw5*vd%Fzg2cDTu_?(vI+0q{vf?~ zjbWm6+Mx@7oA2Vh%Y?D7!AatipTI z`L>?7s^|y7V2|(@A8V@S7}MaiyLfyTX<8pBnsh6>y4|#}Pv;w!%I`mjx%OON=Ub8v z-rcTE?1{w=+u5ae#N!pD%Z!|$ZmhtH@4;>K5a;0H+&gW3aN0EqGiqH;;a`WM&Z~(m zQ^jH4p(a7;@||7VmHgGUCkkylCaQZ&@uzai8WYsS^?J1Q?0#103LT}>aZN+@76l@- z*izr9sQU}EXNo$EJ(!Z!g>xGlPq%n za%?Tc_K#g5zFdS;374K{TRBH8Y5i`@i?2qP&kPzY`~f$@ozY9(5N0ROJaQdNi2J=g zuiO<$35S!ES@bmg{+QtW*M5b>m!K7b>&?Rf}|7T_Fg$_ zVm;O4f(?=5@xNzZ@0u1|EQCLODV*u(X%&l-Qf51BztmzoZe`NW+K8#Dw3@Og%rIqE zP0e8)Rzy-qs&hlPm$1OPe7wgmNfL|PWg`zLb<3;ONvo@n3+)<$!U z^dwgnsTPteb;;@2n7=_CXVoNlK=Fo9{+8L+ue`VUmfz?KhyLo<91Y$eT8n<`z#@uX zN$|iYP9oTkzBfQVLT0GCvqHKMOdoQob7YK$JNI#Lm^~4@TAsB{el}pC5gA;!c9Uz$ zTv%gr(jLM*(|Cwv5vIiTP10X9(K9ZHzh$xWce=*>ZJfotcb~x5J0Br0+s{)c2)WBb zpY6#7&>Hh#J8VBtIJ{uQwSi-ibF8Y)%<)sVHWXySAXm9pBf5(WjIuMfAVJV3_{a+A%q% zit?t`9#oBAMhHin2#Lp@Upw^69@q=@vBFlbL5E!55mlK;r4xCLFz?p5GH~bU`?!Q$ zH18VdRg(PYY}A6Hn8rERENSuN@a>N}$w(p?EG3{2U*5n@TKCp*keJ{I0c|7}z)-m~J*=Y-m>H>PjYdI%Ppn~iy9UpG+2r;~S7-PR(%^C5 zk3~Mj`u!4Y^>&68)9Q7{)TtK~n>Uy}_O~za`?i*FnucuF)$Z-->3&tCsdL*ST**Nu z<_U+Fw`8Diaz^tPn#?A->diGp=8qC&7_v-a>wm64l5HE)Ow5!Clamj!8nvbP+#Svh z{!37Kc`AQt;3goLq})!Wcbx#J%Vv!SeiFYAaoA|q0pTiD__4lrmZoxqbR$eM@>DZh z`UUG@KxN1xDn&`;{ZT@f8F|Ub4GVSlx{rOsyV;TPfnd=d_wBWbqPg^_U(iWTB6vcD ztlw<|g?c4sqrr&Ue7pgfdfeaP!;}s({CVmw!h@^soF4QMVDKkt3Y!MnFG7k~+pUpP zsx^yGQ8QlIq{kCT3jTW2(y_u(6Oryjb@7qgx-+>ANC6;oHgx_sv~- zmL<=?`M;LXgNfa30hRB7AGn|WLT)@iXJAW?ZDUQ0I<(H{tX~X?Lzktsr+4ICd+n=? zO(2jyq^(WQDKG9$Ris&b;bk{76V1JB=1!%6QB3u1_9VS~1%pv~?a#8Jr3cmTNYjX@ zbFb**olPr^2Ikb?3>Kp-2;Lc42-haOjeLHH%(W(tU1tW`ek;T%bkZxord-aE&YIpi zI>uTg+oya%2ZS>Y$0rOuKL5AO_DrNeEx@;)|NmgvceMeR52zQMQCyJT#Y82Ba&;-ey zQr{|SRtEv{YH@cx)mk)Htq(SM({V`EX!|(G(<3^k_V1LrwQ&9&wPhExEV%}Tr%3g= z;lhgzi^Avq8NGANc(i{(#X2)IQ7P@X)@4Kz`#h?4bcAA9Z))`wRJi|eewORFD##Xp zN|n(iBrf|zbMo-dYLleajE6ubN+q%&`)otdmAVprsds&bV#2=uYMHTZgEx&)2p!%F z(&dt`lr~YM)fZy(i!|dlJ}u8gEz}@Evr_oN*c&I4Q>p6pqH(`-O+AR-`v)lpP0JXV zP(0pyvu3g{^?^dw=5<0E>-mZQd}}U_BrG^1A>|~(zY6hkbSX8otclgq z?wU>4D4O>S#PW((8Z%EA(PaDt7nS|PoPa0%Ywo(ABDQb{%+O81K_E@X)a-@1bxj3x zrT;=Qa$}P|x3Y9DmsvY?ZseY#oI9>mfbL=tRTEAW1E#d&M~&ax)De?Z%dN+TMX}oS z2rN~Z-KNaUHD(p2W`yNbrKWq|-rJ@KCzJ=^qobd>m|av!%X1B?3Y$RAix!f*_^mD~ zZq4E8A?KSmR;|(DMGA#&KgJAswvJDEpObFAb2f*IG5GR61o=7Xvh0r*{)c9%#we6$ zLagBkyypzAW`sv8u{H#dahH2;)bPTcYv(SuX5EIwx={V@X9~-VM3IKz_unGka}Yak zuGd3el+`ri?7Es9A6HQ9iE4iEkPl#aE9sByrtb#!;9`c_suYt{oVa=RcmP6@L|x(D zeZIxRr<~5~n%-8iFt=oA_>tx#jj2(8kJbSFr~FA*wz7*288YXus&sUEOvA>NG<`&E znDdT!Ry=t8>I2QYd-S8z?%_F1Jv&5{ChT<84k%{5K$?OAfhs^K-QA)+`8FJDzs8>b zJA7xNV^o_)RXt5k&J6npmSqGVIi1++BnJ|i0ZL*2$Z=n~bIgu8Sw+2r6z9**P7#IE zURD0&!#H;at`>xjAT=Z7ew%>SjMH{vwC*WHbL%-^!iZeB3PAw)Bcy@&r{Hu%0Kpg`y zE*lN&q(ipU%E>TptRPldyKnUz)%~e6!R)QHb%NKz>wnwn%bfMNdOI zr7ZX@m*1|3i7?nl2T6)Umg!D2X4jbSYQYNyt82eDQ#6dOCKSy$1?W+Vr8@KuPd7d& zplcw%5UtdA66j+uU0*wNJ(@X5N4oKfBP5d0e2ern!rD^Lk4Il#UAt#ZocT(qLS6vi zCJ|3Wljakkzk6Rf`*GIAD#}Le%7UaxnS?B0*1VwDTe^A>=5mSLNQ(nGQ?MP zGD%@YGV+Rj9ii3(7{$|?MkrYTGG(3%s|ovBLF!7?w>eU7=qL*lP*RbR4#4U{ug#6kf z!PJkBhpvfW%aJF7u7~jxUS1jYWx-UYQl~wc5{p9|udT1yXx`q*grn07@}=QzJ3Nz( zpy%T8|Mo@?!8__;Z+!frE3ZQ)r|+#zK-ixY6fUR!X&<`GqDC$S0yU*issZ&g;f;7e z4?qQAaYvpIJA-LuIqW%4Dz?Fh#WCj&I{Fbkr%>RRSDeq}!fERDh*#Qtuda!Y&oA2N zWTCZ6P>H9_15`H+p}XVZ|-*`NubXQxUmo^lFF%=`_EmR?~)F(s5d1h zis^JT@3kMMDl@&Yevb=X6bb<>*1yzkSKj^3J92U{%OMzgCD-%VwO9 zSNZe8vgO3lnj5@Ylvz~vvSav*sc&i$l9CXu#bV_}OvzYeWKhLT+$|@M+{ZYQmpv`C zak7akHY41opG0sLIpVf+#?Y|NT$X4C3SyJB{^%{K53EL^T3FX)oTzih8o6H(sc~A( z`}6UB_(V`={1L@&VCjRch0BsRk>(`A`&?Zi0I?JF;=Nx^%u{$_4r{+UW`OJ%680_=HO0WgMHW4#>TVA{tt$I zx$k28*!~lxNr=ydN>T*VLTr=TdcUAk>_k{u)|>z=*LZboS%LA|9`@R+BEsG1LF&2D zTM|)@4E!#LwRnB{gbHkBt`7ha|@_zv<3CwrKX4>Aj{mHaAA%*fQ`ie@})iUeAr z!h+&kXEaWOFHbV%lMqeB-W082^!Iw#=ae{QvQ)W?JB5|6{p&rTon7T$Fu4YRL@lXP%yEnJfBIBUK|@Fo%GNd~}R-ea9k)mpUu4&wFD zqILppG;A2_T9E-H4_a>13S^_1 z&<9uHsR7D~mq37sK%MX$E)#h)Q?o2=ez?gCwtU-n;ZYLW6LMU@InXoWjvw) zg?`>jsiQnEHhNyR?00ciHUY^vgf_qOvlEmM*GS^Y-#J%($)N1{ zW0Tx?(dEf^C@xXCKR!~`WomYu`;H)9^>icCuRH(CF~#pG(28z)f}FrOYjwJ*TZ*5^ z!t!l7E|V|a=D)oMO_HggD6vltN&SfKOcTpZcLhVdWX`EbabEd@KTgYU4?e|~rug=7 zA?VG)B8n)#u0Tp2D3~Mt7HplIDp*EKb9vGwp-q=@dDfI=C_1Kbcak$(3P&jCyXRq0 z3=0<={@$gKLNloGPOpmYwKwbItt)2GdUYP1>~Ya_s^QOt5fMyvCqTGZF#op*1~02mb^oU)%Ii1<)0qaupOb^aRV();X7b&=V8|4 z1_k$ZmK$gZwjx_f2-ZAXSL;~ZSNp|8&$pHySd3l>IA#Y*KioY0f+E zvu;e;avO`yP5@U$idBx+wqUq))vKQiRp?l;2h^oQ;O31{Kwz)FDU`y0;b3{U`u69e z9hKyT|1DF;m=PEhSFihJO08FxJaS~;rPRx)DtoXxe~fu{US6E?9TldU9`_bI2{AOk zCKu8j2=IFh{FRE6?s7O~=w?q>8mZPY-|7Nw?rZY94aI7xV4o1x5s-w!?%dWZ}GtrC1}L*EJ+r9PL=aT;?*c9ugl+*#rqdq9fCWq^`6(Y9!h^)yrEcy=pQNhfC2uawc%{#|jVVsv`b@ z`TDt$a*|2}2tG~q);ph^Nr!#MX~)*Uy^*M2@kJiLi}Q4;Bmw0%T$Y%WDZ%mer(pK9N5e-#hji%!rpon6=^r`ZxJ1?Y`ewTvs~V=%hkC_te$=NsItt;1AE{ zFfo38yik?N2KLuwvxeney!d(Gn?))8ysh!0Qb zDSU6#K`gj6j^5id*GeD9J>t1s;ES9jz?0mWq}-G;-K)=lrI=Ow%MC{+Y@^_I6Ex<( z2g88GKy>is2PgT6+D4c#0A`!_-WJ#pe{{;Y?Se#pgbJ@E1YkUoN`Cf%O`tN4&3Do4 znT+7g2T>kvNuWoPq`ChYJ`-XklBb-v%AwsfG=KDcCJPmIDK{B$Sb0B;Q;bQ|>O+cL z=YE2_n)6m6^=A*3JPk|lX+>5Pim2K!rN`{z&(R-dH~Ce%ysv_Qw>R#S$ZcwG7!ut~ zDWu!3_;@95>v5-}du5NjWFsi&Yr$K?7wYHoskvyiBjz-kSq2^3^PSR%bhy?lK zNtoT)IVio9qe`DVWsk|W@B8@@4A)sDVT*Yb>^H;1$qmiq{SW|$6qViB{|3eHMk!BC?zG&ls{ zhY85mRA#s4OQ3g3c`ESzN>u0vx|iAcIB+}0r1Q?Ne#YaH3>?}vl)Xb%93Uine6Xhn zy18Jn8@o8}8j$1VZ=KxlChY3Zh}cj6?MC$eVeT0HC{>yL=5q_j!$SnK$3`1--D!BA zh~sV=n|!3uJxRS^#GAZP`|a^(cRo>O_vu*$_ad*!Gz*#JC^O!EV(!ueG(U!r>dT1R zx3}a?QP>HinAk@vCnOzOHd`6vV4p%Mgs`F34G_^Xg(v3)hNVYXsAP7<~%8RO9fQ zptP;C=c&~Z3!Co!QF*X^LyR@=+eK&XiYm6ys4h4sXoAdP&Py>H5{-ppeDLpOarAk9 z*&9=+%%-}oMnh&f!gQL#-u@oSM6y>nHr6sgd0O|@??c7of*G97-lJ0pe%P>wO{abS z`x!}HpitVaEy+1Jw>#U)EMI_YJoA`Drr0aqlj2faH0TR@OjYfeHXwcUsW>r!zmyDL z-?}?o$3Fu#J1fd*^7BGCuZNOy6_|v{?_6~~ZYPoQ3F0Tq?AIhL$%H2*q~Y0!h5fBe zq1KnDLNY%x`7d?&(}hdY-6e7Rw2#dwecFFLC*1!e zg@%-5_DRN}WLAF**3A<&vVAiZVj5o=eO@0IVt;*R09gH;1vl;e@Cy1)73o2ft&p-* zZjBWxNVr9BBpi3Ek@eb{oxcKdOAtPaLs#U=KHD20iVcp8Pdn*8W|5h3IqmsZO34nn zDr$I`G~N$x*lsJ(hr)T$VdupSG;)<>h{o=+laA3Z{wZ=mOy;%KZ_r(wUwxh7Zu8P) zVxRrq+_EM{=Tn=Fnw;UE-f?Wqi5ZTqs=K`k^!=O2g%#t?ENx1k<6VVn%RM5>UUs{b z^@MLFi4xL|EgZ48LC0}|IZVYxMm<>~V%il^=M5DF7Ce~_f2#%4tmoanBKCyWJKk^r z;Pm{t;ae|xgxFIS%EZAHhMTe6;scb+z!O64$tPJYvz4N)91uXs|}?>6*KXWSasG z;~@=#6SqTQkaRfy`4UKY!McUNDssP|D4&vu3PlVKeOaToUi3-}#q|RtRXdwd)}RhF zVW8WmjpEnLQtb7!84kyY#NGtqk&(MCG2XX$27-4L1{ za`GV>D^Kq|2dVtv}?g%egZ^Zcdb;UGRxCS*2)s(5XFh`^a;Xf&w zCzn|bo3pXuLK&V;oWsqv*<(@Ue^+VU!KQO0*7NfvlK#(+MlPPenqC zf^~T)oEz)&uLr*b(q7Bfl5d#nlC-q!)7Un;%hi%ozzgWAx9REX4X9aN{2l)&+;Cx7 zGwb2c$9d{X0(&fQycPgn6J-n(Jf89<@Yx%FR0O>F-^Eq=~_l1W864<(LyF` zF^sV`KlWEzZKe0gM%uCTjc})0GYxQYt7Wt={?9n7D)5S!mk6Z@Ld{H$!d(>@a9OEp|8hS(HKm-9v_&l zzjZY3_KSakN^6wW)y;!dtm|XfieJOGT+blSi@0eg2G5Ym6N>3I$89d^NvQhPYkhoc zdc8dXY8wc&)z&5)U7zT52BOB0QVm^Y^~q7hd<2BE4PIyI%i}n(OoJ z@z7WK#^VVojDD+@7WQ8!`GlhpP`Jf*wwb|lS5~+=!l;WtKbWf$4b2Q{J7Axc&jmdW7y_jw8K7?WHx|tkr zSEK`bDL>!JaH*W8S)(=+PLKAT^6hclOiq%hE!^>FRx#n(CxTFrj;O8G?fr*$e|_&q zSaxbJ26omNqp|CKR;MF7GgAjO{py(lt4+d4u(8cd(i(3hoFoEbvp%q5t0*O+`l(IX zj%}kK;>F?eP8p-2qc5By*?8mo@zgEb8Z4Ri=-gFrP8t#fY_ErhAub@MB34cWFO&3A z+6!+-i!N7sM0@<=Et<&)gC=Ht=^M43VOCzbz0W2R)0Tt;6x@o?g~TEDM9kDYB00pg z(OozxUX+X#RP4XOZgD?w64R^hDaQZ%?H}=j{igQYISn=8ma6cjMvhF{Ym+=twaP9j zTNz$v5MrHkxdnpp7w9NpHE#-oSOCjK2W3vlkLy(VB6VUY0;(&m`IeyRCHn7*nivpz zd+#A{bW4;TWjeVh@XoIUvnC@%Wq8q0EK9B>3m|-X$O`3RrMc=BA$tEmX*Tp=cg{-WzJfKo23*A-&0VyffWMs9}++kK?mGOgl@UWKvq%-Y<&iQj|g(3 zKbp9EYL^OrjR)>vs-E4>ky5E?VfsSyHE`W&>Dj(+m9=6^%UF^PfPk$jrH=)?^vRzg z4dsTyvgy6OA(rLka+;eSNn8rb>6J~$()hqFX6?}&QOoLb*yq|qg9tb`(a+(35esZA zH@Xz3>>roJvx(*N?P-6}u9QC~|kV8Lu+wvPG(p_eWZ`<`_I|ZhpX& zJ)>(#+`1Oj{*gP=KD(@bNNtQ2I?US(JhPe(M(_gGeZ60Dj%$>~`bh_&$sGvDm}I z@{0k=IEAv7OO3c7uzo~mB=1(KGEWwn!# zErYR9sO7?*rI$u-0mIy*q#RZ z9>6Y#e>|a0~o$IxJN!G`Jfvj?}G)V*;)w^BPwy+QFO087giE zcMO9szM{V3x}Fr=JwI#fxyENkcB=-6xqnPJRqc08ZFN0m5Tjf7}N!D++1&jK*AImA0?xD_*@2@$Ee=@h44&%R~o43 zrDelc27O4GMiQIOu+6=Hf}qQ9=~hM2g~DBW-1#)tdEam}YRa^7@oQbXF8E?+GFiB^ zaM1ImJ?P+8*szCzJ(76NGqkKR_Y1q)L2eQ3eBcn!kHEU}ZST)=Xa(ptOIA)o5W-UN zR$1Wsuh-B5z*Nnwy|@3%>}HEc%<~5O;W(o3LpZiz7g;o*I_#H&w0y7YrlLndz;tIv z*e`c}%3OcwO2Vwzq8I@LC`C>$xWsg70yb2KycxR#U4$(EVFN#GAeFCZ0_hKcu^GNr z=pg?g<;2|e7NjbBKZM0_RE5x#W8%=U(S)v;S-_RTUJFV>?k>l88*9y(@7LNr_M#(_ zMSYh^;?x0!93U6=XGL{d@Qp4LeI^t3@_^+lh(LgN2o$jAp%d1Sdi!>2%BFo9sh>R= z<@1KxDjXeu4PVW8{yb$?kuz`y@O@AC@g?V9akey2TExP1VE{LV zhM~33^^aIOt11z+2i#v>^D2`h5cZGRv74L^v8X`Sqx%^_dfT#8$nnhR!t90N&ay)V zRV*&WN6q$R?%`rS;it}E2QCLOJMi-40=k`|YKo#pCcHTk!#`JEvCKJMMe0ZNBO)d0U)>~BXj832-5RU%mbQ?!y-?NVIakT-62}%an@op6D3($W%qbM;E+dNOj5?+$7r>K?L3^B^)9N9P+A7Vk8g zxC_U=e*p&OMkLkr#_vBcScU$+81`nu3KEF3>)Ub|x6wS?tr-*~Yur2Ak^(Zz}v1B}(-mX=u1O;5+~Pm?%!uHz#;B^_QQxUWZ?16*~M(5VEi2^tD&Pz<|B6At! zE~!O1cf6pVl=Rz2jKAJBzAD|H7Y#zWt}cZ4lra&_WL@(3eZ*X%TP{dH@h@Jb(s=A; zaNhYEfWWJz`!N#3RV)(y!#mmPwVxOzN*-<(YJQb^N?F{1i7 zv|OE;S=;x`mxe${n8V(%KU@-9Ww&D!l=oINEEiU9VvfjsIHu6OHKgDbpU%WQlVicZa@}b0#olK4hVh=cKqN#)yR%&Mrkb(-phxx z)n4x9*Ag1q6W0jrGMrg?3_WNw8L#Y9>V(fl33A>a&5t3v$H3h>qN+zN$kB11hkEdQ> zu>E1Mtt!pj$gL&I&uVI(NZ=61ee;hGN*=5&e#guQ?k9*B6wE?qwqeRHFAC~T*{P1U zS~mUkW|+aj39v5-C8lM=07q&Y&f8D7GL5EaH5*>+>WAtMyAVavz2qX+VUBA8sA*~x zfzthg+xMiG%)7x=uTX}+i2@SDD-3mQhE`$3NuM!-q!|R#_lJ26w8ix=;J-Q9+(*rQ z?2Xce{^}N=xZ0N4z=E0Z?A8I!ZKNw=&rY#1sb3&66pca7p1=Hrd50e_I76KJXPp)K zLgJ8jFC6xB>`rd(uy|5f%y}?!Q0d*}f_PRQV{oe&MOmWSLCZhp+7hrY3f)}P8RFl) zsBM_Yn6ccPi$WF9Rq125OY5t4lH&g9Cw|it?7)#%1$gzyR4Qnn<+J40^G=Z<|KFZ~ zD7e4aPz>Wgp1s2&7gTDuvupZhku#PgAQD6^PFqdEB403ODI*(csRkdKbo9os9Whx< zC@PhmMB3JY{;4GU05el@a4jvHF*WOGUIR?W@g6zHV>y`QXJ^pm&ig`c1IrpfPZaX3 zBFN4(8J@{^AJ~_B(Pds#%Qp)7A(CBEtZk!~=yxZb>89JC8v$9*aoOD=uN42qs-mGB({N)` zq*}AOvcOdP2Ue!aN`DW@-2ig`@2lcTjsIgTCR8)pP#^ zd07xfe_f%$xP{#1;cL z(=T(L^FnE_8!9obub0G3KRgoikYZt*T~aKJniD*nbQ`*w5bBrDEVwG7D#qI%a#O`+ z;D)IdZcosE1R)aWy1B1la=nSbytnSufqljPC&~tH%z5Aa$%Xgo4gD_KT?>8_*44*@ zzwe9rs>F!^)*R9(-qJSSh_CSM8RcHo`%xMb!H8wLnI+=*5vFk5t#Z{|7?&5AWkohn zH1~bt;M-0zu)%ZcB7|JNPw`cFnyJ>#<^5=^>DNS69p_V0@_Nf4gYz!7Oa!f=W9Aov zg_A%-Q-5?dX6I!Hjj5CKNQV*x`1%~4E{&D3V+ALJvrT3M&ynlAXFuuQcSL#Lw+~-$ z&pTXskSSarp5|h&ch?&UPXzuDo&fNv>rvo6NA!5Ja3yKACpCq6H{={z?#9jGq=#)mK zdv{#Tx=m^#YX8Mwh75SN!fd7Cqp860w5RlwS&?8GF-1GOZA#cBw0#bEg*>bY{Ev6a zJmG(7H!Fm{cB~i!neU$rJ;VQQuxc}o|9t$|6&eCfEMw+=tje?^2(sFYWS~FTTW1w5 z{_z&1W=_@i4@ONXzb=RMf-~f4@F1;oyQ96|W$N~19$1noiZ`T?Q?#EGMISgXt?91a z3#AX}Ri}cAH6HozfUGkmdlPm}mJxjRU92Tx$&#OD3~F(<{U#)iWV&sYvMHv$f^_lP zl$+2Zi1EMN4fdI*O~^2=-Mm+1J9pZp+yPNH2o257&z9A$Im5iGEpfnN!$cDOh(nz40idfiX9aKhRA z*pa6;K3!TtvH4?E#PS6*gUg3;FWNyuvQnS&jDf}z%11Wf4hyqoPOto4Xu>O&TDbtw z*i*kz&J+Z7^6>T!JBHcFhO1%cZ-aOeE;`3O){Sl)}!m0?ZMzPg|L^bVqd{6odwABz_}$) z|2RrDu-ZY!{5ciL-ti^D*5;teSa8;q1&Dm>-!(PVW&QqpQc3|NnDyl_WoA7Kf}s9f zEE9W2=S*TPzJ>s|=9z)XfcmpwqZXi0s|}Ne0mMRbe|sLrQ6-VL9=_dibbet7v?ZX0 za}|I+GUKgrnfTKg&%6m-lIN{I4PH@Fb(5x%cXb-{!9DvJo#jV#Xj7o8@g}_hFS4cnszyTM<8p=O+YCR+nBEO)T)509NWT@B!LCCH_frB|9wO_}q0r z=v8YdyaQcNpy!_GwTG7jhypKp@o7c{3*w{TjR!?wgmi>;%v^MDNP_^YhrnTcCdlc4 zv#rDrVp32lHIOuaxPHKJrR+Q874$)AN&xvJ1wk0Mh-h@=Lq@~$s0ELm-gG9 zAi(+UtPMaV`lOV-(PMZfE4Yp(|E&cF}xmHPOH-VMKGZ;SQ$4f#LH-ZHAHHfkHC zB&0;TMWkC02?1%9mX;6!=|;LkP*MR!It8S=Q%XQex&%wayzkE}saj9=gmOFRU&#!(0$1zQP$IP8jD2Yo&S^T5TZ?tEJ!gY%AQ ziM^8>n#A|brN-o+ndF5}q7T(b-yCfCXi zHDU&dKzjz>p(W85QPxksWr2iVVdzjCT93MHgx!v>&2qd-?lmzzU2X-!{@gYDz}Sy) z(#RIGou3)9xw#I27mmEJ6g@8ub)Pv_^<|-dj)g7i_yj5mKr^R%^utl2@+P;50SW@7 z+mNjiumH`&dxS|422OFWIYHwpXJla$n6s{GWJ34v0Rm=7hvV4;>o*ke0B?~RBqVk7 zVb6CkgxElW#Mf5 z^wBdw>h&{;?~7jCDf8yf{YpV9Dz5y@K`X({ChdIZ5H^c`xA4RT))4OWy!u_@49lKw2t zOxpKt8Ame4J*mCC1g<+AJn4GGUT=gp1^e4sL%#R>+(+aKN*y@>$f6XMpc6HNJ>QCMAj$=Zz z(Q!)Zm)G0G!6)b3)@t#cP9hEhNeIp_s?T+~PT$!iwPNmC?}g_y5+|7$rexib1Jm>0PbwBCP^L*_hpk1=W4LS+2RDZLM{1w$-Po?Q@ovjBy@*J`P z=kE|Jg*zGl7RZx=AN1O6uMdeqD`Q>xn}Yh=q`1@R{b5-MANjVLY|vYfMO>7ii^KNqk2{aRdeN-j3D+-hd#;U%ACuaCDqfo~@GG5+4zV_-&$Hgdq zv#?Y#Ne$>1q|Q**(;+D12)jLr9G0n^-0-YA))0K{%a2UOxJmSO% z*#oE`F!i9jKbNIZE&J5pIMt;Jx-Qw7wNuyL+6weOZjL;`5v04T)}%b#EV)yWHgWL< zi2Bd9s|$JD=~B`xD@T#_pTDq-!RhA52jq`Sj-*mozR8(IAuGZGolq}P`CY*^w!hGs2Z)NEKvpP_uQF6loErRf#-rg{ zRyx-0Zif>1>8zoX7a>=DQ`Fe9?W*#R(?p4JgXJPyj@MHyRLQm54nYUoV_zq}gW0Sv zjzYAbAxX2;e31Ps*FAz%E%ETt_H4$D;R}On7i0HC*~CkA^-ryk6mK-m6l4Jk z$z>Jh#Xgx>+};~F;&`(hc5d)zk7+rb%>0~|BY(6pRKwy zuxp4&2|rs21=dhMRoHAdEGu(eq`N6+>2^?w9rw`7EUz z`j!C%m(u;7YP*G0-bpsLs^O7hah084=5d0~Ncwtqv^IzQ2?gxBRVk)(ml05l_Rupc z%>)EAD zfF#raobeA(wpq3-3`~S5uIjn6eh)%yh_@#622 z-_9>4OV`tVr(x#|+<16sN^|TA6=$Lvs+c9_D{- zB5FN=id}*XLHP2vCfKhzrLmo}i}09={jee(#z=QUsJDbQ$YQ&0^wmH!e)Hh_f|(6D zq6h=z%dfi_#JQ8<>2?BDF*4DBi&>0h+>~w9W?Y~`Pq`NbttzQb9K{P5Yk131(^y1=g|#ag}C0 zCbS6``WV#mDoyvMe7xEljif~;`%jGhmtM~4N_YbiW7dOwp4y7J)csrqU{!2K#>97# zP5Y&uskk#k{jUE7*Z^wld^*vBK&%&F&4HHg0OzPd#~x3x}S1%9w>k@--JSz#PN zA!$bK(19N0ww!L}{N_6BJpZ;kUIz3gf)lQG@Q%gKS*Gw-Dr!>fZ98+ucs+DGD;X4{ zK9h9;R|0yl^#?)#%idCBDnfp$5=NbPCprzH#hJFi8~5$CR=y_#q)DE9wrOlyRmOqO ze3u}pwxc=I?EdrgnX?8wYObtRjJKzG>j})vPH(?@HQ++=fBDFW!TQiuE^p24wCsWI zWx`nXVgELwv!6mR8!Z)d0y$Ep>KNw1{&G|oxyW8(-^CR=_bW4p?812?K5lbYpYr5M zXSome*dwLhLUl0-PYlrNfv>tw&Rquw0zQ2}Di8TyP51P~-K9|KYotnlb(pcn)(?lI zK_}D);U=Kgb)R{HVWZ+OYg=RikOH?2Yad=%RgMejE4O@iI@d1u7V{{>KDR>8M+xhH za+mQ@j$?9AI=A6tDyUxQJvYc|*xm-vNcwl7IT^==9U=)-uPNObgenefIq1Y3pgn$=dB_mV3r%R40yvKgmJ8hJq;||PKO0;57 z)~>di9MIss+;qkWR*Fe1^ZEn8ee`?MM;GULWA|MwWbzINHt{epB8gSBG=?m!*M?8* zqKqUE)PUOes{=?T<+qO>O}zQt9;wO8S3uiz_4mesaOjU9v@>oKySu@T-K$-!w*#Wn z$rC+~2J8_O98%k*A#lqxiG$Pbr-5akSvP1`;$x1CNo#suy&C)I`HXe_T__G1ej;G) z$)M8zTBt4Wu+8ZD?zeGQ!$$Z!P;@r>In!IL1Z`QV-(sBdI6R)#zV?t-phvXPpjaVg zWz6kUdwx4B{F(6L>B(a;TN(U!rt4Q!?tAy%$eSTSHz4`&vLM&`PeIW`MA`o!6muEh zMfc~1y(#B3!SD56z;jM`7cSpLRG4Uf5z@aGk<74He_Oid{ts|uQ|;u;_(qJmyOG5R~Z608Uo#5r&%e7P*$MG2SN;PXppF5 z4QdqTd=H{k)!NTyCi3S#^)6p_e#{(_brcoC# z{folJXTgi6-WwfY!ygA8H>s|LS@PX{3&W%HSH7xm9Qe%)|z4u-9k=tebw**qH>6cehC zA&MObFxRW?-R&t1$Phl;_Wv1pe|xLWnd12B&+wXX2#QhAz1x&DKL-6nXlj^Va-lkQ z;6LKHm`WUgQkVaruj1LIpuUo>&g&dZ=vc+&^16 zVX?SA;!$HUirLM!uHuJi63VaXeg}%RmNgzoF;cNHuhjJhBWzhGHXC!1^_wY_?}UR$ zgux7p`xzy(1{TYqOLeYiNw0wpp5%a6S)3jr)>Qcpu(qe`m-|0f=eA?e8KG_#m3*tE zQqoPtS?S+OGiQw}cM$S%OX2N_Cxe3}QETYQu8vri0DU_M(ERX`!sFS|fp6OrAQ>VM zSZ@m%n>&91?IGLM3+g(eTJ@h|MYSC#1K&hJ{0xy1%Q=+s50$tK+M27kBb@>fXIfbz z63fohnA`mQ$Lx2{z0GmCyz4{Y*%4c8Td=Uz`i|P~-CaoUhESHd*B0naE3G8=?j9Xu zAZn-F^pQvvb@%I@E5c#EIB|9IK5%Ani1(jKX?-?f)ydiaxrU%02r)M3HJcDIMp)P8Zg zWKQ!2_RhWufcu?aFUzUmL4d&qIvV?4{99dEecNi_n-g=eEDX@pI33}481AOJ|ct-UV+h&D=R)BV_ovBlR@kZ;!S z46-^?cde8NbDHeEyIOU60nBppl#f6Cj2LXDrV8^d7s5{|S@LrvGgQbvkd3SfDiYfX zQ1Qj*gP&JXexabK!XO;^r|i;z^OQlI?@PhmYSUN4&93O#ZJxBa+7%+vVhyz)pxv-i z9;d3_^^^i>xDqq6eeuy0?f*DYd5wW?m%`q9$Ww?IiK{44hcdSAP0Ot5j+0&uC zImiU(1-GB7)72G6$?9K(KpnyD&!}K#yDSi#ZOyR{X5Z=)9N723SgqbGZ z$fslNS!=~~ePPhfGC$$yZt&Q|Tku$P9}b}L?$i`#rjIUKD@Gdu!G`Qpr(=HOo!wuK zP^o~QpSvD^ax9zYdY;f5qc&1?N`aT%4M=b*QI+qgV~Q)umRYwb{&WJy1F$Q3I&#*W zlded_I}h9k#Y7i8xVq=nN+yATp8%a5df}olz@T@rN1p|C1sw$>ht3Fy7y94x#*l5E z9X|~C@R+tckJ9z(a;d(_Os6jLRvbJ3tX}yi*qD>v{$;z~8}#JyBy!K?$p9E=yaS9X zJ-?JHQX_PRC{5j7xs?3n)~{3F3b)7OI%LW|xqz$ry=&<0(UkPDZCtXu zQP-mlZL)+`@uzOoBZmqx>JOE*r-1WLo;O1`YmEVj$5#ibQlp#Q3vn)=OT1lz_0q&kjE$>5GkDyYgi55{l*zW=>s!bB3c7X*4vu_-McH{^-`vwPBG>VU3@y z*BAZ`aW|%&4vkI1DS3bIJWjK2r5(cfJGZ18HKX z`*tEYTn!}3#c#yt^rcncKpRz#*FKN{`JgoBH@T?KEsu@)R*SVZZR(hAKu@o5sbxJo zWnFVlnP`cIykY_Q*gSWwcI%5@y9rZMe%93XH@5bY*rO-ac_Q$lN{rc+f^Z3zxdNqhw_-j=0_ThSeeWcG!@5jrr*7ZlXG&)fdLUzGE$rQAF^ zF@fle=6)t(EiOvWS+f)p?%{A8h}=WtfaBLdz2BF_w>`%dgiy7HsEbh6sUk4#213gB zZF1_`q*fqiPqAl@PF4xRZ;P!%H|VyIUEtVf3hk&G@?{qo}NXkX?c@Y!(3Xv z8HI(CDPz)7sjhB^g|UoxkgN?1)|(c$BRj@4JB&Pni&}is*Bh+6iMfi(j_#AdP9R^q z?Y}#}aQsf=h9-G^l{#{?dTcKG22;n|J6M$RU?}DGemC1m813AmM82!XqRaf-w)!*H z*$mmLv%1;E&Y|tiDy-@=i}IG1`^;1C`UA0~ME7yv<>TD9u^pWlb8)=y-OtS5sHp4# z9XOSc@IahPqNr2A7*qf6U$mJwgcaLON|%~1U-kB&W|vJhMV1xf4RwT*1>;#Cn^v^6 zpy~UFG=eI|@#)HN{b`n^QjPGoWJU`cNB9SiExO;6mU^?>g6Y^4w6qt?KU++H-TOIJ zsdCs2#b(=%iIykLr~do`Uw-S@wqHld<(zNuTFm*-T56EwW|&Uuu)g33I(Q?o)e!5; zRwPA9G|YBa4v{AyOIWSWrR;RlGVNp~m}wRJ)+&j$J8WoG-TY!vHdt;$-QqGm$WtVv zG9WEY;4tTP(i17eR&vpR@9Vrv=~`J zCHSkL1}*51F^X;N`&3Gs$E1{@zgj=ugEP6KOn3B5sdN3K>`v2#!&=itR4rFioBM@p z6_)*eOMsT6Bh|TR4mNq?eEyTaRMJZKCyi}KQW3QEohw>9)7U!F-EHCcvg5t|FDMJdgfVI0RDoN z)ds<~YRZ=uRes{C&COioJB} zI~`8!A|t2%-N#8wmd0(-pTaKAy(xQ#3~Q$=u*3N6W-)@2tCMLQi~{%~h6N;g#`i*6 z2CRKlE2ek~bb=?RHyT^kS5~@@oE9-pquO-Wo8s-@t=2e@3AVo{0 zwCbx#?Kd{uwV?^JALrK0{AZe4cvYzJ;x!7s|Lx!|A2>7%@ZU+|dx}Gc@JdWW<<10_Zh2m`jGX7%;!m*Xuk4#BUegowNh)4^=OhDWx~rC&he-J2SJI!tISzo74RzgjXKO$T8ss6SAD9v{sIrxZl3*)gy$|SIy-q9~yFT%FvQBb?n?CDj-Q=Q3dP%8{HVHEj;Yn+FvSd zh!Mscoc7H7$>}ijA;w`&t3uBI{v;-&!fid_pOG(bMfV>thN*H|7d~bPBHOS_X~OpL z@_g*t{uX6UIru+?mYMOJH;YY7LSQctFWWfz_nN?=#mH6- z`=5)Zpa}D9PQTA6RQ%ts8T~0Af4o8Pze~#~j0j?o18?x(GT3rM+>ULqBq1h2tUR27 zm#`;I^8c=Cbj)bBMit=^V4EeX(sTLP^+GagU;OX&d(V#sJx_|^W`pRyqWql$#zb7) zFIBdkF2C*#ShU{}39~4BHCazCt3D#=qsc?I6UT>TZC6aDi1^t`nfC2BCAJzV6(0MN zWd!rs&MEHBxKw@9r~<25&DGfKG0e=9i1_N{cMpUqEfw+Lxojg7dDoB`*zI#gsPpR8 zRQ>BbVl{OfCQNIoBj2~P4|g>+epNd#2v8Y@(MVKG(RwFBJD2dAXeDG~|xJ!>qQ{U=` z!f9rYi*t>nV(Bnrs*Jkbo#KiTJ~-}t=_F*XTY@4BkAX76{qB={Xp?8*Qi2G$Z$@rN zeATSQH3qSC7!wTp2uyuD8=7kfn`x>VJ)Ta_{OtT1W^SOcyRN1wO^=b&4~tQ z0sfX)WhN;x(Ai$I=^l~G`g5u(^c6UGI5rv>F@x{=+uk$VSY)QYDBxq)eWx1Y@ZT{l zSQIuwhX|@lufN`r39Kkj{(nc0LZ5{N0;oTg>w*>*x^EFD5OHchi4#+Wfr(99op$V2 zsCZ$n+dwg5LbwRwE}SM)$w9N7Bz{~4#lRS!x9u;6b7hF$)+lJ5JARMQKuk`O5VN5KD1z zDdDjF`-bu$Iu@*=gEz|RRl5Be%8&Q`Pz|;I`;jBa%j==n^={1W4LXI{lU7l<|2;d2 z9)ae%69XzLHg%ci>l#!eBZfNved56Z)Sq4(S8uw*7RDcMI_hZY{0sATJo}VgB^MV1 zuG>lL3Yde-()`YDVE=m-N5O3N)}wp zMLFrPb7z@axUG0=)PRdiVyhN=&V1;9F82QYj9Ht@C_7O=3&)say!@9+Yg8=BI%;b( z=2uOpxIvlsAW4A--U}HPJqYLb=^$}Flj0BiOwR8hc`ZTX`dDJne?a;P*!6-{!rR;t zWPXPy_Q$%)^>duM$(ES)^XL$#j$>o@H_+@IN;7ro{9BwN(Lb+Xu2p?&#E>B1zyVSM z#Lu)Y6v|pPenx-@%EuN&oY^M_lsDSj4RMsT_}=F@6SlH71=>{gd)`V2JzbzETXXINdl14n%?5McMS&=?{gy;Zo5qU}i>gpM#J(M;xw;Y9ji^JM z*-i#wg^D(d>?Y!cD%N8EHC+DH0V^C7NnK2oRCuY6F<3oj3s%9J&JzWd69! z`FDul?7%c`(Co4-PZS^y>mjGY#T}QEgX9LBbR7>KWe}a9Hhh2@nUkwiMUUgZMTIxM z6uaQ8h0Qr2hag0Nf&v^f;sdZjh^KvV6Y}6Qw4-!G^C}jEEAiu@NA-FZLME|FiK45* zX^jbKQVZ{ZjLd?FLnfKTj;Gp)6S~$v|MYm!Vz=qfZN953L!6ef!!bkq*N=P14d4gt z5P5b)cCv9O;Ly49DMn)@P>CbxgGv&VxBZV#b{7OOlR28h!v(s#|E=ctQ~Kutb2ms+ zcXtVN5HB4bf|G^Iu-BvGMokX*cVZV?lt7pRODP+WmRa!GZBP9;&*I7m&O$8#z6Dk; z);!;=q8APh8{(jbJvLS$YH!Eo)ZP8)6$aNVq?IMV18`JJR`0O-@YOkcG>wZY{iB5#NsGJX?(C${kN2u9^*%M*lW0q&?RK8*c-$C*8R>Fi33SNhXA6C}k zdI>Q}`ip`F(H<>Qije*o7C0DO`XjkH8f-_up0iI(XyMUjnQy+x*VcvrOI*EO2Ixd? z54p4ov?6HQ%e_SoU;4mew^Ruw9?idtlnL1W(*;QsV_%1iTaLsw;>Wy zEPV^UwRmYIBfi&=7Qj26o%Pm~(#O*Q1*xf1mV>#@!io=xQ;%tcNx&AFfhD8K9dOjQ z={X7q*BU=s*dF!iE!Y972M>PRy$gqJoW}?~n0%p0UQaU~;Gm|ayn8SdD0W2FAs!&~ zsVg5BuLJx;JikqhUGjieyyg8@y}j-38)VBII^5f{;c%s~qi`w?xTSCPh217L`Pg3s z95h0|ucEdLGM&x}YgvQWX3K+Iw$3JsM3i)lKF zlgWmb8sXpUjIcCVMrFN-u`3mEgI2?fW@sX}C~k zhLU$uV+0V_rHY%#)X>KFclcMau(v!pj6!F`ZS2)kN>{^!%NUw| zR1u;v;oovMISqb4FuMirw7Xg8rHVQn(iqVTZaD1iGzH4K+HycsPWrNNHvntzm?R(s z={TLZT23HdK-M`1ILpK&EUV#saovA-g{W>aMay8@dRT%@YqQxh4GgbRx^``q(M{{CtQZ4 z9`IfkJX>hYsD0XQA|gB?1CQqvuX|x$0F-)#w)1%0zh>8U>)c$q5i$89_odt0Qd!#e z_-J}`^B%*7)Wm>JIUz4#`j2} zNXLrtevT1)i2f+fJ4ZQsKfO2*{&&x`Zpt4?H>>dKUkmY{V~B;~LJYxpaQhhKYSLoY zf=(%({%cu%FIG^OMsV3@JTW~?TMt+dB17%u4is-8Z$*{hSqW#`Ke$Pc-ZuYw^+MFn zIwS6w#3rrf%@$S{UHyG63tI#X(I$WY7F~Scj!Wiizhgk|=cVGY=}e)Sp9;kc z^R>q#<&Bl&!zX78a3s7PSJERRM6-|UY&1zg&tB*<8_?H??)OxVZBNmZpA~rnb$P{? z)JrlP^bw1SLL?jH7X>5yz&ZXh@Yvy*aEergcRvrvpDagxBZu&ENLuE$!}TTFoFjfj zJcSjLqCr<*N4*gSoD#ja%L~(LIWv16dCGiKR5dFVtc;2}LqG|>TN7?5X^PlFUM{l? zeeI6j!R{`U8{XOfc~bxD*Fa}y#_q29e%-2_KwOiQ4sxrjbphP>lub#SYvPxl+_KLTjx(eE&IgvuQN@f8%gca#y;!KkQQH2eiBIO z8&$K@uQp7NP-`=(h`dMz0OCSKVk!J7 z*E9Z>a)@K5(e2y2a@#;!<%_6L(w{HNcuO(Pv**qNZh+AWqmKFkqI%S}ZaK__#I8N8 z(;f$A{@bhhTVGF`ks-!!jZX-**b*ThpKN}NQdo^Ll%E`FIeVj)M-15{WK2W#KD50Ucm{%YNFLHRawozjRrTVDZlt^?91IFH;9}di zDtfXiX^|(sxEapRN3{5Xj&qP&lxnf~mGxw&^|9gf7J%AxKn&!L;=?coAWxlf|02wo zJp1^_ApDUl!Ec-FuXj4}QZb*{Zn_)_aR8HH2|-HLwsq*4n=8LY>%ZC-PQF}HM>}&; zXwYakcuY$gEXs*@EyvYj&cDda*7`!(*KLX)1Z+Fq|6syHlt?016?E$m3yVxPBGdj` z@RCmsD})X4_%v)jHm*CepAxywLGWo- zwzoZlyB3m}l|M8+)a<45E7{EEA1!}Zdx1_g)0O1gT=t9+r3%>5Q9>qe>sg6?ze~ikq^}lO7AiSP zAtnYHQJVe4{m7kr>UGpQ-l7mfeS2Aqw7)eYkem0lvLZWeBpn;8`T*CaCJRNLdN5@$;P&Jh0d_F}=LJJx-Ky9I?S;8( z&wg{Jj&#H|&gxIQ?rJr@Xb5I~=+*+90L>;XCD598(R)B{n~|oYc^&VIgoBJs_FQ#0 zffr-MqFqB*em_~{3%^ZURNVUqOc);KEV(0VM@7GRU*D|FVTKH#!8uGofsq{p|5vr) z4X6DTBN1mn9j68+A38jMqXj8fX6A1Z6P!lXEQ2RXvg_M(^egMfrlh+&jwz1gxPlB! zSVzb6qi%##(Vk9JS{$fSXw}&hI8;cC$?K$@cu$S{PW%9wN4m8)0gB(iWR!?6;D_N- z>{UdUtE*C9UEN+idG`H7j8rqR=5M~ZVq)T&sM@3L&T=>2UJU%&f`oAUfjbZ$q+j)L zdCa8R4UY^K+tJ}-lNd~vv^p6-JMt8fbUbt~Cm?W*qzK>s^VAiWWM3Bk_>mhslB+^w0y_Hfp>ReS>KO5o*^8PeC>XbBzvQxgV!PbAV$Y=TUBIyII;HdQXp30=542EWKF4m) zikRujM^CkfoPZm;&?Vq2;%9olgi8D5L>sMoMYAVlK~4S8@wcMENlnA=dk6`?ey{4= zb*)}ciq7LHGM%bU&Rq2gF*jxVb<9KL@`^ctH zLcIIPAjkamjpOqDYUq0PEy~EGpPU`l3a#NGWM%O86G)e%<1~YW7nuPxlduwDq$~5> zL$nfGfG*%ET6L9jKpxc;w^_xrd+@R_47EdGs($=*S&!w`v93S(O9A5XJAWEtd6)%j z7gqvB(7bf1I$8qE+1XuF0biDnMevpZwX7iCvtn~Ar zB(s;!kjhsjf=X@GOCJzPM&N&BBoi9uWPVdm4n(Mf1?S`%hlhd*W{(NRM!n}ryVyDk ztv>&TKlAy6)9t*O7JsMnF0O+4_bD&C^Gi_OZC+L9&uxT_tfda5@=y#0y)~k`PXG{M_8AmoHj<#PnJzUs`9j zSlOS?VeUOZ0I#1|c}zU@E%I{;VRwcgi#*s@)6W!XQxGiyZ1ysY@7miVUAGgMNk~Mn z)l8NhfuHMmtRC&Q6GRS@+lLCaS{W^1x53kbVTqN>jy6hdOx|Nkehf+FT%w?tu+UEH zel-iriDI08|DP_hv*Yc({c62Z*HXdp(rdKHrafF8{N-AwyfwtMzJh&5fH`DL@^THM zUaJy)Rhi``_&vLPLs(t?@>aD!zp)f$!@(oZeXEE+F{KV3jCypG07iRs-SVuG`sdP^ zSW<7fin-S>%F)htg2QP?K29B({YEdX(lR}T>2n9`d42<^x?Q|y8fb4b`1xa1=V3OO zH{5n`&##)1Jed^RELR6?L?Wlu^|w9dY?cddBrxaZtVg+tBQxSWU1(W1tz_Em#_|lv zJNW>Xjg6&+IGr3falc*yB+et}9>l^VhP`f!+CSJ2rZ)1&le>Tdu~lt&+eKF}E^zfj z<*55%SbL;?By7w{Z*q7>BTbJVDm0e-+*TB!vR;TMyzpx4_32O0VtcgzoA}Ye^orW(1E&cWwmkDc6UU#+0DNZC zk@zUe#l=s`1Meg$C}=r46jKRF4b?jFS&#cR6z^PE{7%i~E6|f` zS5+c7j9eK&JMri{&m7BuI^g)-N_|ue!zXg<#4GkQZ)dg)G)+VSz=fFfqB=ERQKNt? zs(-OtSR+sI`B(Bl*$r1b0rs+bOuM2E1McA$1*4k2yN^! z>uoI{ncO9tSF86Qi>yKNKr_r{C7H2Kk(!o8U$?!D6PtGwx)czCF# zl$dJXO96p@b9{6_Mm^PtiR9qlcV}Zwn2nc5c-W4c=i91#1CkBhQ==#6gopj4nDtwe z`mtE~lU83`HYG?&me;6;-(OWXkmvRG0UJP6yEY&))+Rfrlt%X9A6D1D71IuXU6aq3 zMoy}Vl$8YSs0)9ZcG(f(S085#npr7k@3)h1xmxIm68^SCNYXt~Xn222YU?QHxal}J zZuPpikKLK7`}J@$8cY}P@5#ph`cE=)?vtT(HZ%|YU7ldh(#za(=ocBkiOsV;?q?3w z4VTR|Ub;To1uL`Ei+&#oN}_jAKR#xSd_K6oOD@*fA0Ze$(OnrJhyb&J{0T^Q70MmE zDaHM%pz!8*q4H_i-a85@qISg|zqi0fQ=I*t8!veA*Jos^iRC2|C)6g7zm}rB2pL&MWha3*c>H)yc5#IXzb8orUHxm9@x1G+8wia#P}=I^%x|{(du4AhU|!?h zXO`@7!uma6D#_D@=;VDmGsd59&ad7`$PawlPc+n2KSb5-jIuFX_h*9OI-9m0Ma-6K z_R541*Tk|)=`IWkVb`ik`834~ATA@2BZK*aeeaNWL4-bI$l?#O$5!_(kEdOypKxa@ zu&rTAXX#$|xWk>`uax+uTy-Bx*U+m-C(PMI_1(7`9qvcYI=_-_tHL?KE?kfR{WF?t z|16YiRK|)GN0>;(5?S}O~9?DgZ zxthn%O&!hiNsC-g0ok%HPJ!sq2@2jI@E7v#>R9LA=`+- zP@}zg8Q`InS?Dl&M3Nxn@d4{k)$pgL1H@w68e;8X=-K3u!F=$-`Q4rJbmo!0Hfi7U zC`?OVOW#;C+2EwtT8NosM%R}?Z`Y4q%v&P*QdNk;(5d|zWdN(mYx3kaEbaqn%NHB?04V?bjZ2n> zy^uX0pMjw3hqKdtbKNp@9q-0GTf2v{xKh48RK<4B=Z58_6;RuWoe-?ZtcFg|i1`u{ zk1kI_$uGN|F9~`N)rOPY$@Rzg;VWG3p?pP2@D(PO7G%v3N2@Lh-Kl79nq>QqRDo+E zs$UuorzmPV%EJA8{t?W7c|9<1%Lg^BAI*QvKq%m81g`00!+Ri}iH31vz_;30w5SLr z-O~F|h#Z8NUtxXp6&w3T<*mb0_GV!11J_AU9srQ}kUio~BdWimx**zmq_smGFmF z-feaxgqlf#vl1_od%@?&KDIE3=pzNGB1o<95QqaO8%766xV{sE1cwY)=#7wdiUo~5 zBTuEV$**z?;&3*NfqEaYh_f?J>pO$7H?-8%A#^6mLv<|s1nkp2Im;g`g_)_D@&3Oo zU`{pS?g9;`m!!^;Ze3jRXU-Up(VI!&WC>OGW}qba)e%d76as^^@)fg6n93^(G=34% zZ9d+`QT}_AH2sYm`OIt~{!qO1jr@4$CH*UqSc=3pg)=Dm+z^!W=mnZFY`IX~y=aME ztGBCY21B8kf`lu}dtO5GU2P2n?ZhP&!iju3R-g*>=aYuxA1N&0IXc38V7TAI{9jLINZy*=F6LD3-*)PZ`gyutQlp>ViS$a3tZH$wm z@sb4aUz|LZbxf15e*U-vD7;8QgKX8W6BZenr4fngtPUZp6DjIH(g+!1G)7gl$D}~e zNh4t<}?&^#&o(DSGFryq1 zr~tdv?5Mo>4F-Clel`R9Ks1cWv2O;0tQfiDmHpq*DC$D z8<3M^B%h?>ck#S}lg9(XhS9-weN5->$A0mnxgpS`UH*L4fk-d~-ODB4in^B}=~y{G zV_Tna{*6X?OhwOI6=w9781~AjWJ2&|EsEKLo4`{+q>i8>5Rg;c+^0BRQ5_-P$SfNU z4Fy$cvOBmZgm%OO^H~Lo+0THi`m$)YRaUObaa9V7Wa?CNxFdDwHu-{*yC<6DkWn1} z{cxD}0op>&qoJ#(y9;)nwCOgwrD#K+IUq`yV((~RL9+&3N#LVOy}ChpL}dhu;9%CG zHD82-e3d&*#1u@#Lku!xGmW^zz+YcB)1`I?Lo;+Zl1RGbE~2yKP8Q-n^cM#y8O`Q~ zzmTH{OsOz`4f-!z%fPD@Ow^fNksLBz{B!yT(6u0Az3}TBc^K6#B`Y;V zKxkcFV|s*h@}TfTd696T4(RR76k0WEm&J|47p&=>RZ z8)(+KI_p`gx9e*J0TKuikCwkeMYQrlzv|pTY>N8SkII0W{0aLQ5G0FAdO07`me^Sq zshCFx)2g8iSOYaSpo2cv`Y`oEQpnFr*IKzr8FDxiHR!8K?&qxG>kcm9W~10>Q|RP7 zgOw}x>>+??H3{_ODSB6`m{3y3EI`?$4KZ5GZkG%+ycqd@^k?wqs^<~wh$Cke==?+1 z9dAmee0cmlU0UyMGw{0rJG`@dd%0+`caK9>Kt)$Nqsu|RUmygfQ6C*#2jUUqOo|e} zKEZU6)rUOaRFRa2N?eV{Fh^ z^g8OL;sc?Rb9XTIeP30B?t>IZ(ASQI70ZL^*k$9i$b5Eo*@#;^(J9ahp~VT;%h)sD%IkQ>j=l3pJy9O!=nB}c1v_f{RmB%$Q|-~5u@l|E!1!z}+6HRYAYSG*%lI5Ii0&g9jI}A(Ld=HMHo~|C6E@Y8 zTGsq*eVb5&U+W?W7EIrN=hAxd7NV@Z*W0B6C6ce7oTwh@|Exq6L@0~{dE1OzTOcJ{ z1L7!h79L$8h5*Wy2nLX&E0A%diIB{(=DU39;XjI{WR5J0vRSnK6QaOk~}5vfqOtZ<$*w-m0Zv>37AkhT{m5+|9sC#B+y3E?5; z#8}i)BHYD}_cJ$c%vS4{O6N6wX-tZ9YcmZ&^-C)R691_?45IzI9WeELhe%+2p|E2DJi zslrxp=Mv5mhuIBLq${NSQug&PvjoJ=zlQK2 z(wEh`^f{8;c4QFpf-kl&WpN}&`|&WPqhn}n{2Aj>tj6B%mC=Uq^FF3_CIPJ%PhUGp z7M}$F-Obu>AZWwRTDCvDKw(^Z?eta4TT7%WB2l0no2_bhAs2t#qdIZ4>~uW7vP9$w1Cp& zDAKem-?Onc)mNqU{yV$l1_Jv#1_IutpEXB`b`L@nh&IJWnJ9)`0$WS%>Zqiu_1a`) zKg2~f@3ed~HqH7Vk|UN;ivHnKoW%1cGu?*$SGnZOD6>dcI8=l0;OuxYz3Xq)r@dn3 z{=hjLyXC-qjyn9XLRoK2O2La)mMwR}YZc8Sq(|UQc~r)yieY}fpM>DmLpb{8HUIcKa=HB7te?y<$nVv7OwE=(!S-%YIBLG*n*P~Rka@8T|LG0 z4EoiSTRj5@#xf?j2F9aEGB}D1ETS+1kf#ka^ja~p+<)IXTFxEg=*b-rjz`ftGr3jg zD{p{)M804j=$cebZ+oO)l+v-RVL=)H&Dtz$K}`hX8QZv?ZpUD$MR)F+fQq^tXsj0$ zSf8A0G1n#H^-)H3jFIZKEoh$_A1Gl?#hqk*<{!10>ao49Os<$Z9vYag>i(2$qe4zf z_*BM0?EdqGrwZ?xT%NGjNd6-G692@|lkd_nV-;S_?3^K7uiQ+2gY?X|qIs_Pq4p=s zwmhWIW%Tcm)BV0Re`;Rw&oI=ip4BhndMxRPy`+A)!NO_OnSA?*nq|dI8kgbqNMgSE zD(~jsN1RNg;aw(woyy`qi}`=a+4&~8iH|n!l@rEv-RNhkKP9hVG&er7;h+V#Gb%2= z`9rxBI_?wW=^rhBmW=PaKED0*GJV%ocivO}r$Ng643p@JzB^8 z^t3H;?6A1S1X%IbR|1ePM8yt!3#izN=6}3r#f#A>{hJ}VF|PN$B)Wnx*lqAb+c-04 z0)No9{=t2hOw6CZ{H#YgH521KAV5#{=hrSgzRJk^@7+4Y{lKEABv zzG?Bm)>Zh-bzbD|f9}LSls_XS-am+Xa|L7;aV2t_6h~6+AcSIIwxGo8t#B#iO>i-* zJt#Ut+Q`SPRtk)Ga{4c+?0pRwO~b-&tp|tW(BUTb%oLHcIL85KPcI` zkm(^r!6>xO#26t_?~{E0(>8z%Du`_1sCZU z;##Rx_;i;pRtUJk=zM(nGz>YN!AoQhP-O0}Lg7NUxpMIe%NgUMCSZjI{_poxlI-v$rKh6a&t4@P zN=4}xt#bB^^7aQMuP7-^)4dZM4#=)3p)59wN6h-pj9_4F6T6#Rpz6y%Iup3TWyTjw z2XrmOgH4zDdW!zy0y>TjAirP|^heZVM7ApSL^QkHi;#vD5Od={4MV5bQbELsE+aO7 z=e#&T|D?o^6!-#@gHSn`W><-JC;``!)1EmbXIf{$L&L_*4t=+Is_07ZibYViW_2!9 zyuaE65vvs4r`xzPK16L($N5gZGyFU=)II`jO0kM-sM6OrjKewKC$%u*1SCaKmxK&f zWUu;?@roQjqNIreyK-7fzQ6dTZ9oDn;1+A6N~cO_+SulHP1d%Mq0?1vvq80^-;m0(Qk!=;d#x35Ph z&nm;xmxXV6bkEpui)Z&b1!eRIetkXEUwknF9hp9qQw-=y*R*&E1c85lsgZvXmhHf~ zDK5qVJU^9pgV-DV4QgxqJ5uzE6r5bX6EsXSGVOPU0xKv4NKSe?1+CyQ*Frt2F6$%r z-o)+_2ZI;$j7Ko|*Z`joah0;{oBexiq1esm+@i0jPTs_F(gqwq;EwBqp2CI+gy?AV zN%SwZ!nojR;ryKgB~jWZv*Y!P^v^&}%E7HYcfOm~b}b{qZ?IV_jgkoHL@gAd%1qP8 z(dgcFwN&cfxyJR{Q~r6A^?2}c6*Ul7;Q1$oRUg%Iv3J_T$4dT+i$U_99m6~!b+J9K z(_rQCwAhNJ4V~Sr)~T|Gw*Pah@_dA8UD#JDFR_2>&dO;w%EG&VALYIVC`p~}zDt^n zP2Cp~xU%K6gZwms9$Db7?JK7?YVwZ!Evx7{&1h)&n`9}Wo$qlhSrm_{g?G+P{%WU+ zlNZ~^I!o}WhY~I0$@S1uo)nZunGhS^T=@jUu)r=qs-ajysEq$v&fvGU!sx@b*`UuO zgt)h{bFU_Pz&D;A&M-}#gW69}m0)d<1ZL*516k$Gzui->j<6*Z@_OV2=YYrH!A@oP zdc=p@D72c<8D_@j0m+QK{^vIFl_^+3Am2T-jt%k9KPFU2MUY6auaUBEVHxQ6TNY=af9MH+_ojp2t$@kNv>;*j$RaVnrQ`<2C=7+pcxvGk!?@oF#* zapi-(p)+uH?m2gECVy!t?JaE~_{5l&>2boX|B`K%k4fwwH6BHNcXxy!jD@#^RkRe- z{1ocq#YL=(!abbPM+urV5L0KZAkui{y%C0xz~V5Yz=PmW;j7_v5a6E9Sli#z*tXQ$ zX}gDIuINfAEr;CY(zoKsvJ|KEdV8v4f4^E=`yHqTsq67TbLy1IZxw=swb5p+)A$2v zWzx9K*es*XAUr}*mv@7f?niidU~s?TufbDG_9A(I#%y2B)2V-kvDuK(X|&cHjqtG)9DuVNs2j97>jZ$ZF4;PSlZk;`Mks`Lt93-Om;2cf)jO zY<_|BihZYOkG@-h*}aE1(Sm)C*v47%Sq50Pxm@`H{t>k#cn-1Q;WRGVLwK-#lop&j z$3U+o%)0)^z9h#68;kn}`%gpZOkzcTdzVG=afa@f8cuRaTVkVWbP&9@ZwUZq1fooQ zz|qmNQ$&18v97*h$ZoLEj$vg~Q=^@E;^5#X<-e1xcDsvtZd3mx_I#UO>-8au5Rntn zB(V;H)zTYT0MW(nv)wFD*pzrEJncdf^%#C=&C5IgQXzFJx*)rC!&pze>O% zC$=&0hJmEJHQ?l~JpUJRAFm z3T}2Sl7?ZxZ|G}MCqtaA8ufGwzBPS`$%q;1PQmPe4MF1sg-?#D_3~P_nD}*J=|=@o!^0@tIu>LLwzS7Kx>fFA8+K9%~KN^$V(+8*Sz`vp_NA) zo^&TwN8)~wFFbBCB&p;4=GNSlN(zgY75&DPU|Bs9-w?FQowT z0X;ckaC`R{+I=t1gJ|hc4Sa>{g!q|R_r;(92enI5CGY=yG96yLakv>`546I!&HO5O zciH?;IlJ>A7}Et{nBELC6XI?8v*JSll`l_#<`BT z(yU3g0OJ>b9Xm~KvUI&T@!!c7_U^@M#oQ-S9o8p*59gs?#Wn7iNy^<0TZte+$22QgP)&& z&t8J(+u}B@YmASSlcXwIR|cHBN<$H?aU5BJ(xDbfs;7c9fbBYGYoLdUh+7u0=IV31 z4fl71`1u!2IAP+5@g);b6Ef+>fRG>{at0xi-gqQw4@9o(3m-yVjQIJ}lw$Cs5QfyM z!rjHwe_g68a^vI82d`=MLB7;Wg6pY}^pBV1Wn_B8q;vg}v^Pa3Ls}r|xJ%~%Q3H5| z(@MEIUpmw&`p8?Wo1QbEKOKKnuN{7G?xxhIGReVf5$$ z1NKe7KjZtZYrD2<*Sq(5p7WgNocrA8eiIDzG$}}#N%8RTD6~LoMtFDt+*bhJ9b()^ zMigEO9v&D^OHIY(_2OPZa1M)UR@<88bJhpr{75_VbWg4KF%I`hJl4UwUnryzH#){wfI3ho_~KyucWyk|4AtbSWR&N$ahQN zZt|QPAV}Op=@E!I`Tk)o3cwXqSZ|?EY)2pS5kOKmy8LX#^|G=-lxoI zdxKa@t5?R2mMCgB39|%`IZ_r)y2~8mms@`c-^1=Qvs5-s+t0Md zkZ~$jrx149AP?|e`tS1uqZRkza5ON+^#cri+=;dauH=<8s^gJ4qpZm zWHSFp0w=iz4+>(M&|Uy9wThg-izLV=lb;+1d^}x@QL|v&**A6r6Twm`UtP0r5njjq zoh<@?4)57kB%-1@rT6aWPs@+|iKXOki4ozjRn-7IL9V8+tO>6KhV4G5^?K%-)u(I_ zA=sk~LzXgCmi^Z#YGok`a|EdX=Mph|+*C91eduXrlBrIfJS0fPp99((g~(;8Jh{a- zu=itcX{K9XnlpC|J|&Adi5?O;40kAun(`eA6>FhB;!NachNL;vw)=rcS!U!A$jjB> zGPU%{Bw;4+j zX@BX4I|7uqks!m*r6!9PSfU%@Liza+YpSqOrZU#LOzjie@|f{o^>YB*nkX`eNa3ml z&d3`EGL#pr;hK4ryrL~634X4=jYdO(82|H@E0*}F4w@mtk95T*iI38L*~F?^<B3=Kq40HcOv5hYM3ax zNMHBgSV!&>6cW~p_PdO`ITkY^2oa{IO6kn!Q-_IAP91hY|4!Wt;*J-|BF120?r;ny zPko;-;uId-bF#;Nb1{HDnj{m7zq;s^11I5o_7wjc#28Gn6}p3RgPF+U#IM)8&W!t}~^{R`QJGm^zUNqxg-Zs!Q({ONY*<{9M{ zGMa-aLzteSVXg636jj8QYN*2a?#RLo!4nnlPTYDX*>}=h)2+iIIDRfQun^$;Jmd=A(&uU6*>3+1JW7TkslG8SC4p8ex%xiPSM)l7$)ZBH zBP*Myp*s&?y9WnD9t$1&pOW(=86&QmiqNC-zvFw(7Y6eCN+@go4#_7Lg*{le+s4Di zp-^8A#i;UPstO^5Ge|>)S4BG#gkQxS$bTboJ4eu<0Gseo7scO?LlB0M=2Yr|C#fKo z_HzJz*8d94@ZRGyX7v~b2czd%<$@#Jc3by@hSK@`8Y0fz&zbFqW$50Z{ddpo4Kl+o zLdfVhz!};O(gOtED$O_DN79E1&IYWYJIk?Z=c($Ywc$W*d>`>AjPtJ`3-p94A}RJa z2AMHX4F|V24#Sxqu6z;E4fGAH0g4~U>*uB5oK`yI!M^rreKwF#-Weh=2E<*x0L+NG zpBBBgaC0500IEdngF+2jQS<7Wn#jy2kb1yo`4q6z6El1Gjf7qN$14&3lM}fpOhHTy zZp-;e0`V-sXd^{+R<17z2{%u6Z12jCmIOMQ6EE9AfT!ZT;9>7(3Osi^yRh_6Dk=qx?r-^*}b_~#qH`$X^Vd=vsAvQhoC}Vo0`Dl*0L-=_m)ud65muJfQ^OC>9Xm3i zu_O=@8;UTzYQt@PbV2^oeWe%&@>?(4v5jLV@6dtiS0Y_^x0QKm@=S)!-1{6y3Y&N5 z8mNY^yCz`!AgAYKHGfkL=N*C=`d_Q6m!L^)#Bco5J@(k<=Ft3Gv~Al#hfT8DTyw?B zAQwd!VXnwTQHzn1SFL*83vU!MJ>C7 zlpZnk*OiqXn_X!R$$t)ebxW+Q`ass382{($W&NiGnl#f~M| z|LW*W9cdMTpKu*aj&N=RjLX^I-YKqNpX>Okp!6u;w+aH0u-M5D;trS6IrzxP0qZDL*Nu8IB_9 z&%ErYiee`ip}WZCOef+GZ>YX`Qtv~u=X>zPGA@9}=p8#T^d^ja2=8>@B9ibz)QD#p zifGwG5svOu(LJ#(0Ke2{6)oQdfULD6J4&-ir+VLw8ZOWe_%4J%t%STgIzlTo46pII z11%yo!^6Y7f>t_yXXp!^4UZCn+L6D5h#6pp^nrNvjyY-Sq>&9mCst~pU#M{{p;#8~ zo6p(Yu|CEHFW0goI>Z}YNsU;#UMT(dazF+)Kx1xUOaIX2FEp7~^&v=6>7aV7>B!iZ zEQyfiIqr?Woe2v71OsgGBN9$AsEpL`z4dD86vo&ehs2bVvJf{RUf^&>Rm9swS-iy3W_;As()Zn;S)thF{ro7Ng`@!eg@k|!rnyvPlSzQo2y3md zfre@gxq!`=qNBC!m(K~=o;oPCy%@Yu`gu2gd2x5-?jW8fXYAm3X?n}cC=sWJhmwdo zFV@pI>p{WH1}6Ih>CC{Z*0$bm5%RK;x@6nQU8gO~gNz|HeT6fVX1AG#YZgm+xGf<`*Z8#%ID2M^!f>wOt@3PO9 zwYVNuo-yotx?+rN=sRR;#)(rum8qXv|hk0RxDTVUp-ygbq-lr)AI&c!31(?x;{Z{FaV*OnVvWs}xOJ z5$x!S0&FYJ7UlBF zAcT;`w^8_mJ*&s>+Z@S${C+;WsF?JtfP8SN#ogI=d$OVu7qKe6cqo>2E+vx+U=2W)F_$W_%vssK2*L2(|E54vCGGGhp_DTnv!V?IiiCm(alZGs zTKL(6iC^ebSVgv3maPm@H{MF)YwjTl_kQvkH0cC?gGzn9;oBO(Zu?+1s-j4kl!M$b zDb{Zdr=99~)TQ#j!h0RVGK|7Hj3ToKrT(d?=*^mqI}O_(i_`An)3z zAy_3D&|HoxX=+ixyPN3<`A&SJVkvfLMkcvUQjY@h2<5eTO?Xo1O;_vRc<*n0bEmn95Z>M%Q-oMaCG#&V}8U)-J7hRV-llzm!{)gELm&L5v z)m41{40X;{%CqbVYiEB~V(XN;oD={z%A|>_YEuS&V=ABbb80^!Oej|RhR}BHh$4Z$ zsKCb2^P<&B1-imW4!El*(z=?Z1z{DtgI;| ze&m7(u`_V3-PF29io2Cz#89?(fbRHLpAq-%^;9f^&y%E6T;1HX3qp3Povz>BeVH4+ z#G#HZJ6(VAHz+{cZf#XBo6m*R{65zc*SE#1&8*%HOSx5_3o9?!%3F?#=!)2jNl8_E zuCxQe_K-$A>&Grdg;ww1t=xyT71&P|DWb*>Y{JN^ISq)j1$~GP8wLdvL#_%{X*$EV z_hC|L9(r8g(iKAGqT%oZdE3 zheqYI2Al5h0ode0up54o1AvXV70>x+uv-^Tdrq>Ng%p$hY}t=*F)E~ASp+T=-uFn% zh#Y2$Y$n5LDB@tDf*djLTM3Y7>k5_7Hv}2b=--b!bckam5yU*)Vh!vmuCC#VkZHpe zm1qfpcB{fyOsNL~^or}Dm6ry$4K43CH=UZ!V_Vvt3*S5#M4ogn$QFiHMwGDkicbC{ z2aBe^k=dB)RY&3pR)2FxJ>{yK<511LvLAH@H$@4XxeFtyCwH{D#5sRtVzN8Xo}X6HgOX zZOTw3U~;}t3fk~p{VI0$?dfA8#AA?e5wm61nlm2&1-QMN5QJ_)_k$+T6);+NPkF+% z;8mGwZ%vDZb9KBf`in?VxLQH|SmHW&oq|#vh-~2eq!0-f#y91*4xqIyx(r)=`!eAA zSAEp0&x8NPk{N5Vh}3LhR80CWU7Kcmtg@p;Y1t#xfe9?BTbWJW*PZIbzTtU`Y#$w= zzdM(pt}y>Cxj^r<7s*vz%xlQBQr#>90>6Nv^o2RxvZIoK`LT+!-H~{5Rv%h;$`s> zFCGHSDfR1T4D(>MoL&gM5Sow6G*!)6DOn9)tV5PoqH!l`D!(KEu;H$&O&>J?5y3fO zVO{hu%VaSQs_Q$|q4SgD=zwy|Y*b(`Re%=nRcC zpHIG>beEBQox>zC3ZD-o-!~QS|L&N7`eu+?@qhR8(d^<2#D9vA#5!2`bboQh?UyNF zUK>Xs{*Qa&WqxY2lgYvl*kROByyVrR!OXQ=rbe0ak@38`7JUG+=*@22%{F&hT|8>=^e{`g?zZXo$<7&Ri->ozP%kK>zBo)3^NV+dERt|bqG;wo{>IBhuo!zc>@5jcmM+~BNc1VZkpW+|hclXS$XZnXJA_xbc)Ty7c zWqFcBB#|#OC%)5|kii>)f=<;!8Up8wBoRMA;jj>g#tgStm^fs9ofuR|$#eoM^ei;( z)j_H(vW^)Y#(ek&A?R$^#vMqK#rEXIlm1!_9(PR+!`oX2%?Ir*3lIn7`D8D7_HR<_ zmBnl6NpX&qR4k*M^L{6&KsmBAnIZ=l^GtfoE*i;Rkq_b@%8!eSi@yzC_KjQV3AePX zljxT3?P)o8MVMQQ{>6==*E#P_+lA(CoLCPZJABnmC-J#YYu-Ri(?~P1H_J(L`!0f| z(pAk%ge>8D-_B*Im^dbc=SzsfVFcp4>Jtk|fRbc?nB<-4EMFGWvZZkzq=(fG9853z z*8S0BT<_$&zzL?4Xn!LMSs)`AS(qUcX_7fr;9|1kK) zk%dE7czA9M$`IimKh1Gh6cpN*los`B|9MEmEHF@?YoeeJ6VuG71^_n1cCQ3Q+}m&}cEYhtF4Q z-gFh7ZSA`uQnqMetab4{SEoOz^yBL}>)>2CoU)gZM8QXLqCn`dg#Ryuc~Zjmnu>OR zPd5mc=0$wmqm|kyw+Z~qFzw?Ly**o6pBbU=_f)v*%eIU5O7CsI5?ckDhip=>j=qf^;;=Ot*BLv4LnlNXd`t0XuB+Y-R< zK6o=?bs(fX3w0Df{i#o8_959K5@wq?otdwX-2rQ3){3AqoNc^JgxeV$ccP_J^zjvm zyY|X|By}*dgnER3mOf0Kws3Lxq(F6@z579J2=ux|6;|yx3O@u`*TUq zm(Y)Uruj6P^kbmTv9h%PMnZUoJDL(v-DC-^UMi*2H(A!@%M$FVY6CU#%E*~{0Z73rV?1r+6#I=PGC z2kpq&dr9gU-$mWsyw2wuZ0AnCr}J(&C`Xa`T~`mIYz_aS7VUy$SPSgUsgT4Vwf=*Cf4s!F`-6aH=2lOQz3*hN~CEX&xJ-5idA6B%ntq zq*=O^EwkR#+po2kKf=!4v4g2D^l;4gH(Omybw6L3)A!dqym|A^z z2?ncm>HE!DBM&^~?&TO*mSu`^&3zre47-vFCg7QBqP-ASLt#C#P?p`Dt5jUT0dl*Y zo!t7!JrBwn}=D6x2IJ~ODQQS#o0($aOh&0*Z!k_=l?q}rrA;c(`d&U zQ5eZMmawCJ+@?3L= z`Op~=2nBWme$-B$P;9%S*r~c8*O{+!#17KTy7yeXv0rW-`xrz4ic~~Svc$N?%eyMf zxZdRWQ|^!=!Kn&wTg*Rrg*W?8J8|UZ{;V7K$q||mpA0&uEkE1afNP<=l`wj3YvHUH zw&`PG=Wz_>k~`spr^sF>-cDJ$vLxXi9yd?VF`mx~{l23g4a-Jjs1$!Td98E=ungD; z?+&Uao}e^$3;zX*LH`FnEn*#1A&Pk8Kzl{aFO&J=hs0{aJxb<0VoyX;*7reI-5^=U3n;fd&H|65wc6)54RK+^-@SO&H!)OjeoG*(0thz^{bZMZM z*L;WU{W$}*Ei*xo%CBc{GW=_eNfz41{%B=%+WNtti8_j7;zAg5-oM}6qKdpw?ZQ!2 z^xY%iSLH&E6@|tBTsR;%3dE5j=npy|?qM1@64Sb31C?KQpW6831;zambiQtA_=0b( zR@ynvWgc||`rh12;>8RFj0L}YQQcFr1hiF32T(o$I%|eg`Y6q zGF}8DQbB5+ANn@~%6F$J08ehIx|GDy?p$>|l&w}r%ggW2y7bFbSF2vI67X#!#aXq_>OWGt`~sfnuPI{f{UD6SWC*5Bf`4gkWZF2IZ%oGyXNaWcqnUZ~027^dj&f-`Bnsz+$q< zC9?%p1tvVvl5vjH=zT6G~}w`64gNWWTb9{ z@zZsWY}zGIpX*DpWVs^3U^P#r6ibJ;EaQ`_vdgzTDUjGCFHa-?B?A}fhVMHLaDVAm zmFeX!@A+*Rzlx1gnYWiOS6nYRc(dCS-_zN-R2X{jCUz);&-uS6;#yW}4>HaYUy7jP zl&(4N^fl&_^T#)ln+?F#(r$4bqsc zf4EkYk7*xCdj&hnEXrBVn-FClYGv7l?9r{fR5GKPn=Gzb06Ad&%7xaA${$T9#H-cZ zKB3s8eoel;i`7N+sE9Iea@A=y$U?^Jly?2|{Rui&WUDt1>!_>MUo#7QJP0T0e5y_xwtWH}*9@Y9X9KSahKc$G<8up2G z22LQX@})$$tO#&Y+u{Vzmub(?lNEr<;-Rg95^l{m>+H8VOYQ!qt)r`B5h+#w(xxiJ z9-4VMDoQ-zg(%YM*!tnhp1CdO5k8y`?Q|6-$Wkw7k4Wc*75xvLQ;y^hS2^(?W}f^04@Akm(#6iN@t13y z!QObbgkG9F_ht^H?Uylcq_`-6d5uTCSZ7({A(J2Qw7pNqq5Hn@?F={#zwXz1xlvU1 zQ&Cdm6c%Su4H~@h zQVqHV{$LoYL_9&2`wH>X2L_u?X=AfzU3(kC`sVI*pG|*DSGKgw*fu%oj<2UGy>Fcj zam;^T09(O1y(81BQwILN{I-cu8leN_-W4YQ&n=SM@tYMeDrxaCzXUyJaO48;y>(2p zl8y#3nRQaL*cka?C_~!QU)}Jd#dsYJn|1QeG1v=xolfMNOX(DDch5a?1;Fcmru8?4 zIEt&grS%8Ur^4fhgLAX4f23CcvF6E3X2b3y`NL5Q&sxj59!lC_le~SqSt~^g(E_p;(6o9CP$W!p=p8#eIc0-`;wKtqeLMBx<6Qq zH{BCqsE2mf>*VQu3WN8dE|;$zM33}8`sFDG<9XplP{>D>A2tLMBY(M#Oo85p)M9Q2 zWK?TNpb!6m_1ry8>D3G1M*?5RhX9kaivjfNXYj@b)h~4B&IspoOgQg(TUI>ecQ%}9 z`5WS%>>Q`zx)Igc)x#Srs-eQJ31D*(u4CbsVj&^GWmNtc6wqnH6Kcag9u~P~Uly8G z82dZIU3^?I*Pt5F-Di=fDym%Kk4NC{`5>|M0T1_!5JyV0PA$iEG+v~#tiD2&12sw6 z5>#c!P&))G64&Hx<~~-a5Nebvf9->YYO-k=Q7SVrnEUK~PCj8|S-PDsOLf4!rb=45 zn#{%fmm1Wh*|c`ElPplg5tdO!3Z&2v%75H-@?c(wCSx|9?Dc|D!{#rRb< zoCfenR05>d7~7k?{GRJ0rneSzH94|`dcLjWbVty4_m31v^MgW+%Y{9}wRP6!Hv}ug zV(~2~-TgWc_S$`)Z$*I==P5miS!7@Q9_m&3$Iy^yV$a3y4^(C~6{gK=9~v06KqC0byyG+E01#O+4dXOJe-c9l_ zSKwYF+ird?l4jwX9a9mJ{ivS(s1ZUqp15m>HC>CCPMf#~)#L5fYl^UmxCg;@AB0EX zU&#C1$1FBeI*I!TPHrPrS>Dx_67;W#KKoreuN=u@%yUrJeykRuGA82-+|k`q-oCUI~Poc zwq8{3>}p7|HZ~=^Fv4nY(6ETaZ2xGXLX{7Ww|CCiA*LS?Dbq}CefC~{_3Zj&+H&`>Z){FothBe?5;XsW!?*5!_vWJhU zneERB^nvyf$!b2^ z?~H8lI-#PUW>BYT;<0{mDYFl<`SdaDd zON@N_VYeEycFY#M1wj));~HvDlyj6mmm#)lfl;e2tF+gtt*|GWUUzP%$1`8#^%wsa z@40ESEaE@($G`QIJ4_SVEgcwUk!t!E zK~-DpsfF~gn{T?~;sieT>w$zuGIBlUFJjjz3skbMUm16~%o5qG ziht`A=TAsLX)|ppVBb4kT~SVfCF9Dys13atX5A~8HVau}qT>M8rg8x*49O|B6II;k zY~In=H8}b3s*qUTFA4Qx^G|_MMRlgV`aNG~+_`3Pb{^`T<)ins?L}w4DwgR`0htp) z2^FccKXi2A)qZTIv^~4IMcSYXnAgg9fYg`WF844pj>4asp8nHd-c;$lJ;`TuTG;Lq z6zW0|O?rd)q2Jj51S~#@6xKwb4 zrl_7!haVQiAGFVsuPA^di^WZy;H!`&Smr} zsfT#KNAs+g&)MDf^3(sGFJ}x8G_=h2umjyBGqnIxGI6LA*o~Lxtl;dTe0qx+#dS(} z!PkBQ<7H=2{OSQ$fA^N;a8a(vUJ1)#j?8z7`^sF`dC;e;EueF+MzYr!jCqRcml~AM zxcf|W{L9ZqXcy}0O%)fR?!bAJUorrRI0*>#(^cG$CwA9a^CSe4f|8l~*ugH{&_R^|pZ4GD- zLH!ZlmQHy_3APozRTw9W;!jzhgjHPN0S>$)Sf7e>@7t_VT78FE4$hJal&k)dk#cGWaHqL%%ot>SI6%Kbrmr*IvYZy*ju9&&Dj(CDh1=NAZ#2KaJ zj1M?7+{Sn8J+Vvzw>pb>Z-71$k#f_Y7N@ymOHb1HXY`z1s4C6_ao{chI?r~lYq9^7 z>x#(Ta{_4E_fpoG*){LN0En1`?wlx6Z!vWnrvqAwm~+ zEPvP!8h`29O)0UHM!wPldpvm9t74h=lIg<}$ZuoY05@@`(fI%tKL2M5<2hIFt}a@a zyvUZ8mPTZxboU++rx^UR{e#)#XKAMaGMD2v=Kqsz(a`TxO~ zK^?VH6XAV(Yb@}B%9)yP=#?`}w>&6u_dboxbXaileqfyG$B~K$ig{(=8Zer``6uT4 zj`nAwi&nA!Qj0^<4Ppib^whrKGfWHwml@7j6cY(dM=BiZp=D+uEOiF_X5;9js=|;D zp17_ZuOYbygfmMUN*aBbB6k1@4e6M2ob@83Rko<#eg0ScN}P=+y~Jv^A&U**e0;|C zZ2pcOzPI0;fvG+ELG0i&2F{6%pFm3ma;nQw18$;GJevfUn+9dlXEoi7-UK~PkW0bI zH%^J2^=?iL+EaCm+&?Oa%9X!rW)Bsosech`&`1ok)jIAYC(&UUcp&&L2%f(X+wPC7 zEaTVyb0t4s*51BQYtb&+_UcEXPA+<7FMp=Og+ck>#IlA!i5(4F*qA}shyAvoXu zGL+mev5TByuo)f;B7B5m!OtRkPBIQ`7Y<6Y|(1OMfPQdpIqGs-|z_P&@T1mk$ zWLVdKK+zVjw<+q0Z&QQ%U$8-mw>e+PET)A(9zk^`+R0Z98mgj%0oc0%(wA@5xpMWn zfwkKvy&2EJ4k$ez%0K{SQ^qA+dE-1r1TN6szh6?t9a}GvOb>`V>bg8h5_5*@@MaMm zhOdcBNZ<$s>q6V6kid39K)oR7MGBUO=5T~T5a+WDhS5O z4rXR@rBg~t*j~dnN1XBdxnmf!%W_TrCG3udn2O^(?OiDFF0oe6X=&GxoZ+_wx9dT=jC%-4u#s3uMiYQoUkmhq0j zC#*wE6vW8~g^vUD+{!vDgRJuar4hX?mk)US-w>)c?t6STdt%6>e!?giU#VB{A*D15 z6E|yNRGo?IYJYk3qL#nbrd#%TzrZJd=bv_no>Y~$q^ za*jpi;|Xk=G$4du>Dd^rHfMkmOUS9Zr?R=-^f4Ll_+z;o7{1@n_YWbDv1IDGpBr++ zi)P6T1Q+mpq??dL`B1{Xt5~d6q&(7WcfMv)lbuNp5SnaRh!lk%_7Z>nCR3p)A}zt6 zP?vgez-D1-nKZy28Ld?{6=7r!zPbvb8}Pk2~09jt%_?Q`EmOhtAJEvXQw6cUP5F?#34= z4wJ#S2@+=n!h)l`Iar@jR-9Lm66gLY=Zeg< zL6KwAjV+W|R;^0Xq+T+;J3)@8R_WxXyD$`2RKzmcm|0qVe$}pzG6&LRygw6;Ch)C!3 z{MCIYF4{yX9MU`io0CU8GD}k9=_LqbpAcYAd1>^d-@|8jQkJjSHEqsG%!s@3r$VUM z@=05q!$!Dj!He>*XyS5>PcQN(2}*Ov+sFNie{@#X)J(NDx6(5TS-FVjLW)c@i~IXE z=m;aeIt-|JT{WVaED<&@E)Dmd$^n>ea z8MVweF62dXDz*KHCKslGa5ZsIKqw#u=0aRmGyZL!^cp79O~vt8JcC1nYwuA(nd~A^ zzU8oIce%+g(=yxn>sl2=By_7A@kZJ()nTVTc=tpCw6st?ozfpEbx4QIQ3Ox(}aS{WuXC1v{Li~;D?uV_U! zC`rLv5cy_DUxU3g(6T=VZ|HS?-1LWKaw~|#)5;7;7^i?BR}KFe+E(3BR9wN!Q*j-h zvOWFo{qk1{>0#?B0-i_y#9UL_oo6!nTur<&CmUX#t9ZB`IyalDXW*GRGWx!4@NP5W zJW)M9(+{SS*=Tm!*&*Av;J>LM<}?c8J@a6;AKjWL(=9RJ7YaSrTKao_w><+;NZ;~D zX~VpKRbssOqM(MI$fsQE8}sjUu5SQCyV3d@rPvc%al9gpFZUj;6eS(tz4`CvjpPjb z=ydkynP@d8Mm%R3XFFLt>+$mvewty~vYjwBsT)b24_90~fS;SIoz*4o2L!j|Va$J( zg_f}tsa%aj6LQ{x#OGVg9ejoUCIesHLzjz)67O#6Kpv8~< zqN0i2YEmN!<%zBz9V$2wKdaVmn|2C%`7|`o)!SPtSEr-#@awvcv9U3iIiCCD1&>(> zQ0#$4XOQQ`@~7uLEv?H6Z0m|iav&i<)!*WiJEVXu=~Sd>D(*84fs#o7b+*{MbO(A` zik_T5M!Rni`tbo}7;vAzRH?wqAM&4u5<#%g1I0|Ph9oc<^~%x2!CHs={Jb`&(fbT2 z1il1qt!Ly0rc6GmC2x`8a1}e$b+%JD$d7WPmRom{G4inTBc77@>OCz-;@II&EQDvX zIqT+n_?(J1t6oAN&v#B~&iX_*{`yGNA$~3(7&Rranfab&$}&p-t1X(vC(<5`G)@ze z1)wEM2bbF2O<#Yf(WG}CFU%0_=Q$p=aM#5(vTaRG85K@OrzUVJkzpcy`Es_8V>0+e z^ncHu%$}+b0~gYKno4y~5%H)$r($-oJQ?l}g)14;$@|Q4Okm?EUxCe>m4!*z)SKZk z5?ZZFp;`rEIaQojtJ&N%!(z}achaFj`s!m7)A196@e^%U2RLDU05P_68D$&YteFo6 zHB3$)m`q1o3bU(n)#Tx{a=z8iey)Jjg4wuYir>vSCy8An+XH<*Tb5OEssBj!@_aYb zQ>Ed8)2_+HS7H_7zayW+qjD1m$KBq9dj(_8s?x}nz0KzfKngP=cV$v5YnrC0`WkWj z91CGnkj*l9jMvX3=KEVMV;o6_Q;O2J9~%|pRjXEO5+Ayna#T1aKT@yOiss-*S^wtMo|`EAH3Nfr-3PPN!-x_B1B3tk+=eUYux}CqlUkx>vu+y^&se-$wc^r zP-@moW0HM))uIyz6v(G#K<$CjEBlZA z3q-=E@%3DWWrS5%hFcwjmFj?Z&FPxVVgVV{cI7wvjhyfw>-%&;>la zYLj8F<*%QSblB6QL#f_N-EeyK@B)A}OmkuG4D_$F93jdez!u6I?p)JBE%UvXLjq4L zU8P)!T{F=xZ*{nug6EY&5I+|dFe$6Qj^7KlLd>*Y=R-hEAf^k;F0KJb&e^3~*gx(z^=E%}s7+?#n-L+@G z;ZIe(0)?}m;c-|`D!v>2^Rv{hqVf)o)t87TfvC!U=xS!~kEFS{H|faby1vFCJ5xg^ zdHdMt6jP-t&vXo$KxV|VT|9=*vp)1wT(6&TwyjDhM!y-CgMDndxv(L@ocNG>yoGaK zeem}oS)A)lSA##fHC033s35zX<8rtSL}652o0GB%#~iyS2D3Xp7FBP`6TTtAMPKxn z)@*|Sy{b+!I{G^&p;y~gJCPSL>_Pf@lHJl?%NIKIyk(oDeLwSG|4s<>4mru|3_`6I zgx+2vzl2NUkAc<`o5@Zg)0YC;6r~b)+K*)`G&xm2!*>Umlfj*7gV>A&n|M%KC1(fr z8LeVtFegn@5o@K&T*B5fcC~mPANYMgyQE`YqzPk3s?U7DJ6qECsVg%tbjEHNy@ z$E%H_&Qpp@LSC0#q5XNq+>Sx84w}Pnqzxa!sJolRT13u}m9X)#Fk$KT$Jz?Eggo_)_@1g~-V<>u~9wLMbJFp4Xv zh>y6S19^@ZeD6RX$)RiCJ6UX9u`KR3vF0`%vz0ScgoD?b0MMreIe!!_1?i*5@+9x2 z$lXfIz5MU(_k4Z#$12pX{gH4-bfJPm&5OF@4GfSX3kjP-zUO0ACNtLqQ`dfB{RQC=Ef@Qgu1WOl8dOPiMN z{(Sj9rE~N1TbAy0z67|IR8P;U($!Yk{%oD4zi_SjIwNUpj~hf3RDycO_3Cng|X>jMMANNvfY=n%b{Ig*nf_DHtX!a4-&tkFHg#bj4KKIbG{SE8q$k6jpj__Kq;#y zE;2U0>?3Y46Fhe`y*yATd^>0pEKLGx!pA}B;l?WbZOpYS;s%zqwdSyLC&4T79#qSu zl>f`L(}^n2T-Gb>ZY0BY&gYW<&|d*&(c};}CD_l&^WKF2^8vY!?tWb`(zXEfA zXvd-nnpjri?q06Zy5k|)FTX<$w?06j+22=~!0#^M%fB1_bU}qAObO_VjI8xfDGGq^ zFgp_xc_mYBoVTh*$#V^K&|UP1Z04y@FW(zDV*Avfi}HDg%H_!?YMi|qS3u{pN`&Z) zM_-Z|^`Q6`m@}V{1LcA|LVo>czy69I-PP!nynS$RfTBk_@mvJ381bM0GTAf+ntw?j zpr5;Y?b5Um-YQsUN&z@gfzD<-cQuH_v1%2QWn%|dUX_u4<*F5|K$G)4m1`YRQ05t@ z!|U;&;ETETz89=f4yZFS5IPU5%Jm5w2ZH&9goZda_-C@KIT&?9Ke`mi)+9`=9~^mt zZid=cY&yl*mNm|H(_SICGj6+1F2merhVT8#m+}A4yveB|eUtz^O$cF@cqskryZY%4p)mzd~w(x2m+L~i%Sfs>k?;n%igR%BKOk}VpP zkJH;$fVxGhlapCo0IRoKw*$`+m^hHbi80Tb!nzMXSM_Zi&E*eSo-tJCxW2@ET%0;r zjy^YCVQ*?c^%a2j<>Rf#J5ts#Qv1;uIOja#R%qj#Mxm23bza6?pG$&&Eztec2I(>L zRO&;1Xa9*-D>pUX{{H^AMMfKbmdNF!U*%dElZKfdIs%DY##61sfufJ}e#4=b0|@V`KUA{!UaK=<^|Ma8a8aNeO;J$EVX$Z|#ZO&IPHp zUjqjxt9r={XTTDZ7~>kh@_c{X!)v65+1zz>q02p!#Ul-C<5#kbpvzpbtH@~3qowL@ z$;|s{cP9s)+ff}$dav$zFLXTab(!FjbDREXGO4G`GEi8r5h-z~?GU~0;e5*Og-p5gKO|j+LzD0K9WBx!-5?T5cZ#H>gn%F*pAo|7 zMj8}}5u+Q0QNloCfFPkrDGVis#9-1L5~KHf{r>&}#`E6iKKGt`&bhGiaP~7MdLu;3 z7nNyNym$?E%=&GMQbc4zI11A{%X2H|oA1?Y=@xWTL);U7eyg{fe81KQi|C$8T}16d z$2}=yUKUBZK*J&zcS{zR)9Xb!HE2DmUlbccg=qDoex}E$~2VAeEhWR_=^ohCxz|K zNTK{#p=wwVqqeWtWh&<>?d{hzaO%7S!)k{W`+FVhA(eCG}~WiGITl9wd;hboruCvswRXGs3Zh|oQub1 z;3%NdT%{t>c(J5_8t_%=-!<`1UnA1xr~f8Nxt~*pR@@ltlbDk0b%%dlYi;EvC z)@rX4H|Po^jorCpWBRqXmP}2LUp#(5gw?`4 zCQWI%r!iK2-5x6pLWKBXn7d`NEz3NGbAbCTkRJR>77qia>=4wJyBXREO)rSfK`bs3j#wh>GS<3jg za76Uve%(VS(=mm(C?dY~t^Y#wGzhVG^e*Qm??{d8t_Yj^$zuZiZ)PD42$Kr4O?^dz z0Tz|9=1NX+<=c(=K_yLy6ul=#-CX9w02Sglt9~IYC_%LNPY?PU6Xbe8b7DV#G;b6B<`R05 z6d%l&QHc*kouhfL79!xg2iAaqQ9w&jauAaYz$5O@q?}-Uy9m^f!7%NDM#_w+ltk2+0nP(T2qRYPQV$QUM= z8GpR4Ug%m9sWgGX6d&6_VTNne%=6aEMFSNm;ondu`dXrQ&|4}3Y$25Nf7H1Wp+;H) z>e=p%)63NLVbMM9DXt7tMkZ;2WW%y31;ok)p76kAGE(xT*`*#9jS|8YL~Gf*@|ymx z*_&D^`tH7`dY9-!AL3IAYV&zlX=4w=Z%?RO8+xt!7fSW*#!hN0y8EkQXEMfr(cwYh zFHF^344yg``(A>QqR**VPGP<0PNIlOU}|pwuHb7o)D>Bw**GT}-0^VuRhy2LVxK2F zA8dXZ+VPTjN4!M2ygSj5ydGM$ad`+}LH^zmbGFG=&ks9D$Btt4l?W~s34N`stXcq0 z?-CJi(c$xUedkWbT>~@a;Jp!RcB`_x{shC+Su2lB6kK1_P(*J;KMDk8Ikv_a`wiSY za-|&cgJmTy3ZzC88)G zl@%g`Xy{MGyE;-NwF_7De$M`$HqnaTFW#q|?;Bou7Mxb{$VTVpo9bAfr}s!Zau$2~ z)n%Q-FbLHf(xx^y4{7mtGNAr?vpsd}#|_3M{bq(?R-qrq8*JyVG3}-A7kBN8DX$GD z9}e*aiaL}_*~Q#!5pzgFEXPLW7dh-B&sR^fQN$eBTff8AxOeA2Nr!LMIKR2v?f8ms78)w4%?EGH%qJukhvwGmt3b-{H^1YS(uNEh0L&u1N zePm&uTU_x}&dt&n`;Xwn+ZfUbMLn|LGEPI&kmAN?!+||iA^cKTk0(Bm6*YKaDvEz) z`;uC)Rm%^^SVNjo&xA&K^|)WW&_-d?PNgrDfqqWRZ!yj(JL$6P??bU$u@aPhA$iKq z46tV$R(!18mBi?<4UU@5=zS3G)1_L`SbqjMJrE*+#M@xJujP~mk*M8IUm~}s_S%d! zgU6JLH5QlcF?)*!+bzepbw4&P&bdl4!3et2ZU^xlMW~mkfpzxc*}d*Rk2skMobiw| zzi;U(ic_HhLsQCx_BJ;ix6z1x;6{D81Uw{KXjG3s_IP1t2T&;EI%M_`rFEO&T(S9q z3Vll_v)kQH+iAEGNPlFnsE5d~Ppbl9<8Lik`-88^L@eMs|M%ufF zGrMr(?v&p3?|pOUTBF4(o*!4-nSkIq$_H8z{Edc$0UqC?#rnF3;coU212!#0V zQiwBhhpxaj0PbAkzd7?vXRt~9caozdJ<^Yyi<)QOis zJCYK1vXB<=Zfmll2B<+p&Gf#^CZ{h9-QzFVGW(!ZjU8L~+MR{P=p{)Ec4q}X@GX$O z8S}$cjzc084+P=GHA#Zm?h*uU&VYoG6)~W+xX7~+6!v90TKT8ivVP!8l6u{>+DWV5 zXRQwqi1TEzj_S+e{Sa5oq}#;S?KgyRPo3vXU6oCpo59aMh*uo}XAlxk?1XQom^mv_QXP@j5E+iVx?bLrQ_$bB) zHyP5KrCd3XN6*T^tMKeGne)8r))V;#+OK;^x|0GJhs#+dDw@KuGT69Yqp6SJ@lC_S z`6JZ!4VFxOgFE8**NG#`E2M-|+X|p*2Ii#JkY!Hgpq0|7$iP2)o%>2F+XToQAHw%eZm7b*Q=fCgE6*UbYGvI zT;;p8kp#)B2#6;`#>^`UpZd9b?#?u6Fv2r#!bN9?peHmD6=s*$jb8g*@7Sn6N(w&z z9O64orgp~bzg1I(W(UU#CAoVwNDN>v)ikiWK!tSJ5x8BZ%M}T~TnO7o8syv`_L^&b zU?;R<#Hby~F_?Xu%SJ%rl%b-mELU@ecq`LgKy+4%LQ9bt%!|X^ex-)_UaxI+0Chm+&U<*mLnRutP7TSebV0W%?aubHWb?l09rN>`OXMkoHi5(-g;kLmo+$G>O_mWw;O}Z z&XtHO{I-=K2+lV{>|*|5nsc;~29jnx_@3TTmQU1xn5xNz- z>d10R8F!hZuk*j&AvWjiAuk(P_5Smon6D>XqlQi~Z1Azdn*balTgQ)yn+b zN5c*Rfv`avgI@MeK_%f6Zm~S?s;a8m{RdH3%LZH&x%Z#Zs~=-EH-Wr-!O2-1X=PPq zCB%4I8Y)Q}#9KT9)DdJaR_KTcd^yqpFAkiZIKJ;o2U#^|;*FP9*+d#2?1nl17ttCo zCe2vuHm`%eb65%Yj(r1Bw-fZd1Q}#?CvwfVml-yjelr;tps0(rO3->qMDIH#1EKOF zKw0N6QQ~v?&3Y>xuS)pabg$`#pJ&ID27PG$K$%`1gX3pdtUafk$;rtnNf!Sk5QRZb zJG+c3Nfo2uaSEw#ovP$uw@Kh;1di7C_ZGV5Y#YP1*E3Djh5^ZemleV$d$tk$c5$J5 zdVh2Ca9-kbU*p|iepdEC5Re2K;W2H5czrxYq(NUu5HAd1`O)q3_h#zVq8!VSfe*x` zz2bUUh6_dzf1R!2#!H30Q3~Z1<52AugcX0yxaNEI8ly8#AMTH{EX^33+Tu9U(mLg{ zHHE>wNAOmDWdNhBF&Mf|klCPHf4;A?s`32~V3S9%ZYXbg?#j&cz^15kYpZrwalI%L z$jlz!ZEP8KACbk@vVZjDm{?e_w+3dYe}*dzmhY8>#NH9LB(`nyIsrx%tHA*QXO1jf zNzxyw^D4$Vl|cO0(s@Kzv)4AMB|Nv}2q9l3p8tVfR}ADmy+m$tq~cup#z|JG^u7eR z(AI<5`KB(|%z7={s_sD$T}+!7(&>r0icU{{aO$~Wg#|?4{e34=TAq*-C35NVs_0EX zgMP|E*u?5-(;V%mCI4PQuwm{gxdnBcc?`{`=Cav0N3BRCa@|=xEcDNZ+WO=5QHS(V zm*yIwr1<`Mrm>@2v7oUd4be8DP&ryST7u~`H)TAL zja%;=y3a6bOtbVJPN}9Sz!&1KSXGBWc;ZNaFkZJ^yg+-G_@fl@5!;A1wP^UnTS(8Y z*=64}{{^bwB52&#+e0^rrAK>pE+VI>By&DYEhh%qMlnBY1Jx>Y~>J~YBqmOksI>DTT|=6r1db?fk| zUk&iP&kb_L9%wmQc(BVRRaOcD^db4lVqofR%19Ug{xD+E8v>yKkf>t&mNPgODvkQ> zzXEH{skiTP8C-i%XuB*V!dwj@yLqp_USqZvZRXxbr%b+iYZ9WdJy~P!zaNDsvkzHj z_YPOc9G-Xk;N`%onleGk^!s&CRJ7O7vekGQKm$ z?rFz2sX(=!Tk#^^|aN{oT=VF~8|rWZ%B=v_Yz>^l<68-FMlnWc|GF zEk-ASbHAxccL%%Fu4srGfFKaTOs-rcR%+DND5z=d)Wa4Z+0%hNZak{3$) z1cP+&`dBykr!OJ`e(fZql$0bkF!HtJXk|<-oALALfQeV~QqTRLkZ!PomwcB8R#B?B z`GZ&rhXKHZVq$q9#eismHi(!{Lb)!b6trhjH69E!uv_GdxHK}gH%^6N)r{y zs|Dhb&7j&JXuT|3#b&mkzz6jote1nl{hQJ?UZP+BK;KS+i~=sdGPUXCD!XDoT^A7) zC)kCRVg#`CaJwVZy_w%TMW6IbO&w{8+uk~|C~6+vCw0s#L89k?=HWUm>S}o@jz+ae z`TBU`uXx0!K{5psMS%a3C4AT(64*!YA;ds;k*Al*+T zwdI55dc;Rh@h%eCehV#kKT~gq^kJtY@370cBOsJ7rNIMGqc@IvEb%LT=c)sFy2<@Z zm>G|?`d*qG!rhdV6WPvMg_<$C1Qgyw3$F}L=QQ7jCgOEEuIuWTnnP54lKo>z%|$Xa z1vF~Y4|m5vwE(Z*8qy~1cbA0~Uh-eOv{NeJ@IZ&6d#O{y)EH(w-R^q)?DX*rx*YSG zw6G;it)A3=xr8;nZfhFma(ik6Zh2dx>sE;PxKf7~?SAs8lT?4cv@4Ub()r59U2(<1 zBA>71n?P7pq)6Jc#3Q$EX$|BBcJD8;Yaeo=XYUQBa2|Om70*UdqY`@oVbI=aD=v%&r|<%s!@Ww$ZUS@l!N zpm}$&uC2ijhk`by1R6+i4H-a>2aELy1ws|df| zC0)b?zjdsp)(CtVKv2A$KC%>ZIU|B~h%V(b>vLBfOHOzDl1tgdaA2kTVzyOshLQ zd5$@3Sr`k9Cz^w*jSS4m9@cza_svR9Et;x5ng5x7Ii#y*7V=|M-gSGMef-TeXS7fy ziw@)MH$})G?M4>N)O`B+(U~uJ|8bRh7}SFHK{fW z_%6xzORJj#rZy!)$45511Vseyl^}w89Nuc3{iQ_w_a6~3kd%Y$I^kFi{f%cRPai)S z;AIF4T&(kSj6_=3Yp}U!HBjC`nk@2@#y^n*7YmoHbXge^j}~|GXQz_)IYgjK*SmSU!$HR^j2Rr`_A`A-RrRDCO0LS#ww?{8Vh&VJBudY zSuMmL$$TEf8(8>=G_nM+-<$0+%DshJnw}(LO zQBngc&sJX^(CtCHhE>;P8;#9tEM#6CZ!7`U8DANgA0!zo@5)GfFI*kxNrKI~*Dw1& zzWg9*F=A8l!)Y8~5mQ7JNO+Ajb?p`x4yLkBZPVmHwL^sY@WS%VE}Y{td%PAURn*ab zPgJz%kEr0)2k%;t$W^tYnJDsxj=$fb+-Ei6-61|?JW03uhM1E3pG#88$AsP=!Cf^f z=XM@_^|$|A8N2V8C4Fk5+Y-}U9P`uxye&rPMRr3aZ1};&V5%2W3K+M%tY%&vOsH?} z(+*Lhb<-JE6k@gMoL}4zYM^!+vSZOUeThifh;l-=pD?vsF(r=6^A=pm!Fhu3j|WG6 zzKkfmEYQKZ`;q20TrMT}+y)M&$++#N zV)j1easzDRctv*uJv<_9)T4gp&=3p+E-6QiS}ZeXY|Q?kGgJJB}YpH-rX9sc?zkv z%L?`hT9?ecL=l1=ng}K`u7177edhFfzNg9hEMOJLZlm4`-kGvn5ScMx`5Jy67VWhz zz&O7csJvWze0h3(!@+qlTgu2MKBQ39gkt?0xVO)CdWnNxUe;a8?ovu(@L6$)sUq82 z)jeEjo4vnRO3n=e($lc%>)?-7Iw=Nkb;{0R?F09o?gJ*PJxDep8lKES?m%PxT~?X_ zkc0sp{ZHUO|0?3nliDhPmGH?#<|8~iy0uhRdjbpI?EN#ZOScThU?w{qHcr<(153yy3=Ut=A zXU13$aJGF5k&996|lbE%5YohvqvXBGi2(ok^`v<3@YciPYa+%pTM!w=JZ| zV5^@N4gt>ra6YnJs;?|$0%u#kF7+q(0bLhBuceOtrwC4=IY<0YJ%PDYS^%ztsfL2ZZaeiQEwL{E*38e(Kj#3L7!uA~llA?{%yTo% z>v@XJu2luLHnANY9Xy4jq`n@&J$vm4lW0<9y0pe#s>=+_bDrH?w4!rR0ZM`Gugc8R zH}I^^-IeU^teY2;uf`!<|1X5=QGJewgww@m3 z6xYa#B=8P+ixIM0OOjd=zVj`TnzeV@u9-AW(hcHS1D>H1$X!jLV_gHx_^Ko6THg{k z*>5e(k!w2bT8|w8HZ#6}S?Y5K2kulh)me6RuW;Pe`Oywg#m6>+2m71XM|SfS{RcEl z_VM>~#nqM^DYcmixKSss!5G! zJzjRFfJ$q-Kt4)CBy*Unb?3`}gvo9cU?yg;zoEzYv~h`C-(36N8<)1>T$$;L$&)8w zPQLSV$?^9t5|y-agn%ontNWE`(s;@wPsAHGZhIGKzB?i!ZLUu+-2)5g2p>nHN%V3| zF-!=_kDO=fuGQT{Zf?&UmbV812P(Bv+|l{xo-o|r3&7<_Wm}a&<)HT(D0fISp1g!W zIpke$%^`vYlR#bQ0%u z8JUR+3|P71^(WUXxZ58jx=dEeKp}b zdA6_N^h}eSVX%?B?>bRBP*i35QAP#fo}^hsVpIyG~btN2gj9?l91V zSHp1k(jg9T`*trVn4x8rt6K~Lx>+A}i}8WU~P zc+^4w8U1E=<`(r=B@WGf8|;x@*p|l|oHz}vh>L7~E!2LHhw@9)K%UqOVLsjOJiqqz zdvN+Qv{72Oe<;%{jro-RqSB*^7G8dGEpIk4UX#`ZRM+p-J&N@f|4wXZH#s9tl$4bH zU5f*QIvN{i@U@LmdDIU*fFAtUxRq06t-lgi#VS@H>F-|%C*^e%kG<8mVn ziJ04|Ldw|iNm4+Gwfz(S_GQ`hlg`Y92>8M_zZZA0{z+3$iFA9K?7-Cw?z-NcvUjWB zbZ)B%=?XK}U1qgueq`C+y#Aiplb&0Fd^kNz6y%ViH&-&Q>%a+GJs48%i{kBSvyldH z8d-oQ>*2LGuUNyftZgbILB^9h44>%UukwRfBH+znZ-grWUvRq`arjr3qo-QI3VCE6 zY@5>8p20rA-UTUP!rJtpLL8-*8j8ZBd)SO;dyGpmfW;;v;P{)f*{tqDjTkSc8pmu&eF^X97OvXV$43lGf~^D}RMu z?X@ait`zz{#Q#V@V39;EX-Kb(tqU#VBOq0{qMB8s$WC14Bnx>NBMX_%f z*?5i6LpCR;bU{|6pW0fwqWq%3OluPYrEyG80Exd>RyO(8;IMC zJbF*-B$Z#%Y=5=Xa!4dh$`wuYBuc#Iq=xO^vIl zNffro1pu0=aEum54MO}dt~T8W>S0236Ype#UZfiF=hRt<=}u>}H#J}X%opsmF(XWn zq3zR{;v+)!XYOD*lskMI!%rS*vRw2HXR7uQIj7haY&KVnm?ZdVE8(;ez<$K%>cn+S zfcflfa5K`3f)1vX=G`tIN-$Lik@fzQxVfsBTX*>>VlZ}b*(5{!zgnD82Y4$ivux1E zRFZmPwaS5>GN^SB@2_dVO=;L>>hdz7ftOa&Rm)sj8C2yF{16b^M-gyUTrhZE8j!k)M-UwtW2&h|mAtgcBS!he`&5+6hE2-g;dSpAk4OXVY&^HkPkCz`Fl@ZFQ^V_&2GLp|9y7dk}-IH z7S@C=c)d8ydM~7;B0BF*Fau z$O6iojN;EMo4Zkw+yfl4S84-9N9iMHuiX8|clLO0aS?enqs4ruKT*JrbWk_?9X=ss zWj%QRulvr72VPpS^X!V)^#qwVgEkW2)~vYccJOi7x*VOJ?s<;=qu1NZ*)nUHs=NSN zv;Ud`XL#7VF7J?+XTZU&8k@c%(SP1tl6&rXfzV&i&(7?P;HuiE)* zX!K`E!5Se=3MmDM;i%2cz$8%>Uq)%+d}TFtOBs}d4{p4vo_@~XW;SQD{z-4_)HXVk zWnq+@Ev6avj8tC9^_}HzVQ+-oxMz)a5Stng08o=w> zG;Bg@kSC+GjevQwsT^1{q%FkfsrpYyx_NEie0UoevXj9!@a=R?V}=i5>ha)|&#QL}R6$g~GpWNgdLs?QGLenlqH-OOhtu01`)#aV)K{*p0|A z)BCXAT4D7w3OLi6axWW31en!*(1gNEP09=neuk^9w;_>%VrrY(C+ClRd#N98PR-A5 zHAF>fkEQO;NeN`y$cah$u}d8Wf9$*^!zVl&+-c-fsU{Az3cr&^c+R7G`6s<{xzjxk zlw;%xDRj;vvo!?C#&lb@3#(JaG*Wus^vViYsiR54e<8+Jpqh)`AJLHmie_Dn%r$Z6 z>H!A;;Y9!GO!CVkrk@5*vo?^^Jq&x-S-&{-HhbZ?BMa4U0??4lb~w8(#zZ_q#TjAb z&WYrCV`xCO{f;WOw;64$aXVr$X5+>-utxZM0)Q}@_u_eejoNs6zI^_1NmI;Pk{q!w zIwKFJef;wm>Y?2a?AGKx#e1HSj~2ys0`PcfH6eYUO_(4(NYqN9nBADw5jxU1)3`ivH{XUTI+^qrww|KzARMyW z!z(Npg+6C_k|UTb!E%% z@v(_rlqQZ$bNZ0cO4acG3L49W_0?*vjlfRXV0s$HH}Br2<*BnJl~Kgf!wf9@pgaJ>S{qj>%_9Bxsy!% z%l+On^33sO*D=&xju-`QH)YGsTUY-HupKL6kJaQK&s4!}_L=p$z>+H8ABx#?(aJ&$Ar}9; zFk}bpaPIvtn3Mm0g|sqX>3JstsjlVHWs#)g6mOW3 zUoh3Xd#4?3Wv&Ktg_WJDn|3~v(&mh&izB&7pd{Gzq<#BOt+I96Gtb1F)c>ssJLuRECG`ERC+dPCseI?ht<73)*%usy#ESNtMFWi9Z^ z+cv#kg+lbD>*zzlJNJegU~JI>tR$av{vLS+@%ximZvF-8vl^;aq8`yU-s9urvow{E zDEDjsA9E8mPAD441B7S!_))6u&Rq^Fg+3O4D0@+cj3RT_&&JgxslN%BBfKCs%xk~y z>9C1(M#YsN{d=#6g$x$4UapgtvG>cay^R2)KB%O+^nd`s#y7px7Z5e8Hy9^f4}W-pu(McAr|O#x&A07MzoArl)+ zt(pUrmD;sTbqfCUP_Y&SFqkQ4SION~uc$tJ3kdRjUrzRzY3a#aHo|)vr9ZHzLxVC> z=ZJuj4TF6rRd0o!4qRR-x)XL<3P_OjM?*m9dW~0vxY>^S+Ggnxz?}LjrLZlaa(uYx zo+*d4pisOuzE91) zknlblNZEYL-7YR8kf|m!kk=7Ea7?Gm&CU)66ek>UBgECOo3qGn>i+pJ#FV+N*@i) z^b^xOBPw!*x_uZ*+`ddB8r#kQ>K0ut@^}E6BmP(x-+<&A4v1trBNL!_XEFBpGm#s< zKxTjy?ay#kqK&nL!?exdb;t(70i18C<&*b@i9kxp;K1>d7IeZEVBfpm5% z@vFbjd6(azBn?J|FJDHQ)jTE|A+@(iECg0aF=uaIy%W+uY1H!u)=~iWVGd+LQwJxh zL}EIXu{^`nLDDhI{G3&<%87p6>`&eWCOc-A zg|jrm=95`t>v2zilv1M+_A}X8ltV1bCdG=i-3U*MjHZGt|{FUSKD`Gq9CNaLSr-cNoTuCAy*=ry3fDG7cC366KnOSl?jS>~1Y zt;}sC1HV8>t_62I2eMc^X&WD6^wpfuXT)49iOK!Y*hlYK?t<;uXLV9eD@XG3;gmuk z+Vf``BvO%j2HO~sd>(V>gHMf~IxBNv9B*7!{HJr5U;p%@(>Ui2Q zqPA@1tf1X_E|ts4OnX+cWgHTd80vmEBkUxC@s3Qhg_r^(XzkDBlY4OP) zNtXWrzS{)U2k|WL!$QJ#=WK`8bN{I{fA;Zd8?TNBFQARsuhw(aIdx1eWQYO?iztUm z6`WtWOm6&oT)A8rV=fv{+7V3_@JUF2w3(H4=fo6?L(kfz8gh|*raGg|_%gCJXeG^9 zQ*!GO8Aws^}nO)FZR z_%rLUy?dj#XCoas^yyau1w!FEqwn2YLF*GGaIv2@ouItwx8@y0XGk%?a3kxz!L2Kp zR6nr!>zsYM3+rv)12oM}4`<>9qrTJmn~_tK81T~}H}7=lnw0H6yi>unslx}u6JIfZ z{9@Yo1XmAj0wyt4S@uC%+af~x!+a&6khb2Qs|bM%lJx7B``}7W=$v1G0%@tY?Qt!MX#0# z$@U9k+4a_oY-6Y)kXFZO4y@e}ZIe07SVG_PuSE9n8DMiDI&65dtOpXWd87WTxV-#Q zfgvhJuLAIZ%HlPoF_W3zj-SS0I8}PwD=^Iv*99Csj3B@D7x=T}@;x1a+z;LXFp7pp zxD7fxkt8!|l|Phtaphr?Zk-0{d6xdGe-wg#8D1Gq^5q%YiTmC)w`435R;Dl1)lh>m%GHn-`m7_A` zUVMRG{+xF2HUIdwVDif-aPJNkfNU7p$0EsPoM={aQZNunrwf%-L$H2zs z2ouM5nCLoWNXdQ*r@ExH*VmQkmE+{m-p1^M%ygOU<_-%+=O-a-XT%4g;1Q_|(N~5A zGUJ-p4$*}$`z^QYvv^7qi=p;Wp4kidY?sXo){oL(AS$m@xFS%j(!DzjV-<^K4BOZO>+Z?%&Ymc@K`#i;5x{n3}MfVR5b;%9={^oeC9F+ zPayXU0zuyO=Jz2#aK`ezV4TXn`fVYSO%uyw;%sA09sC#7j?nVmws;{nyotvtfx}H= zslSI`fZaRnjN^K$9eTvTm z9#>>-sc2B(j*%8A<5@RApL{+!J4-Lv?lP!&kH_t=UeBs0aOf3FpQ8bc)g2m|Gdsl* zHuMwj%~OCUw+Q{1yMNg~VXWM>yr_mJ!2@e(Aix+ii<(JUrR2lA%uo)1=2iOm?0$jv z;@{G{36z#`FM!zYgYwZ|hcwMUYmS&uiW*Dx6fg~2C+l963kv@!7xc)`&or0GY5c$O zZ+cpO6=J|+#}dk%5W;^D_`;Qn=2V|QqqMZcVY~KXBBgQk!2^tyUBt`xw&Xm9D0Zgb?|lXw z%3aes|Bv`FmZN8oT_ty`J#hNVJu_Z%qkLK@6Kb^?Fyynnmg_4c6y`_O`d5#MxWVG zg%gM5B7;1s)WO06HW%u>fZcSPJFq)&06jHz$3%IzrCvPpPx9#rpr7=Yv(0_Ll&s|N z;)MrbP<=c09X&}eg-`=R-J*m@?%o*`SU|15ivDR9d|Gp*3xrQzMlj8}^eVdl6j69x zkD|`}rUF{T?c_Cr`_QeXk3H`XXiKHm^t4=KwdOK!`&D9&bl9i2QN+nnTeg1LX{b3*qSOx+Z z(dT!r6+ry09Q=)dVKun2s!G@DiJyu=zM=vVPn;7pV;qm;(WvGmkLY27=-+MdYq5&vN4hC2zzz92nx++{fXd#}tgrZYZ`9u383v%rDB@1SNQJU?MP>OlT^l`#MxWkyB@P>K6fGmj`oiU2Hi7TP3#`X3)%VI zcuwJP+Eg#!vpG8RNtbgf@5k!TM~mNv2}DyQKi!_6o9TU-k!jJOE#)c~bK+cw%-mGA zy-|?5r^j3Z{4$>b0finRuFsj2r+hM==FwH>iRWDv;gV?Dt+Ucb>1|j2vWMX0)qYZ zM}~Z|gAWUQ%yYpzw}3vJ&8R3y%VVTB;1Zzm8hMs?#aE(PO~B@Jfs0BbHYZo8*E0xn>^x3U!XdxP9W;FyM9h>w%Ml+WnC zQub3-tubOp2aY*}Jld&}Ch`L&7?bjHp$|OE{{tI(AdL9zX1Dm*1s%csT7nW@7JzNc z1u}R3?aOQLiq^}7G$NG$0AAQ4>?89x@L=kwgU_Eg82JSs@LFEz5gQL*oqR*V@2h@~>$(6&)Vq4Z!mzjgAf_!d(=7XEWLIp77vkkX$+w6EY(s>7 zjMSlJ-+VJl=bdvM?Aw=;rl-<4m0OL=Z=QbPleJ6SuB}5(r0*#`o-t#G@nH#qzx!O^ z$&V5Es9D9se?|ALS;jX0Y<%)7<>nv{%+CHyvvR#RtL%XiXi>g=FjY3iXjCrO04}UY zng`X|oW2_#r9?Lh?4XY_p@UIffzBl9HK34V8}@mwVA=7xsBk21_Tu6KVCCGqv*6Q5 zRTR3GpOSAVS1}MZHKYuba>dn^nLD75DSD@hRmKU z|2J)?91L&uo?|o0Qv&w}{zJ2KT7Ok`H!8YZw&um?VpRmfBx`_48JQV0`&*#Mcc1@g zeB=*Qu|c?X8|V86QUY61x1V|)l7CwsOdIREy3-)^{JcZdd~l@DiY!FwckPEAzxcqr zP39Aijq%)k8daV^cCr+hcYft5tt9a2+;dV^L;%Ozi!`f0-FODn|w9B7QQl! z+o!Yna6U9QDD?F3O4KuMUL-WE>+Raq5iqkO_c}CTD0eWn`VYr1@Te(frE#f#;T)BMZv#AbjJY)Sbt} zzB;@oRy?UhP-Sj1TfB@LmS^r*gO3A(DI{7okVWv-t=cJI4|Xu$qLTvahbv{6XlckH4P0HB{{3YNvH$es1(HNB&vKG#UP9 zGqwP{C-#d!f6`VTs=v%4R7|ATHl>i5LbM6|PIVH=aDS@wUGdAR^V0uwC5f{94v`q2bGsdE0COSUj@~aFg1boB0#By#ot787|Md48v+JXJ|#`?8x` zd-jDh?aV_G0kQhtAWH1L;|FaSKS2_~T*kze$A2ap9#TXD6}irRWtj0pQ*rkEneM>k z9@jB1ofP)=@c|mb38hyaPo{sqp!Wl9Mb`}E0o2h4y z%O>W_{U1qJ9aYuSMen1LdeYsENOzZXgD8!(bT>Se5&n(}pMB2R+gl2q7L)wpg=Q1dI@Q6R!3|*C29&xB*hCp3y&rI{>ud=VNsK+I_rwqlY-%hb87v674CVA$IV2X1l ze1$T7RY7UMZ6Xin`-oWPS9*DQeNWW%(>}m)@qOcSKb|PW&yV6pS+Zt()Xz|zrZQQq zMla%L7ux66z;be9huQMx#_C2FZ?MES7H5`(dtM&$S8E| zX*!$|J|S`gj994l112o0?8asM%?qt5b(bd7?H!$QV*UX76@|#?%TV{C^Csy!`nO80 zq;KYUN63~#4W)G)MGKFmXgLu};eEjc+Bc7-#=x|MyU^l!&z3%ki5^gx`Bhs7E(o4_ zI`XqTsU+;23Kz9e?U_)k#RON4SyryUA zuIpd`r*#)*S~;}?aRWf(1wB(FD;kV-GSb}-IP)*d?Zw?ff9`;Zv;;IBr>3T-b(L($ zZLo5BhnQPu6qV#XS$?`OePR7cFP|rwiAD{Hc2*Xk>$j1g$9W*_6kpdkS?cQEeyH%( zA2C|IOa?qJr8YXCjlE|~>Cnv|joa|C*>4@7P$B#oFU`QrMpUihPq)eYV>lhRf$YQ( zBYkjPN36l=AWd{NvctDsDozO`Fu}SGMuJ8LhgAU$*2^Y@9Adf8{=HE$3;|?e*E9#kH*M#v#K=uVC;8 zX~M)^idCZ+{$((Z(%qf=NH5wMpVuF7c163&l05vSaTwO&-CJLl&4aweUHaW>hx~yI zyUA=R+75KC#9t7z)^h5}+{nYB!dXRe@43-_2sq1UgFPKQk!zbsn;0Ko-X76n{I+~= z(s+hUbDib0k!lP2@Y`6Uv{t_syq)*cdW+q0{}3dfa}y>5W6&-$OlnZ~?<#N+i%8$A zCY)qF{EavkKJUl45&D^OS9Tat_VNofd~`GkN{H}Pbs1Jg!bR8s`xJTXN63rF^H_v0 zpZ#5hae2fJR52Dd9r(1;fBHdedivMEbKZSWv3jCF-3eHgkpH70Crr;a+QpS-(Rq`i zSCE6dtu?uu<}Ia31%Y@#-D<(;pKG&J^^u$=m=-A5C6z|e5QWuE3N+L|A30W&y_|^> zhI&3aJM2GhuqQWR9)S1Ir!C!~O1_%BnauNDP-wj8x7tTY%pSjfo_4pk{8X_h(S;n0+ zzh$ZTFP3EqXdAFYbW>&b_^DenX;G??Pf{Txic*(6hrqYtV zl+2off&z=g*mq^o$oZnT8BA92139Oi2jtDRhQB=?Mc>yr>QSa`#QOWyYWmco^-?Ql zRw!|uO!7g21Tn|j&|=pub3Zyk>UprR;DinP*fj;jg#NsywR=hNS>1M5e3qbJUXVVl zqlghFog15hQGZtE;tQy8kn`a0Uxo2(9Je1sqj&BPaMfZRqUk$(rO+&6IP= zitLgo=tYR>$k!|Q_;tzlciY**r{_)X)}COLQ1pGO5SQeB?ZiZl&5r?oA)x>O=65i# z$y8G3^6X_q_Zel_I%-+64^s*PVXoWlTM##hn01$5`=)Ix`!j8PVRdMH+yN4?mb_m7 z{LFG3iclyK5UKUO_tAmIdAsLsco8<;?5hYw1#(D%_rKc=zJg(t#hqYvoYU0Cpt<^u z{ki%-Cli`~cFpdZxq;~4HIScQ_*b3MHlH3iz~;rt$;oCJJ+@*dFk;Fc2b13T!I*!C zw~#s)KYBJu%jf3bMUS6_YeIbwcyIT%8Sq!mJbf;;342WAsI@L93{~sPzp2_*XYeq; zjZA~3mDb@9HF8NzdXMGhAgoI2A)tML`p66INm+)(#H)9&glL0q9dI3-{I5>;w=;ns z=9t;7yW&BDf!TP_`Uiu2FbDnVRGdqTCCjF2_+hzxn<`(wH6YwS_k5I<1wz5=g;R=q ziI?{j^njUtVPM{z=l;THFq0l^5=R~9*lVaNr|Z$2oju091}zJ&s|Z6P_qj3)-XlfB zz+1Teu|dfsj0G}4uE_W4x8M{(x&?FK6zz9yheVkH2N&fH?fd)Rbu}c5WIiL(nB-qS z|KJLx&68^kz3-Le<#c4`>7O?-k|Q_R{oH*X>-|E~p2H?=!Y89NJC?{vhbM!_pN$u4Oh7UiWEpG1oc{=e_<%Lse(n=+%IM;hZ#(!zl*j+D* zpY=ube1t^+=CTqe21gL%I&V={fl<2W<2ZBKrve=gLPQ}V1pOA(w=z8D4iY9ihj`~p zW_!N=6X!Y+m*U}&NEy7qn-2-uF!*}u#gj5fBBA+yoIDqKOKI{F{kO0NodA>y?%`9E z%$l*FUtR&)A|(JRxzlPBe(+3OuojfAXsMAn#2JNEPT`-o+dIBgMlJHx*{vKFf!;pZ zE~^vq$&znYNg~f8Xv8_kK2pzHuq_)$F-8ic^p*HIebr0Atk#$5KWlHITcFX;veo^` zq{Tb`;C#@mb)6%5^9`+*go4x9di1Wm6p4eAM1s8rJHY!!3GUF7e_cE~0^41uhr@$SJBxLN1vFa9Bn+pd%GESz&$4ZmG{@k=F{b?`{9kk zz`4b5HV=-7mn}Q|^!yUM;V5JCO&$Tx?H8Qi?`aRU&%6RE4LuliwtoZCP=Cd|_xj#b2ua2<7V99d!BdULCtD2g zZPm7ol~q9Z@)^8Ma>VEUa83-HU|X?B=8iYwLDtBaCr^46ur49-Bh8l8rNOj7%Hi(?4JZt%ecT;ZwAgySif6|_ zh@Lf!(k}{mVLFdqB^L5pt`p@8O(B`d*52;Eyf)NCaYi!@R|+nEiBYIibkwczuEjZc z&`9KFlH}*K5Pu%KFBYZL+qC+6}3rZ>~ zD#2LGT6m*PpLVL_bJ1_d`3U|6(n{x{V{zb{F@?jn8w1}u6|cyeD7mlyfwiHu4mSmY z9syypW2)9WXnH7)WlW!oR13%L`9@#=e&93;Zib%)=;t)cyB&O!-nGv3uAHt!+$ron z7qE}~_By?vmI8G>x2;nIYwKbjl-8LF#DpMNqkeN^TRN19LUnTS1GP@&@@VEA=w_6? zJ%&b#7tBPc4PU?fs{lLsWCuD)@_?F~Ti&kvc06gRBVP5R?!hWk&%=Ta&0xPCx>tsehz+O2}hBk?8T0{?lCOg z37&)rxAHM%4&m4Kklo49d<)ZWxTI$@wLX}{lA73pUi&qG()Z zl_*_&VtCL;VcGehG=`B15gBTgKV>)~vKmZd`w^|`KDH!r7EFDH10T~g;;^JH%<8Cd zv&SbfxX~ISXEFX@203#?hYdSHm0FY~;tWcG%JjZ2SbI)tEBb(BKptO{w145uihMErQJ*P}@05jbJ;hh78UbnLFyxktFgUXO^ef_37FMskd)AUh` z1bCZJ_GYLAC-FYZvWtF;KMavJ+$KKYX4;05Y?e>fAv?5tInQnD2n;z@noDI%HBsd^ z`Au~{(2A09-hI!T@?_$L>ceCBt+z;-sQ`Vl2@cr0<^pl~zo3e?C-#}$;YMPl2Dlm? zH2;Zl8Txwr&&BSgy}fdTMI`vuU~$_I9gKnP@{A<$LN~6%??AMJe@Q$CWp#qe%EPXr zdM6pI5&K3ACc*8fpzhrkULg=x`!BG;1FHTRJO!HL`GI?N`C03x#3|Ee`oxQ9%qRtY zF4mjHlCB4yHoU_;f|#bZ3AQR`)vcsIUT}ave)EepytvA!h<=&$2gDlT)3=FGjFVlG z(rO{d$Kh2z;Rlc2V~#=#yt5tvMFP}?L*HI89Yv}jPaw0Ov2*i*T%@ZNWrXBYmW=b3 z7~jL}|F6n>^`gTCeKgqQ3sK2_oXd`@oKJ=BtnwWiwkIKE&vd$`cVLOHrfE4PZS`)r zAH4P*&r@#VFg0V&P<;^)(;uj%y` zW)uqLEgYME%tXDDvs;5IB9NR2M~+@5(vkQP^0JsxN~R%Gj8d>OS~>boaoaxdK2o?B zQwX@BScIGSu7u$L%{miSyNR-u$W1>t4-XH$N5twrJ`IQA&zPDs*jJ!N(<^10kBOxm zyP9ay680e0nAmKy1-Vp(M6Khd%}BuP8}pG7;8pX9go*6`_zjzrz5xPeVKpP7Ev~CE zLYKXnuCffIQrLA*-Gmb~3UgBxDdWZNT8};6i~n^6`<1$gX65f=qxH;u+uVEpM=(Ll z|6ddhXp;}}V9g8uu%e9ZheVS4(gt|0E4sS9L^$B)wkw3flN zZL4tbD#enXkoVp%N6E}-eSYje;#}0A-n-IzJj`JnuuylKqwucgz0~P516x}~CSai9 zy&Es5AU@Ikz&>cH0n9%CIap{(Z0>H?Vf%l02OZAH`R0;7SfEK-QNVMhIdWPCDr*Oz z1OwEvR{bL`OyE)27ykRz2S0JYL#$>3UG= zIl0CAd0wZ1Zi+gOF_cTpYY4t1?hjgQ(x>d#{et}bp)8q!?zhDy6>ovkKNIL+@Y)6w zYj708s4zXk?;kuN_9`=$S%t#(cFCZ0YcmpzhHH=4)Dvt|Q@P!)0Ep&WdvDe0tFH<+$=1>oIHl!9YnXDi$%n zZ^oHW*n$PAwK2t8mC~+pvSiO%F>OpOILfO{KO9UmqnY<;VDo~{X)WjiMtfWQ$mKq+ z&Gv8l1Czl-Q4Bq#JiRn{Bgr|Q{ND-fuIX9g@RjFE0V5s=w43uJm3Wo>90t(XYvMh# zzQUAZoRBK)_JSE(t|p`~^KN9(AY$6hWUj&^{BhJ+@(^Lfeq7gbQUbI6M=EFCjoZuF zv*>;_eA~>SI_$ed$V?TeiN_alh%I|-C`!rMiHPTQrnB3{w=Yv}%B_$-X{kFwOcw@le(Ii26}pSv8$uT0V11_2Pvs#aGM+s(}dN;^S{2fJD&E zN8~sVy|8&4o&_PQI3oF^8`XRZ_J zfjd`qh0SvPdSfDZ|M}~;9_5C~M9%h3h3nQ#T|a_(Amz1W7G)$_I!uJT2C4IMoRh~! z@y>p=MY>{9w1aL|;> zQQkWKQP8Qmy=8Dj8l|J!@%J&K-&YL8`p=6_QU<_iS&`BS`O*+)BI2O+9p&+bn$Mk? zk|LPO5Mz( zB8~KX{r>90@*M`G@b{1MIfki`(OkfGehL_73vjN@%agZ^4(eRDg}>rb8~lAh2`vdn zY5s|^C9>&t(}TXRR6ab5gz^@SxzgScl11f=24G6V|Smn zlRhXH67_tHS_zQ#IA?+FaAc}PyVN54B6aLT$$qZfS8v%N5^4Uc;nzvmO6OR+qjGqv zLA!HH1z5z7BEw~6|MC%X_jKIoXOkZF{^_YX{$B6&!=g0F!xSmYUU-^5)U`o&1}jT= zu-6mfncnJ38hn1gBNiS~$3gN~ooNNT*j)}t1xfUY8a#u%ZR4*`t!1oGrC+fALK)k- zf+NZ#%NV$%VM8cKoKPV$>zn(Bz%UUSx6SZ-vcqB4B+Bn}c~TXpa9V;XysCfX_&oWg z(bFRP=ij>|&$~s^(MD0T1QlCJHq#?$$-|i#VR3QiC^?TY`_OtZ!dSCI{j=%S3U%kJ zoGNr=h8BLcI!`h>A8~O&s|2_JPF~X>NMN1}jpQ?ML@w+a>d1zE|F;&BRRzp$KlRP( zhlN+8mF$(@4hyqbO3UKEmLftV8Ej&0rh_8&udy-xjqmHmFUBRLf742KTs#Kh*G(P$ ziXDp0v9pKgAa{AVWhHeCSrPWSwWh=35Pjpm>X2+4{<}Ys)REszLr^P`zIk7;g?`++ z@`|K0bN})_G_w*t$IX@ifuAKZ1%}KmhdAZcoTUf1du83p8uD z$i#4;y;(4TFc#fK z{PPe@sS!vL*nO-qrI@BQ9d*9TVFLXiPIKL=1dmkM_|=i~+w05D>&y!0`D}inKoB}z zJg^B#yE)4s2)A3NdJNm5?hcAa0m}uw)s%T-$2wHZG38~#S&|O_O?nF_8BVok4$zHt zQdm~+dWsKB=e?MP))on7>_qCnEg=6DfxfycOKTyA{HC5HX)#f{V9InR^Nf{v1ONSlY*n1b5_f|U^oaDAKbm67t5*bkUKbwcV>o^_MR zNak{lAJ=6cB9NM9?<4dVC+(&v0)Oob5s#<8 zsvqqLFSf9G49l>+vLKb$*O$U{nw02mrS|{ejfvQoxt4!4_<#xjExW$<0OUxv@$XS& z|9X?wa)veje@lnthJm+Ix=U{g7b+_D#m5kfm239*Z?u1IYW*6YG?+MFiuB+WJ#tmm zTYiK-6pXxFg$S9(K*-Ua+$#19u&g*hS;dZjli0RF&MEA1dqEQVF6x0?LF%8puN)4O zs;mOCbUXj#FM@}wKTe|{;4eIe#TF#28w=pw{OX;g?{rmECC>ecyTxA_uQ49n0#~sA zKNSx~VSK5K;v?c3oCWhEY2<#g8lAVMVIr{xssBvRS6wJy52Z!x*UCK$xW$+KOP~kG zNF7Oqf>F#9uy2{Ju{tsF!eqhXQGdK0Feu-^(iGo zq_+!#5$Ts5&t&KXuq}U)(aU0VP&ykiQjqSDf1c6>xeuB7Uqf)i&_!Ek## zhcEacvsaZDkCNVNc5-Q{__?`N`E_Dq>6%?#PER&@oW&X@E8L7;+0I?zFc1o0n@JGq z2}ujllKV)kLhvB@3}a9zrH@x47Kzo(W#x}vQKzC=0m*>=8gr84N^6@FmX2ppj(t>8Vv8EmN<`KR)h#va@S|Z@POr#G(N8#y<^Y{rex+jrtm~xU!O7YkHO!GrA5uYuM~)z z9ptRX@uzRBJ#qN7?u3Tao+zK3Cgkuy?uD8gTBz;l3zGTsK z@yk%Y|B9;8<-ou@iB1s}p%XR!SH(BJR+|X5Se9Ql(f}QSll2BZzA7ik5luPu1^*j{ zE98iB1R;!|Jq*_)8l*}R1QskEFY9IAS44sRscFCub+do#Ts1+bGeXz%Uo+O-F=I60 zvc7(PejAOeu_J-&9A-~X7ks{cv7Yl}Jh{DT|B~J{8WWwvigG(##~Ln0RQK37$?iq; zpX4dxmi+wuywXz7oKLl+xV&&@+a@h$OvF73XmK8(6N2{_OFh*mQpjJYgFG8$>Qx&1 zdD%Ym^>S2f5F!K>w8!iBi-M@WholoQzJ7~>Ac7MF(S(0NK_yavA4sMk3p*3yzA)8# zW=G}L4LprZg9J=m&sR{rc7>GoM}$%HCsTpE`vyqq_&~x#*;ZZ6o51tq;fL!JMM!pq zd(3m;&`dvaMeLsy10-=|L(c8u;8Zs|Yp4oS+um0%u6kxAjQ(s^J$UWX**Y%h$^A~> ztA0oXeUXn{VJj}k?;87f@O@0?+jtqLx()Dr^*v!Zf$VA|VsGhq^%N2y6k>HK1Km{& z;j=tKiBS=0#fjiTY9#HrJ+jr(ek`2GtSJbZPx*H{vACXs^p~>g5!QSa#%nRS`in0V zDFx3XsJx$DCa7{*X5Q@C<~El*q^j9WekIwYTygzK${+__$3b4w!Y~o)ZY};)wc>uO z>Ld?IxrxaaYsp90q1d4p0DCtHSVJoF#a|ik?sr@kX^seM31#7zFTb!OI=f8y(mgZ; z}eVGnAgZ)A|WRVUCgCl}y*CcJ5<;}a_ ze7iZFlk)hJ^X`JA-BssHR+g5KIzbe@?Ocr9QNsB65Rks)GrRa}Fd1~ZAoN=}me(*L zQ?soxnt?oK+?Db5N}4h@LI>j*tpd}zf+EjWJThB}J~eMaLV}#mDy^7ZfEtdVf)o9D z7F*Mv#zXvCO2Xs*^GEX}$?8GX)a=$oAN(2d$-KLz*}@9g`{i94Y@R0@JX8|8PCX!Z zJB$f5c>M+iz^8efWj4qOc3Z9h#fJ4=(xBtDb{*rOy^D$(vW%5@9MqqqFz_vc68|Zo zLfxZnNLXWK=pzrG$D-e+p>2j&{ZkVY&SG)#XQ&x~RqnCmk!53&WX@JS%4`xey~Q8I z87Js-ZV<8atZfiek5L!2EDhH HoJ5fAZ^a_pCZu%Wv2Vm#)c?1#*NjxK$`~iXVkoFlZoqH+@(hn#D=87lZW$Z$0#u`794lR2M zoiE|#(p{wT&xo%^Pn6XRODcXA^?LcyL&N%KNILaKu?@BVR`#|kS4%Wy$hF&w9e)8T z)9w?b2H^jW?z zS&iBAJqF|>Bq0UR<<@KDXLxjHei$8SzQQXLmRsq^LA1uf8iQ`|D>U_q(wj3W+8|H} zu~>P_60V}ED$whuG|Kx8f3cM!-`aY6B>g;f2#_zeL1%MZ@)+7rURD}VJK(>SciD&$ zT2z|%IcyskcrP_2t_JAwKGFb(<_*8zr*X z#hNS^f)hE=hTeVTIzA3X`URd0h29>rtP*$v-JKc{@Qi7LxVs9*sk^)@WuOQ!j4 zWuyO$p%4n^OO4(*t|)*9IX(21!fZvQZUPmKN{>yAlAPAUA<@Ta3c6N+F#@@95B|bb~tzw-~r?8$T6Os7o0jpws#{fRiTk@QSb zt9a>ez7*R~1)n1wC#XJVy5~z-so$fQ53CBNiqV8zYhMC|{Hn~51Ch3>sF0F$~%dkiueLO6FJR#5Sf=r^Yh_< z$tipwel9gMMPFUZ@=v4S{0|L%Lh5%4egOfW#wRBB^O3a}T2n$kVI?_FU)D_>th;zP}JA|m>PXCz)`_?(0ujy-_6+~7T6uwSJ?do z1aWO--hEj~^}Ic4=or=LK~2NukAc+uFL*+J+wVVibm5Qfe@LqlI(yyS-onwiadPma z@gcBhPPMdzR1-Mt+@e$l)FlY1lM8t_(-j!#2x8Z31%(_6?EMuWQMxp_HjSXL`2d2% zK`VI+S*xoGc?63CqZ47IT<%{S)bBs_QiTLwj;nI%ZczD}KNG^<5sq~< zq=|5)q)<>$m?bhW{>IMEF8A~2Pc%A4t(b*S_?O~=xw$!BV5(|&nb882A9r`V3hZOU zHy@p?7(;Z7FuDn4e`#oRd_H=Au7R)TS)@*LO1b^dE?Ou^eZa^a4*kYQ04gu_zr39m zTj;!^ukP(rjW79W1$Yiz)Q_)bn`p*$P! zOp)xHg0sRTS#83?a%ZPYpgCV{3dn9PRn>RHq`3@1B~UBwas`}ER-E}oY|LGs#=G#g z?SChBM)i&gF7)3X{Qe^n8fOSOg!?f&LzT3>wgs}}#`j1`F_*N%GxAAM@I%~){ptbs z&sx^c5|9+MeJw0rEiANB0XWUxY7cF|aZH2x!+U7iA>L0rQjMy=%fm*mglHQu0jHmh`fWE}>aoN0ZaJ(&b z6AcPe*3sT^O3r^m!e~Ij3|W!3x(Y#GRyeoZ++m7cProBUbj5Xv4044sJly?d`F{Uq zcDlWs+O-3wtx`03d1&t_qUBwVl2-6XK1b-d0~wFerG5KJ5GV>i#ETH$c;gQV$(`Uh z9g9>h93V9S=%ClQYA8hFrY<7bg1v^P(-v$J>da^FkFjw&%D z^W_W6(m2&_1F+#T&7fykV;#gwjmTk-QI8}bUGBf*+@^YZN$p!X?E%Fqt0oHo1%!z-hVZP|CRR3^&4C?cW>gZqbJW-1%)~^7$QQwD}k|e)?9-V;FS?NqKO_uJnFkt zCeweDxo7HsEF3oQpu?chfK*D1`oi|Ezr=fEtguQF`5K3>+|hCRCVWhJcHyGp7kJj| z`QHtz3svW4Dha4UseO!lIU5$E_(1(Al09liSnYltz{M|v(LWz}%;}V3>Rr_AI5SQ^y&e&L zBw0;Gzk4TvvB~X$67m&u%zRIoMjF(wn*IO11QNV6p>gp1H7#{r7VOo8GH3_weCSc6 zAH+-6iY_a8B0*t&fEGuzPPFc=tjxpIMe5rrVjCZ4LEOVI-27qJSJ#r^l@Csb(*fP` zet)MQWhax2DC8GncZ<~#BBK_cRLD2zB7i*GWG)7dx`!Pj26<9!yxe9d5`C z3sZQ){viQp8$VYi3a85)2M1>xjId3=1y)(C&U4;&gzZtN@%wjxA>#2@FEuhj(%Sx( zDhCeP=W=q$C11Wk2szlKVRD`FL~VBEjW<-XaF|ktmj&+#hQ})JyN9WQNV0WY1E~FB z$D)YjMv}qw-|=d9b#1~OY;A`jQaNP{Y$B#G-|}`-dq>BUVv$mpW#u<`4Xn7?-;$2d za6^rE{h^T8ia$IgW!}W~1R;0$k)oD}>+3)F`WL!*ce9i^V|F>dtw1)Rr^6;vq4;7_ z16xa_F^Ba!M#%K=_WMZ4w`8un%1|D`@tF3Ya+eXC zv_4ez2#g4c3ec6Vv`VM6z^+z^%pi+I<`o=mBh=+H_)N1E;;REn$4dI~AxxM!N7F_g zIbDhIH)XxK%!p?2UtRa*SLn&WTcq!a*;{R_8%$XK15HN%|E{yYBX?)EWkp3*u7u=4 zR2W?mEwmeMdg3hzuv2{-6#l>jLN($PyZ?gxZ)1Xoc1hj{~ zg4T!$JLpOPII6RR_02IpbR{bcI~jrhMt!=!W}lX`UA$VEeM?O|?p4#8hXGvVvOWnq zy*ScP8H3S;w>F;T5z11q@kCp1B-EzRk_SnJvM!+W^l1xk=0h5=Bd4|U=!+97Rt`?i z{&cUYnQfQAV~Xs7fnqJ*RLZaK-gt1&^uq15RAeyR1YOEU=6zRjwiT)U7}K}*aV*yA zZtWqCg`l-62(|b#Jorl4mm5_)RVh@+X+CLt=fEbPiRl624Qx_4wnj!fDf{}zkH@e+ z1+ABKc^!i*OG7C4*Tif5Lur1#gG7gkvMlT`4fvn| zN^9BV*snzsg%-u)_!)D`K%9jP_COO}M+`iOTqfoed-%c;B0(0W$z8s-A`R!W9!Of7 zZa)>+1Vl?TTXAuWPd^@AQc-aaM!YE@jIlc_$$hZ;*PP4O4~?gX)758-0YVx5R!`#wTV zhHziU>4TaH$SQ;*U@JG7C9q#?nOcLt!@g%(ZS_;@>pxSlXXA)eJio-D*c$#U);1nJ z3z1{e%>}y3y=^&r(oQ2Mhm<38NDOJx{-T~hg@~cX2OYNYT@MEO!Wsr0YQ;~k)+Vd1q?R&hrG)Oy(M_B232J?Vz&&+mlG2AH@s?jk zeiLJ$vJ(M~f!y~LgU1+C%1$DSrr@K&oK+E0=Fp?puaS-;8Djg4y?&HDo@P9`~wA)Z6+TxrNK88arnS0$DDi&DA zhfqTky0Vsq>+q(m@O65G_SQd)h_2JU8IN%iv%soj&=iZ=yXy9%y43A6rU7Bt&&f%v zdmo1TeDb77w@|@xF8Nys_b@6TG}%8?FZu75otXG6^e_4!|4^cjA5~o0twr)B>oA*V zl+7}GFnocFK{hJx%MC9DQ?F)qFTH@XNLNvDu^w&SmtNMWzr>*hQfB-uw$?(`BXw`( z^kkP1p2!Ix#SDDyv1seoXb}6psyMBmm9r!(Nypp{1;G*ZTQ4d3C?QAKZ}(qW;9xSR z>CWBUd};Xf=sZHp5P?gL8A&I8Cat#S_~6JV1}042VGSV9FO0k3r^d!FXgr3-Qr|r` z4VA+cVFVajbeKtMl>f(U$lVPeFtf?_7+@<5{f&xBH<9$VhqlA%#2TcUQ&M*qBST<0 z)u%!82vslH_BldMNuL_Yl_RwG_L+G%aVXVxzbPDoJoje>T8oOtz#Fc;&WvA$=AF`> zmXteOQ1{uNm6oErN`rGBH!BW4{+P+{o{iBYgoII~wWk!a?KOVn6{+x>W?_3VET_{s zl0H@U5^3Pn1s|;rNOG_J^sj;T5K_-KFuHwDJ{q&X5v`n~P3t{?js#yj{86UkuVjHW z;^%BT3f+Gzkk0d zu^DGn5W@amA3>fDr+%;Cr>{V^)a1cp*7VoX8L&^goGUi`*R1+a=!zNKfA4fKmQn+h zf8d4J(a|Gvu;ML=@St-%d;$>Nuf>4OAdmnBP+xVDw3`#AByGj6MxSjMJk{0xBzY4( z@dN8|Gy`HRk<>#XvrfGYOxXh9!TF$%%>Fn9jv%Y#yp>Z6IDXfII~BjT{`iDaJGEaZ=vJm(C%*(L(nQ$=4&bCTk+r_7p)wF-+%yV*D%>pcxQ z$*u{$@&CQ@Z5(xsb|b>K5z0ZLji+Kei)+N{;KQh2Q$%;AM{q!oy88^geU+8p`dwqP zd<5B^q{vp*o6c|xy8vmpBaQF!t0L9MO-Y{v|Mu(rC1_pF1k0AHDRq-{@gAKztHhup zHacOg5Epa;(7=<2E&qOBP8X8h+B=It=cOI2t|b~wu{AU_O!pOEJTs2{oj&rIJz9wF z&W{~4(jwX@KBxy1=1Pe$!lYZ!L1P4G&$%o%{gqkLK5|;YOk&}PPE^!jZFc)zIv;7; z%B0?hS3HU)Z3T~O{ElrZcvUGoCw#;3&~pbwGBIYLbga+ti9ldHNc>vjX9G&dbLXRw zQP%5V(`V1@f)9=}HBH>Uf7k(S&tCfaM1$2m|Bx^vy0!toD0sYH(q2mGzLl-5pq;nI zLJP_(=HSlLkT~i<9tM75MFN8S(aCF@CO$NN4`fw@3oyA5cu?TO#S>WE2M&Yro}MRy zpsJyPK?eL%WH6KEa*tEvJZ2%u1?nOXiJIe8}(z9Wp= zqg@cA|5U$lMEJ(@_%#;SB=}!yo9KsZ7no>-3a_l6PZ_zir?%e2&h-OSK((`dK#kL{ zi50rG=?w18L*b_HL9hH$(a(|`e^qLmv^cj3xGV;;5c2Ask>0&t6>c~?Wp1&oQfjB@ zB?gDFtW^q(NLCNF2#Tlwwbe}D*t%eFdQQF9*VDApCwTcKM+=)tyI*AU0n>2z&V$0n zO6jG_@?q09ccxc9WChfb@#~m0&br+{5Pf5LnW6jC7!8f+J{|yvnvq?-Jj>!B|8WTT zrrfrAtGB09pIm0c!oq5co6~-z^7V@E^yLnrM$nY&8yM`SdstIGvzP_WxgnQ1?@0cP zgxnrykB{qg)jQ275`>bx{*Qc_HT?1YJ*S2gSZ}NS*}8{w^-or>m!K8m)5zUSsSFi4 zFL;o4BHjF2c$j>Sfyw1J^wg1{Q$9!8e_w5Ve-tE9KYSZnPkU$jwjCDJWF-x`($lT^ zD`?{-ag`FUABA2(kR-(sQmcVGJ41wAGUe&Z(MAk+X9?ES~d|rQo zT+1&sM^IdH#V#O9w}##8mf>jcG_p2>Jw!J|KE=?0ZAUYF@v7cQ=-ZWu*ngD(ZVZHb z3S_lFslrjbN5;1q2Rd;+PQ~H6;&!k-j!sM~c5f@Ii{ZR6F9O54>p!{4&^x?k`xH$x zsmgCdctbtqWwS6}+YWho3TLexpA&=PcP_w9ssh zvZikn4DLVU$2a|2{`eR;6}$F)G}wKFYTlH?%4up#DzZrzsT_;y-UPw(e@j8W)Uq`c zR_+1fhzdFEbN0?(ajx-d%(;L#WDB|2HRG+fECe2At&tX_JEY3dvsVW#E|*apWtHwhUwDazoUIjkc6#~|j0Cyt42vC(g2Y!2++%2Li%2W`ceIEfaH8*SSh9UO z!xtKnk_d}KISGk{oq)r(PvRRNbDx8P;?glkPEv*%TTlQ7%zvph><8R^+VMR4O2{$A zCGq}!DJ&w{!k*OQ$gf;jl=2{i|0P(ts}12BI~M2h9#;g_-{Bz_g?E zJrbMXg7mQ{V34A!cz*gk(<02Dso%GT*H}fbn7;QYZ@-u_x~{GlENpEJG+^7ek6ksk zO?%o_zF8({h|8mBOJh}$q!G@I-9&Ty%Sjbw%{wEh^Ef$rAY2*E zLw1dBky}XETrTF){^!Y=&(~wBt88?dXaOOi_u=1d-+z)nr$hA8u}21UBe^B&R&=#k z*jTOi((F3y?odwf`?N*q5-Nx&PfTWu&wYL@;_%}S8BPaT942TLVeH}8gEHT2!TP*i zh3j++#(-wew%pn6rWWf8AG0Q+pxEC-H}T)8k&7FMUh7TlP;a+)nrC)WkU+qZ=mlwq z59}JJvxwo#tR@oOzY78chl#rwHG8V(&w&qPy(Ch}-joqI`r(_ljy2P!9szfoU+;P^ zHyYZhQh7n43@&TEOoL4t{*9`4cs_W}X<_N+z$d9;(VuT|ZcfaQI5Pz02wAqT<*2ue zbIX^Y6L&H(B^SW{fZZQLWf>V6CVYP$Ag~hpC$%nj9V2C8kw546N}oLa4>mm=L4yj{ z{ijgF!lnQWc&=>`ZzCr6?h1pBHM;lLw5u%%v(KDx$X)pPx z--U{aiKYuJ`A(>TjCCNS8Iw*muvyb060<*md=^lk%K>oWAU2c+Jt?w|_ps|(laU^f z;Yb3zpYq-EmSMx%trrZ04=xg(FVx!Wp11id?dyN(MAkxG{{Ec_fX24k%`~ZN8L726 zx!h3r0O=d-TVz*X-KaQ=N`Q=OKxg1(au9F?Ca$lo4JEbA%{~i3>j;CW!zy3m?jU8u zoh{DfPVg@NB+S5r@Jt`lR=>E2I@%F%pqi-mZmVd{DJH@DlST#q5Y+=a_?a?)tbO06 z#d97wf)rP$_GP!3&*>67;Dve_Z}gtj`)~%wgP|fpI(a8&=X$e%8-ZbwGnF(8*s}qV zrR52Uv{&=77^^sx9V!_8BLZ;U4{z8nw2kqvNT4M&?uQJHf0Wq--b-$LjNPsH`YOX~ zD%DMWx=Ruj*E}}&vB~ADZ;B$p^r6CFpnUfx^X`J?`0MR1%O5Z#7Dm+@57t_G+Ur2r zd!xMNaOW0;y24W6af~l->ZI_dRp?nfAz}%+5rw{lQZI!D$s7sswI#q5M?SH{On-@F zWhHdgQ#8Nr=-T6{rq5aV6+csla_s7!K)3A~vwYp_%|_b*n6OIwJ9<5z?F8ZK^3Mi z*3CvF6hZa#(QW^pbh?iJQx_L^)OCHYFdCw+HZ(Vrr`Oi>t z+kxnrD8A_auPE5MFIik=Bd5m3EWVzMzl)8q+z^qM`>86qz<}I&?AzM)iuG&}dHKoz zNIDC@sJgF<-yw${I)xb;MG-+lX%M896iI2MrF(>-6bVr}q>=7!5JV6V9!k191f=6V z{@y>p56;ZJ=j^@qTHk#-qOzlhAm;DiB0R~n>k^jNb^}?Ym{TxzaY#ST;rr%hIL_NA z$GzNoU=2P_RbE~)&@m)F2DNkN6crWqUk=k-akifQEoIam9sQ+s8#nx4nbnARLMh>+7AoI7z+_eIu-P`7FQ+4X@b zg+i?0g~N+hiKDS8$2>08B&y#}0`nVT=f%KSbCyO>3KRLp@nq7pi*n}5GuD@g1HAZ* zL$H+8_w_xC&bGU=K2y{y7SFB1uzf|=nyT`LW^K=@KD$JQgvf9G6`uXFua>(Y^+KcM zWPzC+Zhv;^+(PpJaZhp_=1DA*#VCge-3 zFWcxe@7gY((^ED#zvPVN{|sP2{lL9@=i?iSf1q4e{d=IEU zDg3X{b0>XJ%q-n^SW%6eDzkArQIje-WmzKL_m6}ehY8JjJRzkp%!LButMtEP0PO{l zO4Ib>pfI0xbsqcdNv5wSg~;6&ucB-(2mki9Kuj6=-p(22Y=ebzS7&P{);{kg z4<9A`Ox-u71ctZJdqs$`UrmO)yBN{SJzNcij&E)7ec>j&bN5c4??%-~KQMVKC5w5z z`CeoHXn4flerinqHfTmeOBlEFvm%vz{}vi+l1eUy)!8Jyv#~^yN6fQ8g>y}!exS|H zW$`(qvfkmyBLmV{o^@Z z#%nv>ICH=InTAf}+|4!F@2;fqU#^etR$s#iE(uU#scMh=;_j)=xsIp~fLyO>ZsIQ> z*-?o3z0;Imo-uRJ!yg&tPe9P#8yQ}3rkA}n}AD-YStpJE?N5P$%c-qs{qRq36S=o zY1q8MO1@9y*l;u0->(ij!|DQ?*cXAEJ8 zk~iCMWuRy$xSsm`yFcJ+YqGz%c3Np`bFBOR!ABOxHd2TTs$=e$lFW|~!QrpkWQEpH zK0=MHbnBG}k`AL}mHWbV4sY)-?_-w54*Lb_PBY)Te}=)NU^z#wBUMJf+1gqN;=sTFE7=VdQCqCUY(W<6Q$(Hu*}~=_1KM`x=cDgD+i$|;|L&Pm*}c%# zwg3%BVyybLsU?u7*TEe}G+Co+KQ%kzPCQW~HWCCW@i6r`gRbHy=s<)IC$ge80STgI za$r@eLzQ)!aNgO&nBX=cYPHS)_Waj}70PF_Pc69G(>cd?5Tmg!O^cy>%_nI_wGOHE z$;8|^^G;7PDv`WthvBrmVg1)@ZxHBcpQt?Pr|s;Zu7NYaQO^g?cy@`V_%eUxSMI^^ zD=IdBCs>dbpM>*=#?#OyR_2avrnn4`jdiDiO;^;jmS1b?lk-@fA&*vCQ9^(YDH&zQFz>#gMPo;%{@ zBD6mb%1*TY_@DvmnJ;f$B<5z|)x0=wy-2xsGM=6k`Sr-N<9dPaBSL4r-=Y^=7$Mn_k7 z@Atd`G%rw4FimD+n>KD2RCZS!XVN~N2;#+Y5|%R#`}exSeA0|@p8ZeuE4LA_)YG&a zARx&)i~nXuZo8*9LL~!N%+WYrdQY32YeVC8T6;uNbg%_IVivVzk3hM~ns1eO+z5P6 zOuQXd(5H*cy@*`D&f+FazW>Vf{JrsIrrWP?w6-I8>SU8OvI@hCnlh%k&y-;3V&hf` zmpQ)^3mVDe{MqSgG2ni*K)A@HK|@UQuU`;nI-h8OdeF&m+Eti`pMEx(!zt|Dk3(h( zs7POz)(gZgKLWMI(wer>gt|>dX@h^)rJt4$sw0-6QLdcMYerI74SuiH1wW%n1`W1K z^@kp!?M_uEoqBLoi5TG$K5`8`D>ljs1plLThgU5=6Ap=#yvjALv6*_!cu3eafffszZ7 zFA=*7g$3WuAB;WTYf7!Zs+pQ7EWJly^HWj&0Y2Oyl#OW=iJ87&<7#ue|;S0ZxVda_|)W3D5!`{6I6aZ4R~uUm?XM@ z3Ep@TZy-Y^el!%8Yol?%#$Rxs7~9@{XQPKDfY)VE{3IU~wM=(GRvl=T{xaW_AZ33+ z=t8RQGKDmz_{2N?B^-sVbm5dP8Dd=^F@5Jjm3eQWJYes6gTYjzka#G86C&g2Sy+U> z#Ja$Y2nRkaOj^s)3OQ(L_Wy34jJ|%N3O^(hq3Gcn%d(c1YM4cTMlUl{&#P+cZ;45p zN;~K9FSL=bwmtG^j%~2%kKk2q9qvGOxtRC04eGl(~l-H;^yEtyB299<@Sl6b-l2sQIc$and#1WruseuLEw_M*-$wMf1o56|#?eOBj?}!j- z`x#BAXJlt&U|)E(nD62cdYIH!#R&e%5sFqK=u&E1KQJsb`S_yU6c7KqZxsHqj>)4$+$ zU=VTqs`7Ac>POdWQC|!2ddhb=l;g5@>#p2+^$uQ(B&tM)Eay>Xwr;9{-aqAV7MiPh z5XL#D4E3MHaFYwKi+AH=L69O`p;4aLU%p++s=UJUkE!?QXI;@>*5>0^4@u|eokRo~ zGdH6LA3jDF?1io&U*_!kc_T7grT-O-&*jpAt+j?kLt9(Is@P3Sj#EaUJ{@{-;lUpTmf*O1hDl=Q>sf^E@9{04DMi}a*x%GO6|v7* zUGkzGrL<(vry5C4CYhG~b^iAqbWw$&ZcVO8E|mHG_p7JuDR0RM`$yRsr-_K(hl!bi zo-h;!u*UQT#jZXPI#%9*X*OwFY1odZa1g(aMr1RB#N0!Q_>}NXO$n3f%jIv-YI{~l zt#6#Qm)Dbn?aILid`;_%L!WqJ-3BtYy@Hwp31r|}+AFtqcE-T!;I#U^mJ6r0xA@%u zJ~RARkA)j}{}=XpcSy$OWM^dVi-{j{>fPWt?D@Ys$t#J zfia6Vh|hg~$buoGEXod&bt?0(|E;~3W6h8}SZ52*ijw%z&gJv_2gIZY#9KPkBo~PZ ztY6EINhYmmRnYlZO~eZmT>l2(+zWIk7TOg|GEIQZf7B#P79~0`L?IEuPu0b`I!&+X zYN*KrT^}g(<o$v_zOi$j%PQ1d-(Cp@{J zG@*{U_DL{Nq|h~T<^Vg(2Sb#Vlz@gF_URJm_Nf)4u!GW!-}tRDO?7eV*RNk^`s|5& zOiLh9XU^NYMqV6lJv+5a#kI2$M@L82DTgs?66VO&plLa{9*s$SdW-lukWOX$E9rab zEOhCkS#XdfCOlopV!!6PFZNSM^#NDPa}92`-hh4ZYo-?%5%EdhLcl&pLAyH9LxfCv zYb7lxqWo{VNXU(`j`$(e1V(U=L-|Ce!JK(ab0gzhbmWrAD7ChZp3U<-p30 z0=E_)b~BEg`;SuWO{7OB1JRa051J9FRHOmWxU(I`7Nz35*EpxS z`co2kuno)YlfM_E=+PpXRpdCX6?5@tXWgG!J1ML+xk&QE>n?lJe*H9hdD zL-QgvY(6Cx^h`_N;*FFH+&6M!W{ziH%6u=+wP`Neda0fk?@!wnil9!oKGwi(bA{LE zu|xl)6pG`Zp=vAJ*DHSlJy+K$CDo~vLJcf+wS!fQQ`a#hL0D}C+J%~+SakosWH%Oy z_*v{fS;Z^2zar_!zRn3bVy_eId)y6)zBE`V>0IabBfE%}CFLhe5$1ECYXFAeX=*Gg zfY#wTe0#B{KJC=`cXO;*pADFPr(E))w*MPkF2I z#D^}H3(txr?Wg4Ro=aN}W8Os<+$MysyJFQ}0Q;?xvQ3(cFomQ!ta`Wk9p0&zkd*o6 zZV25E3Uhl4l79n>;{xWLX1Q?^d3emnbO^;nYBp7~6W)5W!?%%_=6!p3dfLv!wW5K| zC86Lp95rBD+8oz#mdiN#Z~VBsvooq7eSj^@-57$X2y{s!+_%5udJ^3i)A`K-^Bfj) z=1;?{YefBo-+szy3vW67&teitm#_52he{Y6zdA4LB%S#TA>ggoxFB4rd}xoyi4p4+ zy@_dTJ;BG$v@$ggGlKqI>+%RnqyGitar?y{EqkmYwVBIq_q-9I>XXdcao>ZldHs)f>kNza z8-+}havu-|xNu=y_jMqBtb1pPr1Q22KUqrbD&dojsl57dxYO&e47Ql@#?A2M7OM;wsPrHDFeCfR6Q5XNr(b;H&i{EC>iA*|jbHnqmB_D+|X3 zjnZf3;rR$ZiX514h@aVFEA~7L`U#tT%7$pU7M20ysZXU}x=M!JZoSKVL3X{wVh_$N zTtA@712H~ZC^9HOEk988@82OHkq5aJlhTv$eLXIv2e-)}~q+@-6M9t3>6>t-Vy> zxTlvy9L|rg2czq>x+|yC$cZ0h)x+#kByJ;9J)BDSl6vA5acsd4J!k3ZFP|u&V=&ZT zVeiV=rRIhB$Yp-}estQ#jj1M0x=_Enjc7f6BrQ#W&oR|&Brg#Qz{3A+}YAKAtuYD zF_(st6@fW)qudG`Tp`!a+=iwLIjs7ImPtQ4^mO8z-APXWBO1f!REI9frM`wI>mQyU&6iMOwL(u)7^ zR2RwMyN9>W2CwC*cYL~?WG%%`(LS7`+*k@Q=0~?Yrk_vG&N9zp2L{A6m+=&lj+R96fh@r?pL4cs+u*Q86!xYbU%>?NOffwTcEwCleD^r*>&(P`QpAi-5l$) zBiw5J{bws_XIDkw?B;=~sx}oabA1+sj1jV$X$Fv{Q5(n(7+!WN8IfXH-V;C~O- z=`kuB|5{Mo=Pja(YM=-BG%`2YQRT`XrB@=C5>94(94~U>c{>0n@~%>zhL5R*S9WhC z$*a~=5}EIgFuRqMN;GNn2(n^hT*mymu>0vYKdwx{a!;jdPe&+py&-O|(MI|x$8F_Qg^HD4RDU-N+##n?reUjd1bXUhGTVdxyOnCruiB%9 z_(-LmbEk_qvqs5hQ9-~fc&w=|30wu^%FULPXpVw{nlsl@(qQiAk?u9&^#NOV5e?(Q zP#IchlLs_5JMHv^5>=Y~9T-}YJI`b&HfCD@`Rk2{TpCZyT!r7&%h`q$&Fa4v8i7?>7?O(IH=0Q}Pq!JQ z(Hg-ZiIfl}cIk9N=D&}9{iCmx3fQ7~NDf(SpgzE(Z`)hQ2#Of*!;x`mX}e0BlfOSR zj`#j$JcRM96AF4t+)Nf_{w30Wgk(!N9~8q573sEWxMk@8$?@~P3gk#4rKJ_tmHi0@ zY(xuew^0oF6|aATJ5chbevoHjXy}-?o2~;TE?!FwGD*eS@jkwSIMz1vmFS;v+67i>N!Wm2KGkWU+5MjVXBNS6O5I*so2 z4S4VCXlv(@Ni3J;TRfH>xp9fCSY$QQ(J<#G9EJABjeYLcyG@Oih0fmC#TNyx$#@QU zy;X)dBRT{ski5T2I55*cMAh*<#h;+6l8u9`k3RJ2s+h(@!0O?Ulv`;xS+k4OFnU9ANV1hj*##wR`n*rRPwlaf_WM0&kI0y8U z{TdmO1wxS__)2z;|NE*bLB2$8VvMM$wRw5R&9HCHQ4zQ`+Je6uk$gozR$?n1{X!#;b5G}IFQC+6?mEz0TnOT{qh4@PF#XtL>fuZfdLe)+8} zVV-}}D9g?Ncp3+B5~R>q)WI##7mldDBuJ-D&cY~e@ELUBG+^}<>%I_@L&}Uu8!`F2 z=z~yFu3pbA52lC2e)-Axgsd^HZe9e$hPp75&X5md!xnuM6Rq$C8OX)z}G?r4I^eS-|eSiG-HSz3_>;aS7cf?)J z)PrGG9WBJj^jznx_txFYS8Hl~_!kNL6IUj}D(LUlQJ>h}ZhTC9jN-hg!S$tBlm5p! zqBm=lR_|~@0FN?ftoy%CyZ$h8Dpl9m@8N$_Q;j~H&pUi35R~mK9@t>r?@fyfiX9aX zGTCBgoai%5bgjsm5mzy9SRzz7A7f_tAW=()*SMrezjjt^AARMnzPn9gQe&alC#cGJ z6pfU5%C~l@3hBxm$sdJkKo`yfcaThl|Mc*EhygA7J$2Mlfyp1EX=KE_AUm|_GC(fm zT3OJP5Ymad1#CE&zuk^<zE(2V!7kiz5FU+MBJgfL(Z%x(S*=s#1@oKxicp&q=M6r_) zI;48yMeg-Nqmk6z)-baI+1En_T|f!{@1L5@nD#5Lu&gHOyid#2f_|NpVIE>qIN1JN z>qT4NyZ8C5c9N&Rt8I6H_wx0I>u=?=!8eebgx8J2OsCBOTrBn{g2ej-A)|vkv8Wl8 z&zwpyj}>MY_vuFQlJS!U0&Ncls#p7LWXVwy2Vpf}gG`wLS4$A~dWdyd6MY*+ScFq_ zEZi7veI!oo6RA%Z91Yu9!rL;k@_K}{Gpg-iyrViiJX{3K$&JSVfNeodO-KtC!>dB|6B|4^+%IiCR=>s(ZhUA30@Pz{-m`FN z^9w$#eXp?XC`?xXE3OPcPh4I5?BzH%UI`ta+m8*;RD{DS(<@IOR;sL$I-%V}cDz6@U6?&_t(RSPjvV0s_sq*}j zFT3gtf|IuS`!1a>=_`&aF}Z-Ce{y}NubRMgx}8ZV(;IkUQIVEtbnSJzKxA2lp2auk z^`u-ZDFublzhGwltL9h;Jaj@*TF~YGbzaILHZohoAD8deF3gEmuBPNp+sAYbxb5H* zM5i6=f%eA<@zX)`2bX_(Xt{P$++WQG$+tHfY!7BeEGp~(WJiiXg{FeZ( zIA5BZKiup%28;-ekB2Tlq9gYYfS z)a1|?FFL9!>=&y_mX~$vl_)!n=9((O8Il_ZD2h_X-DyBiYFr7UwU}Tg^|kmL@5*atY8W@FL7d(vn0aXR_ovh(aKKvsKxy0XdPTUgN)FU zMkjZ`jNqap=$f#v;**=()~CN)!V@bN7L9%ARjh<1kpYEwnc0V)>5oL$SnA}2J}yb@ zSWF!kA}XEPvEYe2^w?{(nAtaKk%J=mOioa;$g4ml?w_rKeQ&MV?LJdeOJ!(bqmLs= zF95FK+3%P7&C^HY*ryYD{q+l$s;i#9GY)X|-E>*&>snlqQy_uhx0OOEsGxvw>9lo( z_|4Kjnz8_Y?DyDf4_DVW5hboG*l@?2*7~AC6DGDfRDi?tF5;l@6_Bics%}b(w64H(^%M?jZN>+Ep+yj25c{t$yK!>csf@{bTiMh!J$-`)(czJ= z7HjcZrYUUMwna_ZP{^yxsjD?0<5tu<&XxGkJ*}x@)Q?rKea(rv>><*B#F*K;fd{EW z)^Mk>bQx>@R-$57yqYwCe{IC*?$eyww>Tu@6+Q9dcp>@5P!&4 zWGz>{QoX5-7t)o|BC|zp2XRPzkbx(zxIjYZPFLE9@Cq-ZF6%f89#FPtx3tXi+`T*5 zOjsjq=9^^mve4DAFE1~Hx`SY^pJ~py;NnmM9bq5@dbH&z!sQy=C70Fh6%~L$SK-MJ zGtT_^_NhqMdm=(^x6oi$H#cBf@XH6De9wA`P1KKBKw0RSo)5^yA6JsDySyYZed!$i z_dQid+};#Ib@5GEd>1pzWnsaOTD#elmcxO^#owa%#=w4Up!CXjmR`$3C#+>Sf!ifhB_EmnO_E@MLnwGR%BZ#*=L+`|sc6-Uc_fV`==KR;T74 zi-uke`1Vy8DfoA;c^hO7W43b{ox^TRk%K}CcpUFLkO_b=L>O#aa&7xzC@nQD*{cJ- zFB%S?spYpldCwi^_5i*7_tw}b`AEMf&yC*J_37J}!uk)U-?-6s$<67*QOQ_h6GR9h zhdC^##70vU(wY!_es6+ zGBhty=sB!>BG3I+GBhL1w-dAg=gf>Dub5r7%Trq~X7)5dVQaw9{?krr8}&?^I&x_O z3>-~PPFhY3n=TKO|LpEM_)!JkSVH4ir+kpI$XYXO>|*@QKr7iRr~B%7H)r-i2y7az zQ`f7{yNSz+-^5~jxj{#~s$XkYeSvy=LH^}->cHcLae!+uNWexZf2Phs?j|45;4p9? z&Top2)2pjv9{}R}m!w?sz8?Xl@L*!9F0*XDFcqr@ms!*-JRk|W|Hj_VO;}J8>zhoN zWpzfw0};rG3v(&F=1Y&M){Az2gn8U!5YfaX9QDm~1L()%e||dp-4T4%A(xe1CD?4a z^ZRJv)|5r%m#%d=(~$P5m--|BEPV$65klbM;Sma)m@(5Yap(Ck(Oe>HJ21{}2D!sE zGAIR(%J>9(d7VJEFrwXa80JbK^5wB7P0DtXV(1^>Wi;%~z!G9_#`Q$QBTgJ|l3X*by zodp;Ot{z8E*PAK49xeE+cKh?e?TbCBb@uU7*QnRiA6~F@wDut%Y|uiUkLTM1$kFLQ zgO%Ixk_!ruftMB)DP8Xk2Bcko;|riq7bzDi;C|-~Yz{Vgson+Ps^l@&4Uds$r&s`) ziOuUx-PWY0zhnA@K%wf)LR=<8*SODP8g-w48vBBGgNf#ri{1MPsf7&L^vwrY=LfIj zavv=GCe9>lt5j^zuW`@@qCNi9^(W0R86I&d?a@FnzQn+?6Ny48OSseLH(9etbNkHus$I)QLI++HiQR#(5=R}}EM z&l_hYAk$o%mB#C21Bz%O7o;WWKUN%n{>)4B*OsgbXX@i_R;^bjL<*VD?;2l8LKtG~ zVBF`wJ*?jhq=H*-`=@@*vMTo*lgC|DhP$2Ow}Z>f3VWA~E1@QWSH|v@$=~3cZ|$Yb zw{}CKuc;zk(UjqG!)s|CMi&5STlgM@9err}ltLHEo;w9bsDNb*Kxej7c2TH;wYAr*j$~BO^&$ zUau$>Hy8gb>_-G~la(BAFl#sBGu~SE*m&!7CW_ChPpc*CfZL(Ba05HNgITtcyKnnZ4S6xN`I%Mq-6G0t6;7=yP)6=U_ig-N_G86O&FX_^|Yhb1jr~g;MGaH zbFa1O1gtl{Q)6$NUF;}|Hi8DRL9lu!a)})~Ne)_TLL=Vbp#!9z`Y$K$-7SE4=Tm6U zT(8I$QoZnI>YTGGY&6Bq8ZDi>8gnRRR(S}2){-|-XJSPw&lwYm>i!!UH$4lR{!nap zT$K2uYx8xgM$(Ckpz8-(+HO(j<{OyF)>!da2+7th5EM{pX3RYE$Wxa-V23V97Atcb z6|d-6v1=&XQ&s2!hdvttyZQd01*>g8f-ra+d+p~|!`W(baUb{$5&XGpukY8|GXhFo zM&*E*^5fI_Ac0RB8-(l&HuI(2gthilUPr$a8t*k8QM3SCM-KRztHN;s{%{IqCoo66 zZ?B0g=#iXHnG2cL8K&s$%z%FQeT4ZDKPT9iGn$;9&MuxSGXh#{Q~Jp^Qpu($r&>H+ zITw{nvsQ6NESsS7G%E}(-2mYp^#@t=CVgCbW)33T->Immh%=6!)|J?Z(S+xps&9N0 z781(*`c?67G@X#Fh}r4JwZJpEYg)&Ln}fuZ<>u!0 zr+UmLHY^=a$$(K@R(Xv|*hbN~?9&4Z>1HcT^!+to$1uXhjRWzcTlg~Ob;4~+1rJc> z_-no-T~#L&PNh~;LLf5rd2(nd44`1PnQwktMo=qsWZa_j4UA~Su~07Y#FZIff(@bK z--B+iKBNBaPFE4jkYImWrEY#%zQ`W(c}Ca0uqPwHMLT6-W&&o{g0*&&1>UYENA|>p zz6}c0h+WCa@9(gmr}+}g$X7DM>@_$rU~H|37|!nFcU2zA;1f}G;Z!T?HE{k$op?AP zDM|DG8<373uDi8~hUVIA5)jdDviP57oB@vTApH#LL*j@qMjT7OSBrl< zEf2C^nyxo2mbLd(GeJ3@XuGpgG|u4xka$LnY(hh{nqS3hC<4^z`$C_|`yBIJZFP2cHrt;w&;U zGg)1`6RjTEj+gZM?ziLr=gjn^Wq{%g@8*-n?8wNH*v+&{FQY>f%*}S3W}-%%!PW*J z{S9&42)qz+zxu-A0fKcCt>k}%E-`=MCPdxNgEDG z0148N*iJk#9UdAoah$G-0fe*}f|#q5CanR{C~1}J)P$9j7TksXLkhW0BZTw}$sbR+ z_@5ZMGftj70R6d~UXzG#0a=17@B;+fPw9xTc>7@NrxJu6)0{VL@;GYjCSMB)31Q5U zto-Zoff`N&sUQ)RL?(V}vI=CEQ3giFPP#Myhm376;qM0@liT&GN4WT-8>3PP6K!sh zFu}htuu|?;FS|7}nhgG3jIg$~9R-b&HD|0Mb5xIqvGRq?QN~~5M`2o{%u_YBQd&Qh zDt(LkIbbZTbq@@{X^tfew!+!CW-6a#L4KD*JeIw3)O<(I10X%}9 znYn6b-|N4HOyv=2PxSLw|$bK`r>5VSrTnaO5%M2B;AhO6MI! z_o@cPwu+qe^b%i93DK+V@0D;^C*`Ic1;-wNwDc5MhLjzvl(Q_s#y|8J_T+hVuX)N! zpQOr+---BuGG7Z~cXM-C;jNTAwkJMd^6=q|--n>vqHF26`xbD;Y?SUu{+Dj?SP6iF z$h&vapwJ#fSZRT3%>d18}2cU-Ddh9L&s+KI~TQ$U) zun97(Az|bW+VHwnP_B9b29aS+Fb_?zKpiQEK`;Xx)EN)`!rb7}pE|GgSD_a7n)X?l z0Qji<8Fx16lY>|i@YFYW2RiT8uT5@^mkt)aC=YxT$#$!Uy<%ID3Ow~=W0uVzxseF$ zjFc?JFUYHTZ3PGNilH>j_sdcOr~OhHt?)tjShyzA@JgPN5aZmLC4qo;DbUQGY~f+E~`5P#IcXJ?%GFeD*Ioahj+lbxL%ip7Zd7dg3GP zyWo7*rz|YE&Bx1a(#T8`ZvJu9u^aBfKZ)?obc~H1B@XsGSAqDG?=V&K5JT?X_9iGb3YuQwq<9&`a7ih% zm~!}!mdq6;Lm{0R(C>nM#K|%?pal0SAL5SA=r7|3g|c!5r?sSr;)YuhTQsCh3H)a7-d`#Ig%@1evZeBkH!c$<42p5)ONy3qaZFIx@nU>WO1Up$}i&EM_H*N;-;E;N2gzZdX_euzAFzr~i`yXzVY2mC;CsRwN*MekhvKQdmj5HFP9BPUh~jjp!~Ec@`6 zLLXV6=`t_9m|vbDJy&DjXs<^z?&ep!^lp3YI?^}$(aya;7*0bcT>7x815YGGd30rWG5Mo_` zyNio}Tu^oKhlhefh<|L*N*H5P3qNT0U5vKzF4aM0*noA(y9B8PMqe_)q*ylS3Ntew z`~#RISf%>aG@Q&{V5ov`vrV!rh>(hJm*FmewLQw=j3uqoJL}dgA9B^gp$=!J1}_&s z>%&#=z+Yg|**e~JRAamHvmJsgkaaP?lH9`6plI(xA%td)l%I>rrR(2TK&O&{>W_|~ zQyPNMkfNNlmarqIVQMP(ohz-mUE%>RP;%LK&gZCb0)3pV=j1(<&>=dgYXor6ZOl)g zH>$N3PlAJWcxn3dHH&W(hGS%Zl)PD{qbEbJ-x5a17BIZZ(xC9>kYZ7eq)p8p?z^V zM&277-vm|b_nx-Rw91>^b(JALY9M0OXIy5N5$!_czXyK%kFfB0Q_1)*4=<#+$ECEe zZj0B&0q;}x(J^No2FCt4iP+t$ncqLk0Zuql$;*8wL55NJNBeWsHTGtvKf8uF;*DOH z80yDQHq{#~r1afJcUhY-qC-)5I3+SpcL}Hg3;i3yy2LzM_ z)t*fzLww|GOerojB#btIF#l)I4X`=^y-Vv3dfin) z?39m}S1CQ99ru3=KQ{@uQV{ez^R13odU2{zB8%VG>8(;;J;{2m3^HVFKV@OOxcfd3 z*p499)*b+t*}7_lN-$31vF$e#Xe@rJ{wV)A(i6MC<~Mc@4iA!&lc}e!|2va{qXY<0 zXC?J {>X@)YQKDTy!>>7cTk2rF@V8B}FnY9359 zig^OP&#av)8x63A7X4f8j;lry1ZsgM>JzhUNCBsIHb}cZF!S9z!shJkE~n-de8wwT zZE6akBX#pwIx}E%pZQ&G7OMM^n5fM?%<_vkj66&+r8VgLZ|%sV$2m4HU;oNf{@B{O z%5Vk3dGGKZp5B~bm8*Dz#&t_0yUokGbT+D?q3ofWEg zUE-w(e-ZOkPMSGUbM-zAL1=@PcKEJ5G&D9=0#;3Wt>{F9s_|#U3H}p!;&;(jsqMOK8ujL_lC2;$$<(~H?78xoAO z>MA(y;p>}X$L{#1tWE%q)%c5_F#JI{FOk%2V4FQtJ>C;gpqU5DEL;i!yJ}ZQW7K7c zjb2l6^9NvSDJ8u%GV`U|N6rJeb}0r{LS#0j@cOOgTDv3G@1Qi4W-+U+h$J@A~Krg0n$-hD^ z{&|u1NAm3}I=}H7VT$JFavrc5M3!8c1j^<`G+^Y=);kG+@r1S%0`EV z4a+6)JVa9NtZW|8-hZ@R@FsSp;p3SIpjjHF|31R;DBwd{22JD5 zGe&<5`x90V5wn!o${tO4v8LPE*ziAk^hmq=MZnwZu9-I>LDGD9HBdY?8p2JXXqTEJ zMi)^apFFpI@+A2Avu9F=%@r9or0%gKe~*q#dLmQjXKn5e?>j;h)>%w#90*Onyol~b zO&y&vpTDEJR`-lt6{Q6kv~TsG%(NvsMe)s4&VG#Qh63SM(F14z<>reB=cO* ztb$YZwVw!e+aPMXgZS*UnTqW;>YDTK{2)B38}Qfm$uDayvxu1Tl2i{S{8&!a z;egXlmX{iY?EDegop%ufqKeSa4Zet5q!`{XwR%F`dRjJXSv-pOV4iuJuQ$WgMD zWF!Z3={y2Me8-cM)*HWuIsc0d z48l68n>)uBzB1X?E~vV4Xe{#e2rMR0*HFs26D=okp(=oOl>Tn*UUy@RbRj@6+)TkP z5m_eGy!^MzN1r`d58{B*dZ&PSabS2uD_}DFwc*IYj`D`HqEv}TxeT~U>{KY)(O5C4 zALlBWG4MOLd7Y@9(7SGqJJS)Gk`E7RNv1tzPl=18DJ4|}$JoG`XTpIJOpCvJz#Th+ zC+o~`tkl@pX;fQRZ)Ig=27e$q{?Yc3tn(3nPYB?5%YM>$5LXCGae1b+F$xy?TwY)7 zofH|IIg>BjLE+Qip%)8yBc-UiTx^_<+oy-nHNFh(J?wOGd;!1TeGJHoZ3||DbXRx4 znHRj6N>-MB_U_VLrMVG-wqviwsW|5V6XP`~U83k;}Mx%&ZDb2w?xh zv0DqXAe?s5A;l(=8sZ@*KGKFNSSiAXtpy8KFNQUrY|UKFy^4qdagn3rW6#eqn{1F7 zwI9RGAXGOXUd0BxN;@C8^;ij5b@zhfc^kR_$UoGZx9>;c1FHP|0eiTS=VEyINQ>*s zm)hFO4)_#}?O0_5Z8tJnE1TF%Vjz1-JYS2rzJ72zC-9sopvX{Qf50Ib;K{S^_C2^D zxy7COY3(l)oc>vfp44;mbSmC|rR)VS;Tu0BO_U9|Lk2@3gp$en?>vb?x}~pL?;b@i zY4Z%c9a@eY^)V*OIBaEZdv3<=q_ep>Mst07bM>?pj0U{DeSCiU7RXAo!G2kP0tIW% zd5WmZN4ks8>;!v5*z+XN91=5)nU+|}YL5DZC4vhaR+9*$8wwxJ0)BMy38Ebm^r}{< zsje2?a83HFz|Eb*AsdG64R$0vlB-Zv5(j*%eSWmnU4wzSkq}0G`R%?S2RC- zOH>H~Hs6=bYLV5js>t?>5?K}dOMbL*qnruvTlU!A{px@lpYmHy2;b?u?t1I6=1Tq$J5-?NrVf@KE{9Ya&z|M8d_#oilGwj^Xe(;TU4S|(2l#3(aJ zz(xf1^z|Dpl5%y4+X5xidi*mHZQ&3w+})9#*v68@^7i(Q(06Rs4mjV)qkNYsV)hFa z#*gloHpqq*VeNB4Hzf0Bo`RR(@#SE=(7rJMLVg7+!0K?E z0@cSI4&Ol_xAVH!T&dTT{kafEnKPylYlzDOB<+CCCQSjAY&`;P>k#d73bxjcu$g5N z=ia8>$)?oGF43Gs$nFU{03)^ElrNcFMMp-uF9EuchLFQ_!fw>j2VAECMMayh?ts$H z16CdC5^4}uab;4rDI?O(YLQ=tWjx%KMS*!Z0PK$en-ZeegN(0Oc8tiOo|=w0n_*u0 zH=kwe%wI!TQpo>ELA?lY$9glI+k_6a(nTqm#b;1=-*GZojyahW_2nG{ zPVU6oPQYDIiSP!8c1DWHYjugJnLMq+KegCv&2__bT`*<{gL?wkmLiq@dq;0#jQWQG+jt z3qaF4#v@1skq3zr{`2zkYX;zY%CJDt$lTmd1{fn$`0}>I8V@eEJs<%oL?x{7V2{`L zT^h1SD>9G1f4+WE>!wn0Q|q@Y9{*0agx1H(xRA$n zb|TO_BzsB%C1K*XIi`JAS}f3;x!GQmE#tN5{1N~@?FX|>C5?Yvr3%b@C^rU0_}LKTi;t3hJT52o zbc|$6;ovhzkign1ji8+6+rb~NTzY97XH5WjUNKg}v%&=OVuD$Z75Dd0+2;0dPW;LQ zk^&P(u5_hPDb^v=&H&%dQW_V}r$HBNMMXtc!h&uc&{IrTECd_2-GqgD@uOG)FHXR@ z6931aOJcxfFUeXK&n1C0?Xv_{GZhpi-xb8`qi^d93mcaRtk~`gP@-A(mHhKT^6nkW z4=WiPs~DszsCj-=q}9CaxYz8IUGkHo+ORv2S6)H9!FaZ54M;|j6ifjbXYUb79=h5; z#~o*%^>u}C*#+Z~sTdH-91}068tLkYh>AW2X3xX@KGLjGrgyrwxmS~2hTaQ-&4*w+ z3kS(p8D+=rYqUk{A!uQ#b>c00-~!ph)3mwxVX5#fI-Og2ITl>uI2gby3wlb znup<$%$pKXjdm^_A&Re| zhz1D{`C`-0PPh8EJzU8YZ&IpLSFsP~4TT5F;B#Y509`fMaIe=?2JTkeQIyB|i+*7* z`M|i}ONaNASdZv@RGBB7tJjyfDS+uyaSst~FB3>H5CdC;W0U=|YJF8zm4!Fg8}o(0 zyHF0%e6XCy(GRide`cchz!zp48+RNi1T=@{Sc^L7{!gO_mky6aAS}Qb)5Ir=W^jn| z7+QQb7UvP~SoxfApbZQ0%reIClQ+<-CiQA95i5d0(&tX;SOo*JZ~e&%jD%KC1zcqx zqa)L@^l2tV4A6a@qRj3+}ZrFeUGvaUPp zHpklU1YATwGuX5G)=GeBB%O|sxTjz=D3qhu&C)ElXS+xN`U5Pe;dUct2siO5t>+Y& z+v4o(+~c}3C&CUF(SI}9syjJLG5$bA8v+-StR zpq0xusj7heEPSM$QjX2FE(v!?;y|2BU`z?g$=Gn44$r{d;H*&tCx)QIX~n*TZC=Xn zCl)SFGYuLqlD($Y*h_|@?{XziT6k9;9dcENx3#t6ubP+eF|=bYPD1k$``kpFo{)um z7z)m`LT#AMy+n0CQf}H=&mVHX^E4TCmVWmi!Upr^6vq@LA30?C=)`&Q$E|OVc(dbh z@@YUZHsa&Qakm^(klz0Z%Ui&@$vGip%?8@ zVIg$UtfSzSJWprLiW}!+Nlo0o`KvB`zLbBM`>v zUF>(;torYDw)GNXZP9H1nwWU|>8h$N`D`ILuNKI%6P;#@6rzDo2#$z^2{ZeI!Mwl0 zpiaD{LnG!CY!WL*eTK>wYxY_w4b}-RgYzAZ4}@=t=m`gI18{E5efK%T7QHq2Fc$^S zMsnFFcXJ5_>iT|v^8dywjtbq~nQamx*|z*ip|1%fm0-%Hb^CW1Wh-;FU7z%Tv*c`T zO$N<}6v;P-(nbd^Ipbe7PFJkZ0v11!A{s0F?weZ5>vH-awOu%ja&s>=5#4e)gr`_s zBGMNBM86E3YP&Q6h-3|ncW2Ns>Cojx5Ej5BLw$%Oe9EYa9kLwzga&(!wrM9ZrOU_N zZaQ+HRaoa0rN?Q8e-%5fxa|ZUpY+02ncQts0MJ6a%{Hkmzt)Af5EV{)ld5)7u;&F9 ze`(qofSzDjl2}hzJZB}RXKEtim4_1&68KHAJd6OZBQHEfYMlzL(Kc59a>GNnAQfm5 z;hk@>5rNQe+(kxTODv)skFtV-0Yce)Ywu*5=Yp~)#(qEH#|>_Vg9b6cQNsn6#NLZpVASv-_7r@s)&6F4 z2^j9-tV!?uoL)~mPBQvfj9s89XIrt%|CSm}BFVN9Jl{ooc%L~;pLSJ0SMAiSpvJf9 zASjh&ZlnWu0{faOLkSv;9J$g$=ci-~2hI|LjdTk?0XI)f311qpmtB~Va=IUENM^T z=dvBaHvpyul7}7+FYw_I18x%-O@%yqHP`y;vYSXIHA}!s)o-T3{@HAkP5tRUp|3ch z2e{6bR(I{T4uW8nqIjzW5ClR2YA_kZ+G1Go=W;X5d~D<>NW)O;H)a+1zUG$ziW{+Z z$@rqr-5>o~sl&#)c*Lz@*Q8t>`AHZ_>(Gd!2#lC1`#4~APO}=U$?=&2ST|#j2#qNB zP+6&lg$~qb>ujvVaj$YQ1DsFmDaHgWwGjfPowt_`GxezJvbot=hqK85UIB4Xi~^eE z$!0mJGS-I*qtR9vKd|F_sD)I~(5Fl>`1#4`4i3$^{=1*X z0gf-KCnAD5wteM3yx;(Zg}8z_6725?{!4fyTzsP5pP-CrQa(>XZc4dC5@>S7eOso! z5r!>7B%r9Gsfc7;by&f-1)9x&l?n~M;$H6EjR#paf|&|k!l|xO+1BdD#%hx%4Qkog z7{q0%oU+2f_aM(Unq1|51(WKhPSVFCO8p?_ywt(rWq16tI)^@ejguJ&Fuu(R01--x zA0E6Ej&kAqDqn2|yT<$M{FM*p3gSu7#8s>4-#{U7yCa(vHrCZNyu@denqU-H`r&18 ztzzik;P*6PQCuYMzOk~_aY6WZpr3;y+V_OkEx)e{Z;fyp|MFoaaa(+c&Mt~QAxD-0 zJ^p)0DFuL2{ud?bji9WX*I*@yPeuD13AX!={V#YJKmaNQc|ktJ$YDsDWPdSO5W&T` zt!}f=cSqUt{0p}@U${33V_AN_yzzs^u15v;leGVu{bIm}t%v+q=O5spC2rNQmD+zE z{D~2jkKzug+A0PN&vJrrC`z#Bx$JKWJXsq9KnC;oH#;xDLMg~yuCt_g3}Gj!Afy+{ zlIF9WATE%ei{yjaX%?@F6bCUc?jTTT{5rf;Z=lfbt`*}u(#aT{AG1bZg(cncO#)0` za`o_`JW0N+Kd>O8)g)=g5}?Ubc?i$e9pT}N3;m|pA$9R)w;lmw6JQqK3`y|IBe26! z#&oN2EF_V|tSvXG6r=|Q?bmS*BYPX;w9lVX(J;vDp=w3-pj@Sz`rB_%JeDgP;`M&b z9Ye)0Si%>cbB5K}^5V4p;oVHC*bQ@4{TCygl<+$cCev0Lx@9U85ce`eGM~^m2kOWM z1e1nitfU6rt8Z}DTXnStG>vt{VZN|AnVb$xwHldTwmPI6?Fk4T909#46`>RjAr1^& z)-2d>41jVJUi^#_h91JsKLPx5PhaNz;?Kr`_UMK00|Ue{svISG^cdQQdYvczAs!NM zr=iyWv@3s7vQ-{d5wRv~+)vXq4GVN*XR2|$Pf)OPgAN1$qw1+#A2-wJ(l=kQUAOQ_ zfi;m0J{5vFkhWqCZsPWkAubtrMaal7iW=#!aFZNPjDaiwy_`L0N(&&ete|^LU-{7V z2@^P>f8&hTEJ==8H$&2VW%86>E1MQ|*i3Y$_lQ7{N{!kVs;IkXp5FC9HQs}@QLc_1 zVl#UaluP`jJE?{aN3FeX=fQ8Cej>CQofmjLyYxo~y#T{f5bpR zxQqABZg4rHCDv?V=L`@9^gea%?N>?#v9*9`_3Lc6H+!1W7*I&tf#3?-}zJf15 zih`UxtF`JZouh%H#P<#~g`ZPm{y7j=lGFL3guu;@CNA_1bYS$1nqA?fn)Uzj(U^| zkf?C^M#FIh>C2n2} z6R><;#=v+RiEfDoZbyQFm`K%6$~~f>VUPzJ(EBlbpwXGC$uZKGKSyv0rYfdq!B`?E zqHjEIMIgD5j9r@a1ik0@Rz(bZ)N29E?qbt~)LErTGc(j6%bDYbIL>*$-5l|U^3lgh z>T&dcX1^j0XbgH>7l;t(p~=n~1j&&KTT2qleHkWQd}h-0ER7-}9}C&P7BxP=f2OzU zyNzv`@`Tyy->Fp4g}T|Bs`J&-G$1pDpHghWt}&5&u=YKYjr?3CEP3I}AX3FctG?d* z0uf6sdwC50zmZ3tPxofsl6{{FiejUUVE?bZW>k|n>z(fT%4;k) zcefqQtGD(tkO^p*B<&B$jDHIAO^Lf%T;s?(mDZv0>z?L_I81zXZaIh$wi^CrRf-mvMj!~{Cl^=cSaPzX&|}f^ zkI61XN}n0A<%O&<{`G5vaOAbB)@)4QCiPom&Jp**{!EGP@Im^f6+PfYYuq9T3C=A<6)Upo9Z*CSlj&!I} z@bECIF10_5@S*U$CRg0+B2zS^6em_>|H@yzpdfxTi0}k&#o~RK+{7;?=%)-br2FjI zvmWf%h+CRyZ{`GCOeW>miO?GJVX1#SXd8f@k#h|{a(%zF=ibEcu|N1;lo^HWeHnxU zUc}(dL^`~5P@)eEh9e&^Ox%rhRe#Kt-q!*%?eKebT^)SmplKxHONo!V@gaJyx!VXe zv9u)T5wBeGa==`{(wV12dQT#DW4mqzqJrwl9oe4c$648k9icDqDQHlNi@Y zvP_DPjFi2Du;1@PdjNt`Cn`*rLVeo1UWV#mPiK`GnSp!)cjNnBiY;`SaBOf(_wr}r zQB)^&*CuM<<{BTJFR=YrMPa6r~%U_zJ%E2`%STT&ys;O8viF>f4?(ufjejL@pE*`X)jta#6Crg-z2%mN9lH)~@eB;Dwexu4$XB)0!RpfW{w-?uW+$N` zUyC;t78#`*j&n^SbEDG|>)rEpbj3n9W+87=Q#tExhef|E)lei6gp8nc4wgu$+7JzAH%&BvYv35ZRwT-*!V)==G6wG}->|Sm@Y=pUisXdtDJHCk(>8p+m zX$|kPL-^O;!oR9n z;LzmAlYB+Z!_B?9_c+I+GgClxeX~&OvOKX3EirW643MP}@@dVCuIftbE40?51EOML zKUY^@h{r4Oi;F>DGU;B#x~W`@vuS8(d@wRF&14FM>xQAKZyro&O2Cg#fDsE_X5n3}5832d0TVpolm-;g@=OvgUE$tTf+xH1Q61 z6^sO;JR~4dxE)itc+sRH8xxLvO;QMZ3R|L0;Zx2`VXy6}XQhT5^tc4uE&Z@$VN>?Z zq2sSbNqSC2JpNXxLdJARX?AhSPXFmfG8*kh{#PWghpHjE(G9X9S zt`1IffhBdfX(y?YJmx$6KZrRn*4odzUG02d{gI7FWyL}_=pMB(c?PxMNW{X>5X8z} zhnx@v(;2Hi&YOeSGbX{gr}x>|*8-|xXxj=!g@t1k73^iWB{vedJBv9k_1ymdHrl6X z2C%+HJ!j030h?n6%07IZeap7UzzJ2$pTZXrCtE^5CVk4dpgR>0|0#@fX^&1A*=>Ee z(ywb^kn}K4>oswT>`kJkr-HUbT6Uk0byp@n2*YevB9c4S@2TVVXDyMBE(E)=Er=%g z&~^QHIP)JL?-Gayn87uz3~Gl9jGj%xh;Fr@0CL4F&kx{+aB6uBG=oPtzjKZhp(`I{ z=o}=q*aN+dCI{P%HND9lO@AdOmW9Cl0K~9kNCO1&fSAt)a0E`gz+pZCP%*&5VO0~t zZ%e!38Bv&5M_-bvoAXEsWHDx_g}lNqe6rE&Sw^1@9ru+8@+!>C&NlwD_RjmR&9Rn5 z@9~O(9WVNezrTMapd2!R?DyL--}TQ}-8)WBOnK^#yu$ED_yD*=9J+HTbQj461z{`- z$#(3KkXT`CK^2j5m#7V5lI&;YqS#Cz8UF)24^PR>jtlyG_}azh@j#N<+X0y4VHm9j zKAj;NnKmnvwY({gyp<8@#`dt}Mj2k?D$?p{n zaXobJv;V9vNX4{~L0o`Uql7sqQMu624*+jhDL>0}erRWikBV^+mym@MHvy*o(MDOG zAVUi($ObfL40@*2uYP$%DR`qG*XFBRkQ+v%WX-~d<|=9}7axuewc{BSoEAySP|zDs z;&)rtuMHY#YH3}~E(0np!J2d)!9zIzGelti@HcG}2n`HMvG4gY9lM*t5x`2HNGIj( z9$5=3OCeE|XD=wVf9=@2F{uEoB8HiQHn;_;es?^H#aQ@z%j7@YEu7YTp2G~(BmjKV zdb?tzmzk;EbEt${%#I6E$W1C9CAIF(IQ-bE>M*fr^K$Y5c3oD)B%`%(FxC1TgAI^4 z;2?wHk66Qkad05!`kN+yFMnLIXU%-dHi1*@Tzq0>x|v_TEV%|}zobAk`jQ~fqO2`= zL&tXSi4?;VK{*_TjlG#vl=;4`Tx<487geUVpZ-rH2`mlM_+0hUt6vwubX^XPNhUxf zds^~Xi?J>|sG`;@%27*^H_Z(XB!?8r-JUOhzuF54+}C*ylG;;`8N?k|XlJD)+m32ycro;2et!?)AwK;!_C-oQktE;pOE@Lw1SD|2 zY*fP?Q?XDpzq`9R_MHoJ4S!WUwoHf$E2?%5PN!qs51=mgPEI@<#6}U3<$=2n=)CA7 zOK^q9I0c`i^?B0|aC#^~XyJs<_Nba6m`VwO9=cL@8rmnn;0fRh;(dCZaj~yN^v{sL z1Vq&pFxNbq^GTK#xH~>{1$a-oaDA4CW#M1%?~5Yto!g>Jmkru`Uf$PHH1(DE;oehS zkXn4013keU&EqFlNXBVHD40N@ID|=mtA3xPUW($p>akYEh8nN>lOHyMT(2Qv9gjzb`LZ3T!@mP5!8ZdI zf@1SNMV%m7bTs+89e>dq(Eox&2>n8sAq(_Xrag{k$VJ2L`63Q|Mi zh=RtvS0so}uj2+eJisitnclTXBzNuF^67)UFsmF7L=Ww2K?zJPMuBhq-&_2}MCHBe zM}&hnYk*~tqz|P9{sA2iY%h*Ly7cKDnwZFIfQAMV;Ee98+M}I>`IH8G`!^3Hjf?FscTw#%HTs!C z{)Ak-yiGv=+@9h0c=rY07g@)64#3dPQ~K*p)nz$Yzz}$xRC`|AYY60jf?S-ON6BbZ z5udZOvxUH*$Ko<6l3@a<+8C(W`iCVU?@uA9d+5zfx{)Ka7PyNK`YO7@m3iBNEBmzj z##XRB8yW=np)bHJscnyfDakNR{SkK@!I2;G6;Mu-OUG~}4X+L1NZ0(&W3KXrfJCxh z75Rb8wBGO4y-aQKupdA+6{anzb2VE5^2oumEdox?`5VA^VBpeY}aE!E@_+LOyWCx-G2J*RC^%%qCVswuACcPLi%|CGV-#I|a8zrT z=iYpWiiG&l$E5jN3oLp?R{0CbZ>&@}*j^yNIPZjn^xSBF^QYikDJoGUQ6TI=bbkVm6qI`oOQL2w6-a_&Q-c6f#iB0ut_w58K6kQXCOO-0WJos zP7I-6vS-HE#^}TfAkSPd$d}SJn2}3HP#`Kre_^?vz=?cAAZ20pCo$W zSy@&GVU&6WAB@T}nUo?PjgfxXhHD>5^BEb2#;IdPkk9eTSB=#{jTh-4wX^kLr`dz)LW6Ovfse^NNHn=CIgj7<-oDnN;0)iY z5t$7G4dS(R)zz@4?US-@-Fg<)Sao`@sXj#0U&#SrawtiGBQIMm75w4fy;w*Xc_0%k z`3=@ry8|BjdlK{suipahoOEWC{|7QOIBQDK$kh0s(FpuLO43}AZ3*UeS{)sBz5b50 zB+mxX36SOL_vAM&GC(v3#m!As+xF$j7AcbzH^>?-$Y;y~+=Ba?vKReZ-KfjSo4dxw z#uyUf%hVzMA$oA4WCK5WvHX^z{>~=2wdPbMV>a4PzhU|JK?Ml)D}a z#rOth=6Z!yowBBnAHM`Zx9g@^XP5Io6kU|2b=Dg8zzh+t#PDL$^dn*qF2-2b5HO?} zhwiNOl2o;jZi%c!=mC6~WiEIuzR?SxKSo$*mR{5`L(E}fFu%RMohQ3mf)=;KQTu(9 z&?=5$j`XLnJhQYl1!AyruD|xp#tDUIrhXg#o}zWAUvTmD z%>;Y07r)Q`q9@fjz)}n!soJg^0s*Z+w&&zAA%HS|zByZ9@CBSf5`gdhh?;rWHFF+g z7W}l;_h%`|!dX4f#X*QzQtpME(%Afk^x-YKHQGU1u{|=e@L2GvyKa!QQnef&N0pA& zS>9-o8W!n&9b6i#8A+N2!qm_B{{dm>#*I`Fb#WPgd}8q|j9_Tu6A zV*@^qv4*tMiNw7r`AV1L(KU&lU4j41!sTf*jaio2RBfPHo{w~H%n>*28tYt8aCd%S zS&JOw7VDgoKQ(NS!ho#AoI0e;u1i5}GOkX&xbLW)MZ*YhUY&JQ=bdC64(UBwZa9j% zrWq{8x|PYtvqT*F&U!)(bnl>?NjpqN)CUX`7P=6^Qyct_+LaTae8+c6CoM<;Jf-ntizKqxT2Z?B-SihS%A5x+jj>Igs0)GF*}PRNFst z1H&iF3JZCb8SN_rIj}=J-nr__U!wH37S$dy;bNAXse;w*fF9LQ($VK@y)h;>)!|?( zh>bSeIr>}-J?KvfD# z33%B<3D~Fe#>yRAJ~d%ft@ONy3eq0=4*9EC!>EG2Bb9PLN8p~3&V5DoJrvGH9Yaqn z6^(YbsLUpA43V`=O<`*(?_XY-i5Yh08;SNnvoK?ISWO`z!X4YugAl0@7i(N@LQ!Sj zJl}gc(O`~c`FR%~)%ak(h z7k`T1U|sEYJ&T=|aFy>$9Tr$PyJDn@KHZGm4Eg)tuJAW;8l%FsM=ks7a&72qzOK~$ z_7#Oy*i-}~)zy`Ons(Q#MMO~N74u%*L}$N(wY*W8E>Di&-~Igj8h&#^B)_F|D`7MI zK$2Syv!p66~7dL7Bw+>Qlw3Sjt1M3G;2uVYAO;?Y|RPPvK z60#ey-0PE2yG#R%{;p>X)&#FMYf78sMcqvfe1zVGNrc30PIMDuk5t9yD7CL1&^eDQ z(Nb!B^-5jeJ&)9~QYVY?V*Ad|vcU59@A;dCEl;yhswDw%FD%F z?XX)THA)`~^QzTBBCSrT!m{TU@J{GD@p_2gm@zqz0mh)f935RqOFbcDmivA2tZPb! z;eP4zw^)JOj9RPE>3Ul=qUT#3Hj8nWmpwz;>_jZEb{5I=PruZW$_7~S8t`npt}TS= zWe?40lO${w&>@2JD{4 zH5gPrccrF?#zE*Y@6%)VLCmKk)K7`1>OJ%q4jJBg%)A+xxeN|iSa@C;NWkG3KmqPl zoH?()-U)Kk&}fa_FkJK*d$=-A?!rtLa=O`M(5gDxs%0LUVxeA9Bsd= zZ``SIqx(cu_{BS>+R!2GRCXzV0ZDFXi>L27p@zoMbx$C3qdOM!TMcV*!8)$rb_ zW3P@Uqp8U_OkYyX!QjV)SJ1uo{lBlre@&Xjett~z;4Q_1h^suxpJ-EfT;##C!ROY0 z_S<-3{`}eJD9ibRguw&G9`1`{^`lvHDpFz5D>Y>0I@6V;2up^cW|2I*A#ba{b*3Sy zkYb7SD5aBA4$a5*^42BQHv8WA9ucn|`=KHgnS!s+59S4WX+8KahN&%m#c6dvrG21b z7Nt$%XzcwcEhlvM71U}&_MC^V5An9WmbDek7o-3jDL$(X67R1NGxSa8>+!Or*JJ2SfxK~j!~hc z#Cd}G8+%Z#&H-csCO{nL!LftC(q1Rw&yN;=PB6cf)?Vv&8O~?sAm;fU zs%@eiy=`7A{<aM@GsO0kyV~2O%>^0Zv{-}uJ=Vn{1#7+<#X8H+4%#Gt6%RbT$?U(sf+5c$wemzeMIz1__S?%ju45He}JiDdG#e0%)hyxYt5XbZbtZx z*THn#H^}y{++CywojAvA$laa{$vH{zF>jP$KGMAU>D;X+d!TGLJl|1Go}MZ#2!3F~ zqElXi7P5ZycK?)c#ijZoSuRQIKK)_Jl&Xe#^+hs=okjeo)hEcXU46sUuu=5(D9`gx zafC0NS2ElepD1skgu(z6%)XWs8g< zA!46-`PiepN>ywfdJbAZnJrQ?@wnWnI)deQETAVUzb zhlXMeX0u|l-433jLhZQ?Aqv?upJDA|8;)dUVx=~w4ZY8OTa~`O(ADJ)^w?K$%ww0W zr3LW6S@E5Z(Cu_i^3U7(McEkT@mY0FaxN_DX{d9rq&Fj7a7~ngG(BJYj%~B2=Hq4G z+ju+SI&AS2(S{3Pj90B`QuZl z6YDpu;=9#y7xd;d7>{bZqz60}d5m)HUct>$cBCc-NndtK6$LjJ-x+A^uEkLamDa?e za?r_59n!t)&Vng3!W%f85}FaeI8k7S?%!TI#!SYs+tZT9;~>+PFl0dg{CI`~lU~@) za>;k?-C9~|YH@wzDk}wSBfS>yg%f<=cI~&F86ZQ@F`s=22B}%$LA)dPb@=Qf@#+6`*xp{dDkvzJHO=WR-W@)YG9!H$u8g{B zIug=}x?XddlLg;Erp-GKv>7gKP8jd{piI0nGeN(#`!mK9$VnA#by%*;ZhfTSK$sk^ zmgM+NSug`bK04)AaTW6u`-RRTy{;Tq&|;n9#S;oas`GpMzQro}xxb>5eUsIBe&_W) znbjlK_#s_SSc(r9y&&{dD|+LOwqN ziUH*y3SD_a#s|q<>~l)Un0Io^v`D*nADQ-kMa`hD-NO?@$sWsA*D5+eRQ-Aj} zRGoEfy26&sfP@VETEB+qr;Go4F;b375l@W0_y2aJ>U#Qv+4w;iqfh_iwrfk(!<^}Q zEnSI=R3GY?5Vt!?XGzoCP?YB;A!|(GvE#?YKhrLhR^)33mV7x!S5Av~Q7iX9{`W8l zvf6po7iGaTmcp($_Bp@JO-$alpWo~?r=_JiQEkQ{xfF=Gx4!t(5#`K(9`1<4*NoW` zH3Tb}P{Z11bd|6Z**l`~?in$9cC3J&JMkL>H#fJHkL6W@qjn0CUHf;|D!`VmEEq6% z8$#x5ui!dtBe74lw(a!&Xo$>~xe@0fW4G5Azbi7f@^Q3gmP?ej;hR451oiAY6u216 zy$LQ7d)G(NgwKC=U`Q{6eY|b4(TkLJWN7&7osD6d*4N;yt{b313eHS6JcJn52V+!= z42B!9IQ`d{|Y(#A*Njt8$fAq@%n@j7(B66TJRb@6l~VMC%lv73W<;HYHu z>wAYK;aNEO)Z3h$4ENj5b_PEFo&IxdnD*5f<4)sN?^UdY#UQO%feY>ns)W9j z&S-5I&&X^UyXk#-$*+NVr)`^bg1KuajT1q{8$Rjez*;hxA@GC3!d*k7FQZ8?GCOwZ zlQQ%4UHDA{;e5GDs2$#DmL#aXDy-rJhMAPK2v0GDk;D!={*p(a$v%Y_x~O6O_@!AA z+ykU-kPYX`lnMOE`;gFh3Qk;Y3VBRq7?R-?&8v$gC2`u%p$9jNlSx~@)P--#qPT~K zm+e!J5}zL@Fq^kNI4olSRPasW4~@{ZKXae(UisKXAV6%~M#>&D1%s``34=HpS$^Pi zcmQhAA?kG|<>aZSmW6~HRZXhRNs^GJ^AxtU)|E_*79ys5$+Zj<2W+FTVcPA+81P5- zWJB+DtiT5EtpvsgtUgcje2k5j!&c{_p*w_;IIk6N3svyUfKG|xcGktE<V!7qIuW zOi~&t1$}SDXTJDEe$F=bw+ub)nq9MMKg28~vM&I=4Yb6pTP$WMWdnScjUQT=Qwx^z znC&_(-Wfh>CArDN%bEIoG5*ruj@%qvDkmekHEP^+w%mBrotcfCL77?e7yX4Z;Wq8-8ow1jE@q+ku z-`9nU4bO9yB=y6~EjcQDI>F*fc~dkWp3GxrQN9}~)t_}-NNWflGdF2N-39dt-}Ahc zH9uC8_Xfc#{X8_kk4hx)Y>}?kh2D>aX8wK3t(%+W-)Cw-9K%f)IEiP{)=}2QMh5deipqr2;v)8Jmr2G))`U0tWda_+jqw za&TyPljoD2TX<92WIbNARsRt-gyq`uzRStcQTui|59Qdh6hRY8ksI6@y6s)5!JKl> z1*7-qF{0R{E_E>5Iu1%_Cn3 zQxm)c4L`PjD>NfqHYQHU)Y!ea?cy3$EkKrWd=@B#yJKB3=VLa`r6;TPU*@ z5Q-j6n?v)XuejfZ#wsYPc<*S;0_}SUDRCu=p!n0eo*AyDtCo4S*K1X5F zXI42H!~&CG&03#@y$NLoVF39Cr=YnWi%Nf<8VhfYRqV99S6!P&Vj-#l6Q-#(@O;uD zxPwaCPJb%HTjQ7p5!~GHwQVf(K6*O^mA+5S87-sWAl4}UC-APHQoH`+5nOxWZ;m3b z)~(g@?Sxx!Ph5$1OZZ!ZkJ^o%FjAlg zJzLZ{cFt>-FU$G5M~vBF;jxjzDa$mEy8kBz$$mXqwe0VSLDOOqSeE&@iqXjrD^!?K zF_n$e>!xy)=V%))R&+V;n;&>=-}%09Q&9LE(v(e9aTthlfnyzSX*!K#uR#y~6{|?F z#r$n80Vn;T!1%KrRaU%_r&?eudwCc&)EVUAv3IsFSrS&NpMhj{Q>|#! zlcC`*GBTv7-jl-{>cZN^+VsEd1R}1l%WqbI1H;Vuazdxy;^r`&4NPoprsq4={cfKX z^K9i+H2(4pn2vzbC6@nm{3{_3<9?{)&EM~f-|ZsaDD=3u(>%I;JS}cFl0Wro@+s*C zNPaS#K|RI`j#DivZ`+@FR=n@6DYYj7H9_9C{cjAxqqp|WMFsq^w3iG!o(|^Q*}9om zF*Vh&(&9elV!eC5%PqgqnRM2%7x)?!lrBsi_FstkVdrFH9~fzR*8LVA?k^hv3&Cjq*?THjWpdts0?@9TrYRn|PKnll>Fko+HI=$+lVqe8 zk|@#c^1F1WBU=>h4h_;a(X*?N{o)(_A|EoZ|0&Ppqs5=Q!=5uHUSO#eQ)>p;MF6-#0Cjw5drm^as04zp6J z#PcnP(LXaJO~mf`-9$%6?`PV+Gv#-gZx`R#`!!Fr(cm5-aE~TA2E9-Xg(2u9F-j_` z#!A9O9&cDO=iTd3%oZjVSAFj`>aDMfp3<0Eg*=dB}&(8 z%#e(YYn;&lwZKB^AQUf9NlpfqHE)YrfLy3~o;p^WDr1Xf+GkyE#k z1yn+Qp2vV{tJMi|+73XYo~-$ZzV%N-JiMxL!Wo0ycIk0XBEI|^q09Kf?9C3dw(IS= zHn9Vswdw^rp5O?R$|nZ23B4X}hjac!ytFf{+8R?fm*^jRG|OXwL_0k?syVPBu1d<{w=y5Dd!F;DaU@yxEHz#cT*EnMxf4NqARlC_M-hBT11D}78BG!_&T_4N+ zT>Pv=jk;{^`XwSJ=INR`xSS9fEk4s6~y=CtH zEP1!Y#qp&e4-bz=MdQ}YAq3cpD3B{ALicLi;3SHSeGco^qT-4y4crFU^+=Z-#ZNqMlzv`cD^3hY&er`5?50&NOkTL zw^YHhIDAoTrdW1Gj-{Xv7CeChG3%^5+Diuq(&pcXKU;5au!L9{}MskyyK}j30y!|aLs5uy$#a{0cNGSfQ^#$i8h?i8?x+H zNS59G;d_<3$n1+%^ae@DrP4fos=oObUEQK$ zdP!gVpL*GAYsxL&mv_S2vm)i!(P3BF=%h@?cr^7nT_(O*|GxJr#d#+KPN-#WqnC~u zy}UI2*n;`Ky<=D;#jW{mz))9K2_vZkr+2pOS@e z6{n>Ii@3Hrc)2HQ<0sg%$nX08`~5#f_B}Y1i<8Dlx^^5KQ8EX-@ePq%)Hd_@^sc2B zXlF3L(`h@l-5j?kAd=p?^v{*hwXkIp+Oxz~La1pIo z&wc3RPftdEaWU(3S1{_4c#cs2@D?&MPC%*8r6ZWh71vd6=C4|M)$Z8Nn)i_WkJD*E z!hTUP>p%5r4<85zuC*;xR##PNG_l>_gf5ci?xlOFTrRJ4FsN>3N&3AFBHXzyVwZ_P zD_(OUpmX5tvWeYTn6|L2B*2=WCwVQS({(!kb7oKe6AkelT!I2RPT$I?|eH2HpCx)rywz6paM!LOc0S~bjn9UY3UkBjNIq}BYuy5e}BQ=z3+YQ zbMLw5oICd4!jzmn|E`9nVFqSqq$^FmA=fdJ<5^Qv*H;pEf-@9nq#lpiZ_xG&x5q4F0Jm%_R9;)wyPM9Fet$zy*R2wr zv|)h1X8RUE+eaX4O(@=6ddb_QlT(~seePSthN_aIaYve*ED4(XL6}Ugj2ttUG3(57 z_bLLo+>h7Z3~yz?r7L`!XXq=0H@)E`U4Lt$+!8q#w}&}xyJ8zUAmO9y8w?TgW>1Ce znb-afn;=UpIDAq>Fa%#DQcOLZ#P~Kzvm0qfbINvEG}r!%8{P=tg8}InQtH7s3}2+_ zcj*K0*3^fXh}&M(zS(yA@maZ&ts(n-GAZuq!>u*;U{Me=3BNQ}z7($slrpKq;YKev z0hSgB{CQowYKD|1hqZqhAuU4c5&k9;+1sO}WUsqodt0J&1)px+f;9p04cwkaImyavCeUuB>^9gur9HsL{|b2ivKelSAEGJGflPrPY`++C*b;gC-(pTt!_R_sD zmFc*>c3HA$`s3{~^rp|Rh8j#C!@|%n-{&N01p@OKBM^m}l+kF*sB0T9^u>@^9gy~u z5OrmTMPYyl;R5v7ug*MWb99#*SeSPoX7liKMnmkZ3PJ)tkf+@Yy&^GOCkbhx_9QlG zDAs-}@@&3%0Oj#?__kU1mEB@$cK^+HnVCJx7uiE&k0!@zKjV-LM4jWH=Y8PJPQq2= zB~MW~PGEM+7U-Zpo6(RM`oQK=s7iGrH+;pfyOiZFnPgYhNu@-zZD)G(7*^s_`4E-Qm7X7Zn}CwEE!OaG{Bx;-Zg?G!$(vDDWk!C^ z_ovW4)4~zQ{kem|QtL+NN0u>?E-{=Z)*xnBzoP_*7Ib=)t};3}nmV(=?veK!q@m2& zU#AwhXm#wQE|ND|c3K=f{CcUQO$lA{>nGY=I&>A;jY4s=$P#v0Pn&+G4BfC}dpKFG zqR!+jURNZhxnD0pP&1jQaJiyyDMs#*9__rl$JTeazFC0P{a(cs>ft`!LnmG$C~^3E z-DMS4DjZoeox(H?TB8AJ&onntrQothdXxy%*9>2-xP|M!I?VU!=$@bh^i_atr zk#_E(WqwT&N-6g%fQ0Mlrv8T9(ak2`qeBF#)2!koQ%3127Z==iF17cg!EYvdD63rc zDj@Jw94TTtY@Q?`j%i(X<}}WQ3DXQC*yO~IAQ_3_&Lo0*7&y%dXQfOm;kGzD=0PT0 zbx*S(f`-VtZo>U&;3Q+eBLO2_-xC>XlFB_dA9* zzAMHH(`$_$Bh@WPoc|?79my!ac*8LGhF-9ol7kl+tm8b5QlA9W3d{HAJT=@NJ@Pw` z?6DrTXyzn~_OY!BT%pNFVZ0_0JR3t7=pgVRMePN43E?MS3pJKQy0b2y?O@TkO97W1 zqcX%Jw{gP10&GMEzfn1Lta`T0p9um+Kj}hPH|Y@3Vn_#zrgMfU_1e8iqZL`x^q?KCquoGvfr0vym5n1v|;!AncvogG-71W!H&Z3iuMNGa;HWvX+Q z+-yjg^D_sX(U5@lSyM4Lad3*pI>ky(WaGpP>C=(l3XYDh2H~qh)@1-U>+{uA#Cyyw zN$dS=&Vr1UbS?$*#@o1~312Ys!+TVB1jELcLpN+**jVK#ID8CeA_VTLO}?w|Vd$1$ zjw>n*6vW^~yElOCy_++R(N#SbI9nu^TH(JT6QZY-u7%6Omt7sM(m@Pb)A>sssT&RV zKa-wQtRu&*SL6wE?aRWN%8oz&i|v(keHW;Bw#=(AUbeI6`xoZ(DM#N z$F-FREbp@>UtQlY{9R%j6L210_rGw-zv?!Kxcu|y&&d^T4h2*d&eWl8(obA4R0JU_ zpSm?8xjJ%vbI9kDr1Tzaq!tLcnZ!e0JTCI=C|jc8|9+dU4>Z0a^%EirmHgf`M3-Q- z)o(FiqZ2G4daz4-`r9E{Jmz4<3QC|%aZc*UDVd)C0S3bo2et6?#0zgUquHI3j6j8J zz12?K$B^$1E$PpwvAeyCqs%QP)r+7rd*JK0OT_j`~Iw8 z3FN|v&Kg_jFJ|PasqI|Y~rj>M4nnZ1pV|eRooRe$~Y`-b)qREe!T3KJea3yXtpqeea zlQ`NMkB1_t@Uw9xK}R+oKj2XU^CRl$#RI^8QC6j}EB*R4L*;yA;3({lP|b5P64e{K z{ENMcP5j0-5@0@(=;`oT>_gh~dpexYc=q3liIJZuOAKrCsen@#0P{#z5FO%amsf)Is*E8)N+Kmu+ih2)8Dm{10t(2xm$9PZRYhOE2;3rV)FLUIj~SC& z$uKBlg%O9n9-FDV9cA`RDopnD?x`SwAwHTrf}a<>G#BLb6$3XvD~3SlRGnjy#YWtm zrCQd1uHh^M(_A5PGbxCa>FV#jY$K6v&~td279tWbMm(TXUCh6FkX-@54+)|ZJ~kur z6v9*($uty$ivie+1r5=43{&Phu5$+AVhdKbJ-aVukyh4el-xlNNnH)yN|z*)y&>qV zFxB;uePiE;>QwYpFo1g7?7_cYwCx>%wVM6+H>hcI3Az)K(LZvq&eNudK~39#4T{>$FLW^N-840QTZu$A6;N-2{Zeq#&602C>D9l_NfL^&p}I zgCl%;dy$QeAw7=~tn4UQ-7HmkdMiP*nr>XJsJ{(AbcRV%YzS?{B>3bKZx;`46Q zk-_bZ&RZVBZfz%j^qp&N%zKfd?KLx%<*Bc)zu+Z~kWNo9heE{{bbjT!jnDK(mueeFE?VSlE1sE!!HN}lB) zDL<|CNb+69Ah3V$$5DkwO|fG9k0kl2eqQp{I) z9KJ*EQMmd}$v$Pz*(ig7g$KF>@bBeeEC& z`PYDZYfuiA<7{gnlN*V*L3b#m>2K>BQmaoEs4|+(^8u?19%(1vR@r3PS4aW}f1_B= zroNVyL$3H?;ERgQub;m@wD0~9HS?Hmx?7jO$q%~K-HXLTAoKt+x)D5)7i+M$(|h7U zf|lxjvt_Go{Kh~5AH_8@oI=$fyvPW7f1I`(#xZn|2x4g1qIu%T%srQWQPjK|jt1t` zrC|CrEh7@P=mbZ4SIHB&v0TQ$2sZD4k+{B@GSdZndThvzYz75#*oQ2|h+|64H7eJ^ zHS-tHDlpC(us-`~R$A|dq*bu_j?vFUvCYo0CV|V=H#fUSmbADqMMZa!*``7KY#n?( zh0vc`9i}7c3!3jO(m@ycos(>^azdnX;`TAejY_+|^tjUlq^UI+yCsnw*ux(LlUYQL z>v4@Xp}DG`(&p9_KxvpC(oltF(2g^`QH9+$Ayt1Hu^8V7lvy_RxGsEJ4Y2hR783 zm@-c1-wGpcmmrJxXKRZt|^LvW=XWX%EsjGi1!1mRM z{WpTi^$Y$XZui~XO2@{oMceprXP>*4FFi&0%!3a3kravJk)i_fdiT}lq66g7CT9KuFODj^y_ycB|uIj64P#`x|qs#5G;Bv z*7|C9haoFKr>W<%1IXlSkNIt8>vrzmTQ~s%!9tyTue+AM9VIF+9-uv`*LZ?0O6wrJ zbn3h+NXQ=k-=0ZB!N?*l{Nh*Ii;-JVK2e|M74ImL1&VSD4d~K*s;m?Xn)B+BK0iP= zZ-GwErMOYsWk(V~YUu{If9KvWAu9KydU^-JO3m#@Fte{?{HEnoKbzeL6WQKS__ZI3 zqp}U~s;WlNp#%J%8g0$^z|=J3WO^vvO2O;Tf8DHd_+>t{c^b71g@9Vvz(02w|LT(= z(Pbp0MTUpiwXtasbiogmt}M}%r}Vp8i@vt~oTyg;4ti0{y}rBG8|4)elaez2E}ZD1 zH&Qf4;lf>3+JD}_<&92F<)FDN!y-#_c`|>p2eM>ogbz7BVECLlO3nPM?d_P!B$mJ@ zD%!V=++~N=WyhloVQINV04CrWfBpWQQ)OTH+fvyRZ(~ke3Eg!hIF&v@reyiuvwVn@ zqbM29?o~2P#gA!x(fFuu=1hA@WU|u85{Q37y2}QqA2j%wDqb|Z)&7jej&k?;qKc|% zd3lhzx%v6uG2#Unr{0=}G9#5#+z}J(c~Pm>QNFq%NDvcW z04Tn}mt|=n7Bt*i_jK{5ZkdM6*89rihm|2fK2xqjw%|p8Ei%^`pZOX;Gd4C>?4iD7 zS$w|8)3Kh%ea$&WU9&!zf;;MrYS+OodX<84XYBj)XRR-PqzMD(siJMF#nM3m#h=8+ z!OpP`;@q~UB%_lm?@DntP;%{L_W8pBkA;UdPQTaQT^wGfTDYhz;=*5&C`T}lGQ2QO z#Q5-yDmSx&cNxl0m+%TIofo`IS`3Y_`dHCBxnZBP`kj4kK)-`G@wRu#CCWtK7j|GAGlX2ZbPHp+>IRHshSx% z4opWn&RVW2s^FFT(C#wF`%27oaa<$r;9ayuhgjwQ!hyweMfdX;cDIn1r*H4s2>t%14_p3m~C86R@sju$INZ+T3 zUUZq5xd$S&&UJR-yzqlZIfsJ3D^FworH?pSNWl#f_S+Sz(1-k67R zy>Thp5Ak+NSSz_n58hI?YK!rqR73D<&leyTq+L9SmrhVJ8p9r%9w~(yoWv!!3K7wMr?5QE5Z6`0zfu<61 zwO*KdQ%Xi=EoP+k)&|X<^mfysTov3nY!QjXB*9PQl@rOzr*ICzcVrkKsq!Gbv zy)Z$x;ManrO-QoxR3|7UeDbKmS|J zw6;=wetz=(fk5q~1&-Rb-+FoPzbzmfup$Zf;PRv$Wy8cXgoj_-bwL@p8(|NOi~&Ew zdx?lO*|Bw~VEKzb>qgw3p94|xl~bNU3IMfHEVkvuYXH5r8L;&Dak)R)UVpx76d?>Y z4e2N|!P+Ec-^uGN((=Zlx88kk2z#`|8XZ1(!m-U74I30>T4Rkhsg0;hXCc?DZ|`Ha zv&339)5ux86)%!urMefCSB%v8L{_pa&*5Y)7j>Qm$;nX)@!yWSb;$w9QQAu#QH0mT z)~^p)UVgc%fcW;E^bj6+~;`8PyNh38aX`D3v$zXgBI2L zci=<72o?i*-{(5tw&CJt`$+~yR3~lL=%58%FbSxbn3)MwL{~g%{JK|qg*#FLg4^DD z-8yoIT1E2E8RkkoR5XXNrm_@e7O2})7@bT{G#2wAW<^SlMIRMzuFLSy1*f)zp2H57_ka124d-ISQEfGwUj!Z(hzrNt?S|u@iM}v|+GE zVJaD&L+YBFO@cqLO@!G=5(L+O0&qq@C(;%s6Eq;mnyw4@c`2#dfhe4PSIw=E;Slk<BR^;FxTiNy-L*g8m=(0upyW3bSo}ETB0lWM7eqr-i-=&|W ztq(?ujh4qbEQ5FR!5jD+6dJDA{#*?N+G3^K?!$YYd0))-Rk;fc$<}; zV&Me>tqB67w=v3gES`7dPabs~!63Z0>8yPuM4#I*95;wMhsqGM78VxgW>)5NzVm(@ zrR-uhHuQ~btp1fQrmHz{3}Mg9eU>AMVoq16pgKh|DUWy6G{myKf}1{>ktdtLkD^*q zv``x7C*-N*c^g-I786NfYXCs!5yJ-?&p9PJHitSaP@b!^%vm{#LEaZbKvnPlyXgL& zorHOY!R@Pb$n8{YSi54jf&EMedx9@!I`kBNW*F1^>|Ov`^qvCd+;nH@wNV+L%1bU~Tv5tGEM*?^$64W4 z*{*8562Wy0cc5hCwY0N$Nc?NV2A!YEZ}L`O$x1xf%7}mEKxfi5!ssN#I{v*6D#m`o zDv@bd+{|%me?kh@HdG~v1X~Xr45=W;rl!_$ZS7#H>OzRp%s2u>Fy_{r;bP0AcGJWFiLoyk`_4!7>_Z0t@E>=-qB)>5bw-=s^1$QHei(pF( z&WbQGH5!Jqwg@QVc&H6yHa;=_<8rr0!-7{YpI-jHl%GhnNY3!>-rH$HO<^t$DRMc#Sjuk8B{QQ5# zbzNE6QTR&yXl*BN_?b%zikhF{ZwOSlj{a2h?n-fA>uuoM!nFYx;~x_Dx-JDVkU=y%5>ABUR3EUXVZM!Ip3ln3TxKfI}b8W zTPC66GIYte`3SUXJ-K2qynbs-1ZmYMoHZ*=qwKj3 z;r|@&e7+L*uP3MA+OO^G^=!cqR-{`9t>sjB%5GCLOTa;GUNKwadYX90Whai4EF3gp z#dfFDQUn`^H9SACd7hJ}cJm3~t(WWpKl3j(FtX+AX#7m+vSw6JMj1k-cA^adeCjtn z<^q=&{tlxVZr<>#@7?itsC*W6_=5xQjU)ft9Mkyzpa!A_;y~IBP7{6fOBUtylBpzy zE#+5`4}+={zHeD)FCj`lX#h9lTCeJlIJCD2v+X&cKdn^tu$y6rA{JWjp6|Q*ss~507q(@H z_UZ}uz{^-h|7TtRHvlSBr@n);$e3};ZyK9I<&xd3Q3kztb&}$JlGWs2v!)(4wNpPy zcC52C_a+JD{n^fz;Cjh&A|^I1C#2_&)rsWHfgfs-L)Q{mfUlHxzMsP`+#=`>D>lJse;qZIh&|1`|~`vJ-LU@CQDM zqtR8AZ+%zZ)A2T)2*Tf@^&HmuBr2)j!>De9{a}0gF!VFy*6VV7iIg8%qQ=tub$IU# zrCVskwpflhjwx(cc3G5P?!GR)DPmD8LKe;V4Z&e_J42&7)J+4D``@Ep=-WYvPy_~@ zPyn)>7B{36p|kOH^Ni0f@csm%Cpmc_>>DEUtnI1;?1w^}eL%K2Vf(Xd-k%+AWT#a-Pyjz+Fldc$#1Gmpl+yxy~j(rhsiZ*t|T?Z2#ur`eR6tx!(7H+l{y)H=K=8FSgPyjXD>+n4BrBEUSERUET2;{`_F) zKenmLGOnG`j6z@?7E{8H7clnyXRi2lvV~r3qsQ(_CYe>4Vb5H@xvvU`J%=*emnlt# zS*K_>b@)0$8*xWOyNnJelUw8cEVPz%uTh_Uz@X{qfsBwnfY3d^gNSqCu$#CnnFVfxH_$=QLZSEUTAMRadxPnoTf(nY~ zuFh48fylzgOp%y(;k?m^Ix({7wuzZb$R3)^PGIuC3+QFI(tnFrDqZabMlZ$O=cZ2P zmJPpzW6n=YQ*%QOO#vclWZ{B@YH}yA%2_R^#b1HV6~*doqhGO)kH^0XCd%U_rgPl^ z3VeD`VLEaBSUIajc)vVUP*bbqcS`t^6K)4b_Q1;!w8EYxoN&yl z)s8u6?PlYa3fV@vG+n}Gs+vTOZScp=tB5z)l0$yOth6lYkinD;!<_oB97gUV12X~)Ab-?ZMv+D#If+k#{RM&QRPUw$L5eyB{gB$(Rk6C#uE(O` zBC+?I^5h?XxKwsYb+}>ELT29-kHLC3S^K5$u^C{nA!7sTxsgD{h2KKGpq#U(H_6|t z!=cI`Y@O^j72-uB7Nvef8_$J3l}4=sI>LwWs7sa_u5X--eaze(2LF;4i4`hC#H97V zEbUb3zL2|_aDqKavAFRzET(F?``%!doXaoW1H&ibKXngoLRqCZwUAgd#+wW)dZHe4 zN^-xIbW(Vv`%+WKlrM<;-nnVGZJg~`@{Y3u{z2J zhmtk#DxUfsejP9EX$9U>y88M)9=J9Ocx7OP9 z@yt#0=f|_vvHaXv@U&}#`CT;RL%SE3?qs8_#byI~WU=ap7$ow8;_B|#s%8Ff@0Czr zv>fuhX7uTMlFesE>5x{BAE)>4S#HNr9ygJcr0V)v^}EB_j`DNqhrd`8Yq8;J^(oze zD0PfO!c@P?58kxFPyZSAAKShhU$()!S6xd>L!wWl+?EQodMd0M|76|xh0JO3{;~fP z7&yfvT4hdlcxW5$9QVDdQi zE)G;Re_L;a*O?7=fk*EI&>D;VU{0gkAGq$XVuS1^n+4rs;cS_g-)HPbXH6eI%%Uc~ zGEH;cBkrg*oKo9@sg9_mSyaQrrT;}wgn(+Wtu;v~-pF)O^-!3m`+mF+kriLhG+rbC za#FN;G~Tr|dY_vbDf+K8Aw1aHfFDbvex?YUIbiZSmyKg872~(k{42%><)$`RV+OW= zF}P~;YR~vNa$L~%r5+1MpqM45m~mY+(}RxK0F9?MpYkm~!;6?PBS6+_vpwbY2nh2m z<-u6o%)ubP>g4?NV3Lwu%F`5tOr}7P939gc4c!su!=D;1k`|J4@)EY$Pu8^mUOF;N ziB*vDQp}U6U|@$Vi9tm~y@^Qa540zz#=!$o@#9hCY z@&w`hh@*8C)0E1dF8D=ljd~2c_+-eZJ^%W;#{3ZjH4J>1NwU{M;e|>}1XS{bRGz15 z^uqj0_oag1-MdHj1IpgNro5J|Zb`qG`VLHkT24MuhdJsCzniiIk{r)R| zKXdjzitudpcR-P-rwHAVO1GR(NpB5-9Z5F_%w|{{3=aPu&H2PHAsyC~=hSr5`j7Hw z@mi?nk{BTFeL^lW+3=Fa&)c7R@>nW2lg+w!<{EvD|Gw$Ai{n`zU~8uM-$6^-!ll2{ z0E3i5&&R&`gVt>q{icC=?EMaRTZ5#XAH+=aQDAUF5&hAKh6o(YMX>-9Xn=MEANRbV zpLT?J>%$r(Gnhu$f*`S$3_aPk6xFZ9!{av9)l_@IYenYO&|n;a@5H+ zFy4e-By=pHhrA$a?!?>&f@;p;-_M|1baf4IUu=VyZKh$K_ldqtq(qYc|$=k zu>QCY0=IX$)r-{x>j&@!m^1DX^%IJ7x{-5?8fnPC%QkiP-~uXM{KTc5q<{NAET<|j zw?3`gde_R{-`3C_-LRap4k#KW5Z>0#Tld>l^Y2;cV4eC1P%(u zbLIMerEMI(!>eD({=@}3?N&CfzHCU7ph*#iROz=+V9w4`pv_ugjoiPWyqKu_aHgn0 zRM=kmrNMc7s!IR$b+-K%rltozfMGP5?=>NJ)C$fq_~n{XSD)b`*s6Fa#(vQnZ7_MW^IxXXKaSwA?Vp~QsgR8+l;ll& zxOdS}k+yEUpcBLciw8NuaLmH8A@p>A|J}QH3BdhQHsWAypr_|Wm0!1!CQL zht0I@jx9*U-Xu(YnWz=Ml_q}MuB=Rz8>fS>!#&+VI~bqDxt|=5{&7f45qlmT{XMtp z$**gF^^&mw*qjPJo_6Q~CVz}q=ZtM*Y|wD1_~(fk`lwVMr3*`}fy2oyuvGNf!@j%% z0lkWSJZL8q;KX*<^U&pYFuy4I*1_!Vfq##4he%_7o8^^5f0pr+2M>lxvnZZMjmf!s zYnLMXTnb8;m6qiS3z_?9fR7+qCTFfofv=CbK9J3^Qq{}PJm>u-HP7r8p&`LF4tdAB zbes)#Rl-K!V-JM0!v1R{^XCS;(R6=LTkykwgj%%*8 z{Mu~Dfx9Ib0{f&PN+`-b`srqeWcSIujS4d_%PpXqCIrNM_KnVPR*6?7E0JMEaEGfC zcRtuL;u?(k_BE*ZRvAK^6O^$N2Tjl9^tdG2mssG%SHp%R76kBd3^K|x&Cu!&_V46^ zVU>-Nq+Fv(m$K032)|6x0opijfvXiSsv>2GjH3n@-%o-be9pI4IePuAztNK*43_ zLw8oNilTQ4)btz`#I5{;lssWeFiW-escZy7NX5n@!^MWap|U{9~}BVEwMTuI9TPPwyEilFXZ(M z4iy*bTG}%hSe|ODq%^c$pVmcU$25Q#c z0zP%{9KJe7XKU~lQCD0SEv%ks&d=k7V@40*qq(d%FQMzl^1PuolDiDMWR*k-RP||> zDeBMds?nZ{fs3JeXeLvc=UO zuOVYCG%1d#?Hrfg=dBX|-W#lUUdFd9Q8=7XqQ7h0Q9W$B@*nJw;BA^LvhHa$hyya#q81yvUHNXbwP3F`waU-2o z3&uSZY$KFL-+7neaCsu;f;uZNdbl)N41ESw9oW8**iCfv^ggIStn*rH0plU#(SET- zMQq3W*xpz2xBA@ui1K`?%kJfLPFNN_h)Z+a=heI;#ok9lsa8#N1Kpk}U#zN|>%3^S zcn`vG`rxJ7jgD-HQh!za8-3k*+54PjvRiZcA!#?8OQZa{v!s_i1CvlecU!lcBbd^4 zgDY?~LK%IQk;xsBkk|Q>4;oVo{?;n9GoJ4&G(LNB|Q?0*e_02aL zzqpZ*wC@Nf->{B;cC2bhH?y0|^u6t(dvt1tt331alrZXK&lGhPEK(@f^yBEKfLpp( z`0iTv6uuMW>(o#2LMdS90+|17^IsKrGvj7q>zDSth4l$x*{utAbOkjxwW)L@AjO;m z1V5Vs2Ld>EJg*29m&&UO{v8cN0j&b_gLy)xPN^0jdN+&Xt-1&=YVBjFqB+#)ZLS(#uFh{PUx*Yl-G+wTk!^4 zFGDXT;xA?D{Tnb9705?=X%ihaFn8si#t2N}D`6XCWQ$JjQv17ww|tNOH65uXnup&| zjVjc|Te z-N9jGG6Y@6MVB`)I9wsoy$y^R_!N9@Hu1zld?Rj3(4ZkGqCJXxK~khw*7c6JDj{cVhk`Ewl&t6 z$Ec6gO7~1FnNILW<77tS_9E`?s-WK=N)cd{O0)}3W~1(IXJEE{sX9cklFr)!{djQv z@2rfwS!=928q8#7D9S5`Ig4mJJFuG>oX(~jTn zEwDi$(N!E8c8asKZkIQCGwKGIPeanaM3hM$OMiIB@)AN?JKbRnQNs|SC|M~&U?Pq~ zoYP-dAuz=M!nD5V5l0A7I<4$=<~TorA75{7h*gsVH#{ih)w^cWuPoBJvedwy2PnPh zT-;PNZ;x^GQBdk|f9S&c(438i&B8>8+oJ3U?nDmZaropT6zBqQt>YcY?`?nPCgQ_y z&^=4}q){M`{ULVyMH$=9V7YBt`2+g7fTah;LG>(`S6-+|OSHl78vTDqmK{Y#sXn@T zkRvHZ=)@5$aDsl`G@_(w9pLg=GIDHvg8&R0{{&f)n=H=H`1>-Ma4N8m{7m)l_6cA~ zM-Vm8jyPYnyW5?ULbyUpVSvSu(+mrvy$wQy7gQh! zalPp&&rQt&iz6-fBN@94`<2P9<*%Rn{IfLSYeTYxOS6VY37UHRTqX$#=%LGXhUJua zxgNh{$ul#ak)?SEZK#_ptGiS;uaZ%{JBz>z>-I}flSXV+wt7=N`$NdRVV-h`iRhgF z{mk(b;$Pzad8Gs;ZF;@C0eNggumO8lD^GY7&+B;2M6{BrkH&SJ)cZZvV_PUfCK1B- z6VQ>wP-F%U`EgCu@|{yYDeHQuHzFF`=OMHuMoOctIyH|sLeQBCs6#SdoGhW>C3@@F zwJB%J;&=nc$cfOCX7PRSH&1CfeIWQa$|N~wp)e^&$j_CY;Tz`BdO`V33G&cgkgmis zT610MZhl*h43zS&kVF2qeUW7it>6hg?-3^96N-jap%I&*t#I%v)gB@kCfn z0^gy@&yM8!54;i%#{oK5@05-Y?q%ACZO+4=U&l+%=>f7<8Nz`8rBEn>B+nVS66407 z{sT5nvcx*c<3STFTwZgU7nW=%9a2vZO{LoHGPg5s{5g0fz9#76_Ez%j(d@sNO-v_N zN2^)?fkvLjYdW%W(@HR(<&i>+i!jh;H^?F82K-ClbqU`Rgy-+CKEr$()STNCi8_Qx zZ3&grV1NF!X@YjG6%LkFm0Fa~R-*iJ8-o8zeDD;5y0i>z$AmOFU7+h4FxZ%mLdfq( zxbJ^QhwMUsGkU>F?*plc8L?64ec45Z#X$JIFqjX6#JE>Cx}FX)f~0_dY(O;sx~VBM zwkb>G%<6yE!cKbLD;Eq?42c*RRBI$GW8C=J>L&e%2NLkb3u3ugGPs)`$&jieu|8EF zo*9N}8Rr>oM*)=HTeIymRqMZ#qQ@T5E<}6%)TyIykXb;3mtaXa5C3k5yrxJkfs?|_ zom+acwmb=XnkJ3K^idpmsB{$sell8V9*r&;y;P#)g5dtsh}24abb2S%x=TDn;St<3XPsCTBK?nKba_*#|l?j1} zBwX8$Rsn9P5n+`a{_kRXlcz(Te%*1L-bjPw(W8c|cj%i@B}}|W5U)MU4hwjS&0Sn@ zeEr8CaBt6B^wkrC`0na(D=oQTA-|3az~=61308Q1>^Cpy-whUmg*sH@eCS3CYIuKP z?s_8gwF8g~-Q(L#whNUL?g3Qw|85^Zhk?EWu+%;i{KK$-rt-sRNF0Tar`1V(jvakE z+mlt+==;6A%l5}wV)Wfi$e9Hm{S?U2YE{Q1?A@p7L83AI2qJg2O@-ju8p-PXPso`z zKTfd;X58y$mDgztYL>aDmr@Eb?c{w8tv}k-FUBFq+Q$D*;l>WYM1g8T$K0iuOX5>D zg!DarZ-11x!>--z$LaQtz(OA)8M$H`A#Pho&u+sE@gm~&4MR1ClARM42~}ZBLshn2 z>P34^k=neTEddR!ScC$i5Du6r)QxLlvmb`h>JEl;y_Px)WCpGS#q5 z0U$*HP{=^z$%lARYZwSfu(LOJz`@aXf~^Mno>OE{)_kJs`V5w9eVHRR?>%e?cP>yOc>4tD|>X4?g@ z2TM8+RnqzY53!OHFL|ybPv`;%RWDc5M(5zk9Ug4gx8necx@N=(v$;Z~Qg>FXYAZs< z@KUD*hsQyroNV`U^hTAtI*Hl^m=d2_rSBMVt zhIYn3_PMwO1-vildwZ4$2ZL-7FFe%?B0zDDZ#!&zfO(2jGY+={A?f&C z(fa>C*^)6)Fc5_+`?URwjuJU!-it_S9^ai^ny^PPeDg8vw4d}6n5A9lirk7ysR(~7fS>or|;|id0#*G&*%MqC#cDpa{p)BIxVAK8pspiLt7%Yf4LU3IbvoT zka#RT*`U4Z!SoSVs3&-FY~ZZ5qTfqu8}VKxSeRf|_pCS8`3s!;_3bcf{vrQodi7T| z>xm}jWSPla_R>t3oog-&?)lcNc-|Pudx|FQinb93P~kT$o+~;CTc? zgXhX~p81ti5<-A17POMUAEuSe zb_0{C&A4o26Q~Iu;~U{u7-6f#Nq= zzYjtOy@f-C5g`6)bJN-avQF(Xf?G*T`jVi zG*48_t`PAq;8u;so_hQa@?WlFNnKzWV03NYoqd6_`aZi|hCz8d!JIi(A3UdTM%%B4 zhNU|WTZzBCes7U_eeD>&pg0q7s}q0@5P~x9`YsM-2C{HYrSi>`r=h74jvw2U_q-lf z0uOsMD>d6f;wSFtt+Pltw437|FjFxAC%Xct&zyNdxl*#vo4nMJOTBKq6J<=#O%+rv?e^I$K z*LnF-DUEWhrz>)W!T6$on?Ws4(+4!<{PrBZ>nsN#bH&M2V#8LnOA9^+_1dp_rUoA4 z(t_$8zp=!XP(`8Z03Jc|HMlm%2q#79#Ibp+=FaDD4}^9ORO!U*w02&v#hCM%EG@?r zvwfw;qNc=@dR?$HTPHKEvaDl&&I3YmX`i=-SD*#0XfQt!S(uH#1t$^Bc&#n!eUt$~ z6?1B(=v39RE`%!NpW^+QA=ayH!d#W#$ng-A6Mj5}a<8XL76ms0UG1?i)w1V;1fps7 zie3}qO_k>-ck6F5nsHv$s~A~$2ska+jNAcM2^z8&);Eh+i2Fnp#VpfDW+T@SisVgk z2*!(Re%@2vRP=)%bWc8G*I1$#uhz?8;&sw8uj}|mS%BDHJb>Y<-5B^wjRPl_@L_z= z-ywrO*0O|8uo)$bY{NWDXu6)+LnpQY%VnvE^FPMZW(ifRc+Y|8a@~mu{F=r%8shuuSA7P*Prx z)2b_CSKtAw%4nU|*s@GeD|k=E2&@Of0#y)#(sq;sUTTs*oxM+{HpciO(}7?I%hrbI zv?*^JGvXmtoKVP+M0v5pD4?wCzkF zcLPJ&H5P|i43eZD975^qzB~u^v^e)t9Zd~5!x=zo_0nEq=zyIu+WU*puy&1`VJ3Ep zRlH>mG-y#5cpc10n>Q0V14H^UcR}AETwDffu*#}#WFav8g)X(VHsw*#8qIjMRNQ|eC|iL}3OWdt`*+O@QpbwZ zv|0s1Qg!ohgj6O>?rIKr3%S&e?LM%vMqBZd`B^rC`lM{n&f?TY9U!Apl@{UbbF$Qt z(LHVcVrFo~JQjN+pQ$zIIf!$ut}b^1w9uQTtSIo=Wi0yGShz3L}ZA*9kd~VtG1S=h%ygqNeE(SmQJc@ zd&wKM*HDvA63mUhF;B~}3gyS%13kpw;CZKog(n$#1a17GP?g;}_C`y44|^W$2;>!f z`vuKXL{rLTT`M2yM#26Knc`#~F#=c`@IJVaqq$@(YX Date: Mon, 31 Aug 2026 17:35:00 -0400 Subject: [PATCH 52/53] fix(store): gate the git-pull update path (#508) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(store): gate the git-pull update path `install_plugin` gates every route that re-downloads, `_reinstall_with_rollback` included. `update_plugin` has one branch that re-downloads nothing: a git checkout pulls in place, installs dependencies, and returns True. A pull could therefore deliver a manifest flooring above this core and nothing would notice until the plugin failed to load — which surfaces as one line in the journal and a display that silently stopped appearing. Checked after the pull rather than before it, for the same reason `_install_plugin_impl` checks after the download: the registry carries no compatibility field, so the incoming floor is only knowable once the new commit is on disk. Undone with `git reset --hard` to the pre-pull commit rather than by removing the directory. This is a live checkout, the old commit is still in the object store, and the reset leaves the user on the exact version they were already running — the same promise `_reinstall_with_rollback` makes, reached by the means this path actually has, with no window where the plugin directory does not exist. An unreadable manifest allows: it is not evidence of a floor. Scope, stated plainly: monorepo plugins install as archives and update through `_reinstall_with_rollback`, so they were already gated. Only registry entries with no `plugin_path` reach this branch. It is closed anyway because the sunset rule in the plugins repo's `08-shared-sports-code.md` names, as condition 3, that the core enforces the floor "at install/update time" — and B6 rests on that being true rather than merely written down. `install_from_url` is still ungated; the tests say so rather than letting the next reader assume otherwise. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 * fix(store): do not pull what the gate cannot un-pull Review of the gate found a data-loss path it had introduced, plus two smaller scope errors. All three from CodeRabbit on #508. **The stash failure was load-bearing and was not treated as one.** update_plugin stashes local changes before pulling; when that stash failed or timed out it logged a warning and pulled anyway. That was harmless while nothing ever undid a pull. It is not harmless now: the gate's rollback is `git reset --hard`, which discards uncommitted tracked edits -- exactly the edits the stash existed to protect. A pull does not refuse on a dirty tree as long as the incoming commit touches other files, so the sequence completed silently: pull succeeds, gate refuses, reset takes the user's work with it. update_plugin now returns before pulling unless the tree was already clean or was successfully stashed. Refusing costs an update in a case that had already gone wrong; the alternative costs data. That also makes `--hard` safe by construction in _gate_pulled_commit, and its comment now says so rather than observing it in passing. Pinned by test_a_failed_stash_stops_the_update_before_pulling, which writes a local edit, forces the stash to fail, and asserts both that HEAD did not move and that the edit is still on disk. Verified it bites: with the new guard removed the file comes back as `class P: pass`, the edit gone. **_HAS_GIT could take the module down instead of skipping it.** With no git on PATH, subprocess.run raises FileNotFoundError, and this runs at import time -- before skipif can act, so the whole file errors rather than skipping. Now catches OSError. **The doc overclaimed the gate's reach.** It said the floor is enforced on "every route that installs or updates" while the same passage notes install_from_url is ungated. Both spots now scope the claim to registry-managed installs and the two supported update paths, and name the sideload exception. Full suite 3720 passed, 6 skipped. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 --------- Co-authored-by: Claude Opus 5 (1M context) --- docs/SPORTS_UNIFICATION.md | 16 +- src/plugin_system/store_manager.py | 114 ++++++++++++ test/test_plugin_compatibility_gate.py | 233 ++++++++++++++++++++++++- 3 files changed, 359 insertions(+), 4 deletions(-) diff --git a/docs/SPORTS_UNIFICATION.md b/docs/SPORTS_UNIFICATION.md index 4e731278..4ef30e94 100644 --- a/docs/SPORTS_UNIFICATION.md +++ b/docs/SPORTS_UNIFICATION.md @@ -27,7 +27,7 @@ These are independent concerns. Conflating them is what produces god classes. | Plugin loads on a core that predates a module | Guarded import with a bundled fallback (`try: from src.X import Y / except ModuleNotFoundError: from y import Y`) | | Plugin loads on a core that predates a *method* | Capability probing — `hasattr(SportsCore, "_detect_stale_games")` — never a version comparison. The loader's compat check is advisory-only (it logs and continues), so probing is the real protection. | | Core changes never break a plugin's rendering | The **view-model contract**: `_extract_game_details_common` returns a dict whose `GUARANTEED_KEYS` are frozen by `test/test_skin_system.py::TestViewModelContract`. Keys may be added, never renamed or removed. | -| A plugin can drop its bundled copy safely | The **sunset rule**: its manifest must floor `ledmatrix_min_version` at the first core release shipping the module (recorded in `CHANGELOG.md`) — *necessary but not sufficient*. Nothing enforces that floor today, so the copy also waits for the B6 gate below. | +| A plugin can drop its bundled copy safely | The **sunset rule**: its manifest must floor `ledmatrix_min_version` at the first core release shipping the module (recorded in `CHANGELOG.md`) — *necessary but not sufficient*. The store enforces that floor on every registry-managed install and on both supported update paths (sideloading via `install_from_url` is not gated), but a floor cannot reach a user who never updates, so the copy also waits for the B6 gate below. | The core API is **additive-only**. A method the plugins call is never removed or given a new required parameter; new behavior arrives as new methods with @@ -234,6 +234,20 @@ a floor can be trusted against, and today it is not: compares the plugin's manifest version against the registry's `latest_version` and nothing else. + *Fixed, in two parts.* `install_plugin` gained the gate in #431/#433, which + covers every registry-managed install and, through `_reinstall_with_rollback`, + the update path that re-downloads. + `update_plugin`'s git branch pulls in place and re-downloads nothing, so it + stayed ungated until `_gate_pulled_commit` closed it — checked after the pull + (the registry carries no floor field, so the incoming floor is unknowable + before it) and undone with `git reset --hard` to the pre-pull commit. That + route is rare in practice, since monorepo plugins install as archives; it was + closed because the sunset rule in the plugins repo's + `08-shared-sports-code.md` states as **condition 3** that the core enforces + the floor "at install/update time", and B6 rests on that being true rather + than merely written down. `install_from_url` — sideloading a plugin from a + URL — is still ungated. + So B4 is: tag and release 3.2.0; make the tag, the release, and `__version__` agree, and keep them agreeing; reconsider the `< 2.0.0` skip; migrate manifests from `ledmatrix_min` to `ledmatrix_min_version`; and add the install/update diff --git a/src/plugin_system/store_manager.py b/src/plugin_system/store_manager.py index 5cf31cf8..623270e3 100644 --- a/src/plugin_system/store_manager.py +++ b/src/plugin_system/store_manager.py @@ -2583,6 +2583,85 @@ class PluginStoreManager: self.logger.error(f"Error uninstalling plugin {plugin_id}: {e}") return False + def _gate_pulled_commit(self, plugin_id: str, plugin_path: Path, + previous_sha: Optional[str]) -> bool: + """Apply the compatibility gate to a commit that arrived via git pull. + + Every other route into an installed plugin goes through + ``install_plugin``, which gates in ``_install_plugin_impl``. This one + did not: a ``git pull`` could deliver a manifest flooring above this + core and nothing would notice until the plugin failed to load, which + surfaces as one line in the journal and a scoreboard that silently + stopped appearing. + + Checked after the pull rather than before it, for the same reason + ``_install_plugin_impl`` checks after the download: the registry + carries no compatibility field, so the incoming floor is only knowable + once the new commit is on disk. + + Undone with ``git reset --hard`` rather than by removing the directory. + This is a live checkout, the previous commit is still in the object + store, and the reset leaves the user on the exact version they were + already running -- the same promise ``_reinstall_with_rollback`` makes, + reached by the means this path actually has. It is also the gentler + option: no window in which the plugin directory does not exist, and no + ``.standalone-backup-`` debris if the process dies mid-way. + + A manifest that cannot be read is not evidence of incompatibility, so + it allows. ``compatibility.check`` refuses only on evidence for the + same reason: a wrong refusal breaks a working install, while a wrong + allowance degrades to exactly the behaviour this path had before the + gate existed. + """ + manifest_path = plugin_path / "manifest.json" + try: + with open(manifest_path, 'r', encoding='utf-8') as mf: + manifest = json.load(mf) + except (OSError, ValueError) as e: + self.logger.warning( + "Could not read %s after updating %s (%s); allowing the " + "update, as an unreadable manifest declares no floor", + manifest_path, plugin_id, e) + return True + + from src import __version__ as core_version + from src.plugin_system import compatibility + + compatible, reason = compatibility.check(manifest, core_version) + if compatible: + return True + + self.logger.error("Refusing the update to %s: %s", plugin_id, reason) + + if not previous_sha: + self.logger.error( + "Cannot roll %s back: the commit it was on before the pull is " + "unknown. It is now on a version this core cannot run — " + "reinstall it from the plugin store.", plugin_id) + return False + + # Safe by construction: update_plugin returns before pulling unless the + # tree was clean or successfully stashed, so there are no uncommitted + # tracked edits for --hard to discard. The stash is not popped on the + # success path either, so the reset leaves the working tree exactly + # where a successful pull would have. Say "commit", not "changes". + reset = subprocess.run( + ['git', '-C', str(plugin_path), 'reset', '--hard', previous_sha], + capture_output=True, text=True, timeout=60, check=False) + if reset.returncode != 0: + self.logger.error( + "CRITICAL: could not roll %s back to commit %s: %s. It is left " + "on a version this core cannot run; " + "`git -C %s reset --hard %s` restores it.", + plugin_id, previous_sha[:7], + (reset.stderr or reset.stdout or '').strip(), + plugin_path, previous_sha) + else: + self.logger.info( + "Rolled %s back to commit %s; it stays on the version it was " + "already running.", plugin_id, previous_sha[:7]) + return False + def _reinstall_with_rollback(self, plugin_id: str, plugin_path: Path) -> bool: """Replace an installed plugin with a fresh install, atomically. @@ -2860,6 +2939,8 @@ class PluginStoreManager: status_result = type('obj', (object,), {'stdout': '', 'stderr': 'Status check timed out'})() stash_info = "" + # Whether the pull can be undone without destroying work. + tree_is_recoverable = not has_changes if has_changes: self.logger.info(f"Stashing local changes in {plugin_id} before update") try: @@ -2873,12 +2954,37 @@ class PluginStoreManager: ) if stash_result.returncode == 0: stash_info = " (local changes were stashed)" + tree_is_recoverable = True self.logger.info(f"Stashed local changes (including untracked files) for {plugin_id}") else: self.logger.warning(f"Failed to stash local changes for {plugin_id}: {stash_result.stderr}") except subprocess.TimeoutExpired: self.logger.warning(f"Stash operation timed out for {plugin_id}, proceeding with pull") + # Do not pull what cannot be un-pulled. + # + # The compatibility gate below can refuse the commit this + # pull brings down, and its only way back is `git reset + # --hard`, which discards uncommitted tracked edits. Those + # edits are exactly what the stash above exists to protect, + # so a stash that failed or timed out leaves the rollback + # unable to run without destroying them. + # + # A pull does not necessarily refuse on a dirty tree -- git + # merges happily as long as the incoming commit touches + # different files -- so without this the update would + # succeed, the gate would refuse, and the reset would take + # the user's work with it. Refusing here costs an update in + # a case that already went wrong; the alternative costs + # data. + if not tree_is_recoverable: + self.logger.error( + "Refusing to update %s: it has local changes that could " + "not be stashed, and an incompatible update could then " + "only be rolled back by discarding them. Commit or stash " + "them by hand, then update.", plugin_id) + return False + # Pull from the determined remote branch self.logger.info(f"Pulling from origin/{remote_pull_branch} for {plugin_id}...") pull_result = subprocess.run( @@ -2901,6 +3007,14 @@ class PluginStoreManager: elif updated_sha: self.logger.info(f"Plugin {plugin_id} updated to commit {updated_sha[:7]}{stash_info}") + # The install gate, at the only point on this path where + # it can be answered. Every other route in goes through + # install_plugin, which gates in _install_plugin_impl; this + # one did not, so a pull could deliver a manifest flooring + # above this core and nothing would notice. + if not self._gate_pulled_commit(plugin_id, plugin_path, local_sha): + return False + self._install_dependencies(plugin_path) return True diff --git a/test/test_plugin_compatibility_gate.py b/test/test_plugin_compatibility_gate.py index 26834ffa..91a84b8e 100644 --- a/test/test_plugin_compatibility_gate.py +++ b/test/test_plugin_compatibility_gate.py @@ -19,6 +19,7 @@ The rules being pinned here, in priority order: """ import json +import subprocess from pathlib import Path from unittest.mock import MagicMock @@ -146,9 +147,17 @@ def store(tmp_path, monkeypatch): class TestInstallGate: - """`install_plugin` is the chokepoint: `_reinstall_with_rollback` calls it, - so gating there covers updates too, and a refused update restores the - version the user already had.""" + """`install_plugin` is the chokepoint for every route that re-downloads: + `_reinstall_with_rollback` calls it, so a refused update restores the + version the user already had. + + It is not the *only* route in, and saying so here once is cheaper than + rediscovering it. `update_plugin` also has a git branch that pulls in + place and never re-downloads; that one is gated separately by + `_gate_pulled_commit` and pinned in `TestGitPullGate` below. A third + route, `install_from_url`, is still ungated — sideloading from a URL + checks required fields but not the floor. + """ def _install_with_manifest(self, store, manifest, core_version, monkeypatch): mgr, plugins_dir = store @@ -208,6 +217,224 @@ class TestInstallGate: assert (path / "manifest.json").exists() +# -------------------------------------------------------------------------- +# The git-pull update path — the one route that does not re-download +# -------------------------------------------------------------------------- + +def _git_available() -> bool: + # OSError, not just a non-zero exit: with no git on PATH subprocess raises + # FileNotFoundError, and this runs at import time -- before skipif can act, + # so the whole module would error out instead of skipping. + try: + return subprocess.run( + ['git', '--version'], capture_output=True).returncode == 0 + except OSError: + return False + + +_HAS_GIT = _git_available() + + +def _git(*args, cwd): + return subprocess.run(['git', *args], cwd=str(cwd), + capture_output=True, text=True, check=True) + + +@pytest.mark.skipif(not _HAS_GIT, reason='git not available') +class TestGitPullGate: + """A plugin installed as a git checkout updates by pulling in place, so it + never passes through `install_plugin` and was never gated. + + Real git repositories rather than mocks, because the claim under test is + that `git reset --hard` puts the checkout back — a mock of git would only + prove the call was made, which is the easy half. + + Only the handful of registry entries with no `plugin_path` reach this path + in practice; monorepo plugins install as archives and update through + `_reinstall_with_rollback`. It is gated anyway because the sunset rule in + `docs/plugin-development/08-shared-sports-code.md` states the core enforces + the floor "at install/update time", and a precondition that is documented + but not true is worse than one that is merely missing. + """ + + @pytest.fixture + def checkout(self, tmp_path, monkeypatch): + """An origin repo holding a plugin, and a clone of it installed as + `plugin-repos/gitplug`, with the store pointed at it.""" + from src.plugin_system.store_manager import PluginStoreManager + + origin = tmp_path / 'origin' + origin.mkdir() + _git('init', '--initial-branch=main', '--bare', cwd=origin) + + seed = tmp_path / 'seed' + _git('clone', str(origin), str(seed), cwd=tmp_path) + _git('config', 'user.email', 'test@example.com', cwd=seed) + _git('config', 'user.name', 'Test', cwd=seed) + (seed / 'manifest.json').write_text(json.dumps({ + "id": "gitplug", "name": "Git Plug", "class_name": "P", + "display_modes": ["a"], "min_ledmatrix_version": "2.0.0", + }), encoding='utf-8') + (seed / 'manager.py').write_text("class P: pass\n", encoding='utf-8') + _git('add', '-A', cwd=seed) + _git('commit', '-m', 'initial', cwd=seed) + _git('push', '-u', 'origin', 'main', cwd=seed) + + plugins_dir = tmp_path / 'plugin-repos' + plugins_dir.mkdir() + work = plugins_dir / 'gitplug' + _git('clone', str(origin), str(work), cwd=tmp_path) + _git('config', 'user.email', 'test@example.com', cwd=work) + _git('config', 'user.name', 'Test', cwd=work) + + mgr = PluginStoreManager(plugins_dir=str(plugins_dir)) + mgr.logger = MagicMock() + # No registry entry, so update_plugin takes the plain-pull branch + # rather than the remote-mismatch or already-current shortcuts. + monkeypatch.setattr(mgr, 'fetch_registry', lambda *a, **k: None) + monkeypatch.setattr(mgr, 'get_plugin_info', lambda *a, **k: None) + monkeypatch.setattr(mgr, '_install_dependencies', lambda *a, **k: True) + return mgr, seed, work + + def _push(self, seed, manifest_text): + (seed / 'manifest.json').write_text(manifest_text, encoding='utf-8') + _git('add', '-A', cwd=seed) + _git('commit', '-m', 'update', cwd=seed) + _git('push', 'origin', 'main', cwd=seed) + + def _head(self, work): + return _git('rev-parse', 'HEAD', cwd=work).stdout.strip() + + def test_refuses_a_pulled_commit_that_needs_a_newer_core( + self, checkout, monkeypatch): + mgr, seed, work = checkout + before = self._head(work) + self._push(seed, json.dumps({ + "id": "gitplug", "name": "Git Plug", "class_name": "P", + "display_modes": ["a"], "min_ledmatrix_version": "9.9.9", + })) + + import src + monkeypatch.setattr(src, '__version__', '3.2.0') + assert mgr.update_plugin('gitplug') is False + + assert self._head(work) == before, ( + "a refused update must leave the checkout on the commit it was " + "already running, not on the one it cannot load") + floor = json.loads((work / 'manifest.json').read_text()) + assert floor['min_ledmatrix_version'] == '2.0.0', ( + "the reset must restore the working tree, not just the ref") + + def test_allows_a_pulled_commit_the_core_can_run( + self, checkout, monkeypatch): + """The guard against over-refusing. A gate that refuses everything + passes the test above and breaks every update.""" + mgr, seed, work = checkout + before = self._head(work) + self._push(seed, json.dumps({ + "id": "gitplug", "name": "Git Plug", "class_name": "P", + "display_modes": ["a"], "min_ledmatrix_version": "3.0.0", + })) + + import src + monkeypatch.setattr(src, '__version__', '3.2.0') + assert mgr.update_plugin('gitplug') is True + assert self._head(work) != before + + def test_an_unreadable_manifest_after_pull_is_not_a_refusal( + self, checkout, monkeypatch): + """Rule 1 of this file — refuse only on evidence. A manifest that + will not parse declares no floor, so it is not evidence of anything.""" + mgr, seed, work = checkout + self._push(seed, '{ this is not json') + + import src + monkeypatch.setattr(src, '__version__', '3.2.0') + assert mgr.update_plugin('gitplug') is True + + def test_an_untrustworthy_core_does_not_block_a_2_0_0_floor_on_pull( + self, checkout, monkeypatch): + """The same regression guard as + `test_untrustworthy_core_does_not_block_installs`, because this path + now shares that rule: a v3.1.0 release reports 1.0.0, and nearly every + published manifest floors at 2.0.0.""" + mgr, seed, work = checkout + before = self._head(work) + self._push(seed, json.dumps({ + "id": "gitplug", "name": "Git Plug", "class_name": "P", + "display_modes": ["a"], "min_ledmatrix_version": "2.0.0", + "description": "changed", + })) + + import src + monkeypatch.setattr(src, '__version__', '1.0.0') + assert mgr.update_plugin('gitplug') is True + assert self._head(work) != before + + def test_a_failed_stash_stops_the_update_before_pulling( + self, checkout, monkeypatch): + """Uncommitted work is not collateral for the gate. + + The gate's only rollback is `git reset --hard`, which discards + uncommitted tracked edits. update_plugin stashes them first -- but when + that stash fails it used to pull anyway, and a pull touching different + files succeeds on a dirty tree. Refuse, refuse the rollback's rollback, + and the user's edits go with it. + """ + mgr, seed, work = checkout + before = self._head(work) + (work / 'manager.py').write_text( + "class P:\n MINE = 'do not lose this'\n", encoding='utf-8') + self._push(seed, json.dumps({ + "id": "gitplug", "name": "Git Plug", "class_name": "P", + "display_modes": ["a"], "min_ledmatrix_version": "9.9.9", + })) + + real_run = subprocess.run + + def fail_stash(cmd, *a, **k): + if isinstance(cmd, list) and 'stash' in cmd: + return subprocess.CompletedProcess(cmd, 1, '', 'stash boom') + return real_run(cmd, *a, **k) + + import src + monkeypatch.setattr(src, '__version__', '3.2.0') + monkeypatch.setattr( + 'src.plugin_system.store_manager.subprocess.run', fail_stash) + assert mgr.update_plugin('gitplug') is False + + assert self._head(work) == before, "must not pull what it cannot undo" + assert 'do not lose this' in (work / 'manager.py').read_text(), ( + "the local edit the stash failed to save must still be there") + + def test_rollback_reports_the_recovery_command_when_git_fails( + self, checkout, monkeypatch): + """If the reset itself fails the user is left on a version that cannot + load, so the log line has to carry the command that fixes it — + it is the only thing standing between them and a manual reinstall.""" + mgr, seed, work = checkout + self._push(seed, json.dumps({ + "id": "gitplug", "name": "Git Plug", "class_name": "P", + "display_modes": ["a"], "min_ledmatrix_version": "9.9.9", + })) + + real_run = subprocess.run + + def fail_reset(cmd, *a, **k): + if isinstance(cmd, list) and 'reset' in cmd: + return subprocess.CompletedProcess(cmd, 1, '', 'reset boom') + return real_run(cmd, *a, **k) + + import src + monkeypatch.setattr(src, '__version__', '3.2.0') + monkeypatch.setattr( + 'src.plugin_system.store_manager.subprocess.run', fail_reset) + assert mgr.update_plugin('gitplug') is False + + logged = ' '.join(str(c) for c in mgr.logger.error.call_args_list) + assert 'reset --hard' in logged, logged + + class TestLoaderAndStoreAgree: """Both read the same manifests; a disagreement means one of them is lying to the user.""" From 154525beb8f7d182768a148c6ddf37cd47b8c31f Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:54:04 -0400 Subject: [PATCH 53/53] fix(background): release the payload after every callback, not after each one (#509) Callers that join an in-flight fetch share one FetchResult. #499 released the payload inside the delivery loop, so the first callback got the data and every joiner got `result.data is None`. That is not a quiet degradation. Consumers read `result.data.get('events')`, so they raise AttributeError -- which the delivery loop catches and logs. The entire failure surfaced as one line: ERROR - src.background_data_service - Error in callback for request nhl_2026_...: 'NoneType' object has no attribute 'get' and a manager that silently never received its schedule. Seen on hardware: NHLRecentManager logs "Background fetch completed for 2026: 1000 events" and the very next line is the error, from NHLUpcomingManager's callback on the same request -- which had already logged "No events found in shared data." Deduplication is the normal case, not a corner. A sport's recent, upcoming and live managers all want the same season schedule, so the second and third are joiners on almost every cycle. _release_payload's own docstring said "once A callback has been handed the data", singular, which is the assumption that broke: the loop above it was written for many, and says so. Moved after the loop, and guarded on `callbacks` being non-empty. The guard matters: a request submitted without a callback must keep its payload, because polling get_result() is then the only way to collect it. The per-delivery release got that right by accident -- an empty list never entered the loop body -- and the existing test for it caught the omission. test_background_payload_release.py gains TestJoinersAllGetTheData: two submitters on one in-flight cache_key, asserting both are handed a populated payload, plus that the memory fix still happens once they have all had it. test_background_fetch_dedupe.py already proved the joiner's callback FIRES; it never checked what the callback received, which is the gap that let this through. Verified the new test bites: restoring the release inside the loop fails it with "'second' was handed a released payload". Full suite 3716 passed, 6 skipped. Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 Co-authored-by: Claude Opus 5 (1M context) --- src/background_data_service.py | 25 +++++-- test/test_background_payload_release.py | 93 +++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 4 deletions(-) diff --git a/src/background_data_service.py b/src/background_data_service.py index b3838322..fa20783c 100644 --- a/src/background_data_service.py +++ b/src/background_data_service.py @@ -478,8 +478,22 @@ class BackgroundDataService: cb(result) except Exception as e: logger.error(f"Error in callback for request {request.id}: {e}") - # Delivered. Drop both references -- they point at the same - # object, so one survivor keeps the whole payload resident. + + # Released AFTER the loop, not inside it. Every callback here holds + # the same FetchResult, so releasing per-delivery handed the first + # one the data and every joiner `result.data is None` -- which is + # not a quiet degradation: they read `result.data.get('events')` and + # raise AttributeError, which this very loop catches and logs, so + # the symptom was one ERROR line and a manager that silently never + # got its schedule. Deduplication is the normal case, not a corner: + # a sport's recent, upcoming and live managers all ride one season + # fetch. + # + # Guarded on `callbacks`, because a request submitted without one + # has no other way to collect its payload than polling get_result(). + # The old per-delivery release got that right by accident: an empty + # list never entered the loop body. + if callbacks: self._release_payload(result) request.result = None @@ -489,8 +503,11 @@ class BackgroundDataService: def _release_payload(result: FetchResult) -> None: """Drop a delivered payload, keeping the result's status and timings. - Only called once a callback has been handed the data. Consumers read - fetched data back from the cache under ``cache_key``; the copy carried + Only called once EVERY callback has been handed the data -- callers + that joined an in-flight fetch share this object, so releasing between + deliveries strips the payload out from under the ones still queued. + Consumers read fetched data back from the cache under ``cache_key``; + the copy carried here was pinning a parsed season schedule -- 946 games for NCAA football, roughly a tenth of total RAM on a 1GB Pi -- in memory until the hourly sweep. diff --git a/test/test_background_payload_release.py b/test/test_background_payload_release.py index 3f6a72c1..2c90a8da 100644 --- a/test/test_background_payload_release.py +++ b/test/test_background_payload_release.py @@ -23,8 +23,14 @@ it only in passing before reading the cache back. Requests submitted *without* a callback keep their payload: polling get_result() is then the only way to collect it, so releasing would break that contract. + +And the release must happen after EVERY callback, not after each one. Callers +that joined an in-flight fetch share a single FetchResult, so releasing per +delivery strips the payload out from under everyone still queued -- see +TestJoinersAllGetTheData. """ +import threading import time import pytest from unittest.mock import MagicMock, Mock, patch @@ -185,3 +191,90 @@ class TestCacheHitPath: cache_key="nfl_2026", ) assert service.get_result(req_id).data == PAYLOAD + + +class TestJoinersAllGetTheData: + """Deduplicated callers share one FetchResult; releasing between them + empties it for the rest. + + Not a corner case. A sport's recent, upcoming and live managers all ask for + the same season schedule, so the second and third are joiners on almost + every cycle. Releasing inside the delivery loop handed the payload to + whichever ran first and gave the others `result.data is None`. + + The consequence was worse than a quiet degradation, because consumers do + `result.data.get('events')`: they raised AttributeError, the delivery loop + caught it, and the whole failure surfaced as a single + "Error in callback for request ..." line while that manager silently never + received its schedule. + """ + + class _BlockingSession: + """Holds the fetch open so a second submit lands while in flight.""" + + def __init__(self): + self.release = threading.Event() + self.started = threading.Event() + + def get(self, *a, **k): + self.started.set() + self.release.wait(timeout=5) + return _resp() + + def test_every_joiner_is_handed_the_payload(self, service): + session = self._BlockingSession() + seen = {} + + def record(name): + # Read it the way the sport managers do. `result.data['events']` + # would raise TypeError on None; `.get` raises AttributeError, + # which is the error actually seen in the field. + def cb(result): + seen[name] = result.data.get('events') if result.data else None + return cb + + with patch.object(service, "session", session): + first = service.submit_fetch_request( + sport="nhl", year=2026, url="https://x/s", cache_key="nhl_2026", + callback=record("first"), max_retries=0) + assert session.started.wait(timeout=5) + + joined = service.submit_fetch_request( + sport="nhl", year=2026, url="https://x/s", cache_key="nhl_2026", + callback=record("second"), max_retries=0) + # Without coalescing these are two independent fetches that each + # own their result, and the test proves nothing about sharing. + assert joined == first, "the joiner should share the in-flight id" + + session.release.set() + _wait(service, first) + + deadline = time.time() + 5 + while len(seen) < 2 and time.time() < deadline: + time.sleep(0.02) + + assert set(seen) == {"first", "second"}, f"both must be called, got {seen}" + for name, events in seen.items(): + assert events is not None, ( + f"{name!r} was handed a released payload: the result was " + f"emptied before every callback had been delivered") + assert len(events) == 50, f"{name!r} got {events!r}" + + def test_the_payload_is_still_released_once_they_have_all_had_it(self, service): + """The memory fix must survive the ordering fix.""" + session = self._BlockingSession() + + with patch.object(service, "session", session): + first = service.submit_fetch_request( + sport="nhl", year=2026, url="https://x/s", cache_key="nhl_2026", + callback=lambda r: None, max_retries=0) + assert session.started.wait(timeout=5) + service.submit_fetch_request( + sport="nhl", year=2026, url="https://x/s", cache_key="nhl_2026", + callback=lambda r: None, max_retries=0) + session.release.set() + _wait_for_release(service, first) + + stored = service.get_result(first) + assert stored is not None and stored.data is None, ( + "the payload must still be dropped once every callback has run")