fix(core): font zip cache, monotonic timers, resolver back-off, and other core/common fixes (#654)

* fix(core): font zip cache, monotonic timers, resolver back-off, and other core/common fixes

- font_manager: a .zip font URL is served as its extracted font after a
  restart (the cached-file check returned the archive first); downloads
  use requests with a 30s timeout into a temp file + os.replace.
- api_helper / sync_manager: rate-limit and heartbeat/leader timeouts use
  time.monotonic(); last_request_time and the status file's ts stay
  wall-clock. set_on_new_cycle docstring no longer claims core uses it.
- logo_helper: the placeholder uses the same scaled box as a real logo.
- permission_utils: one _sudo_bash_candidates() helper (with the sudoers
  exact-argv rationale) shared by sudo_remove_directory, which now retries
  the next bash path on a sudo refusal, and install_requirements_file.
- dynamic_team_resolver: failed/empty fetch backs off 5 min; duplicate
  INFO log and contradictory docstring example fixed.
- element_style: scale default looked up through element aliases.
- background_data_service: cache-hit callback runs outside the lock.
- config_arrays: union-aware type check (["array","null"]); stale
  dotToNested() reference removed.
- auto_update_setup: non-dict auto_update reads as off; temp result file
  unlinked when the write fails.
- exceptions: constructors copy the caller's context dict.
- logging_config: StructuredFormatter json.dumps(default=str).
- error_aggregator: removed unused export_path/export_to_file/_auto_export.
- Docstrings: validate_file_upload max_size_mb, raise_on_errors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(sync): retry the status-file rename like the other atomic writers

On Windows os.replace can fail with "Access is denied" while a scanner
briefly holds the target open; config_manager_atomic._replace already
retries that (and re-raises at once on other platforms). The sync status
writer called os.replace directly, which made
test_concurrent_writers_each_use_their_own_temp_file flaky on Windows.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

test/test_api_helper.py

@@ -50,29 +50,51 @@ def helper(cache):
 class TestRateLimiting:
     def test_sleeps_for_remaining_interval(self, helper, monkeypatch):
         fake_time = MagicMock()
-        fake_time.time.side_effect = [102.0, 105.0]
+        fake_time.monotonic.side_effect = [102.0, 105.0]
+        fake_time.time.return_value = 5000.0
         monkeypatch.setattr(api_helper_module, 'time', fake_time)
 
         helper.set_rate_limit(5)
-        helper._last_request_time = 100.0
+        helper._last_request_monotonic = 100.0
         helper._enforce_rate_limit()
 
         # 2s elapsed of a 5s interval -> sleep the remaining 3s.
         fake_time.sleep.assert_called_once()
         assert fake_time.sleep.call_args[0][0] == pytest.approx(3.0)
-        assert helper._last_request_time == 105.0
+        assert helper._last_request_monotonic == 105.0
+        assert helper._last_request_time == 5000.0
 
     def test_no_sleep_when_interval_elapsed(self, helper, monkeypatch):
         fake_time = MagicMock()
-        fake_time.time.side_effect = [200.0, 201.0]
+        fake_time.monotonic.side_effect = [200.0, 201.0]
         monkeypatch.setattr(api_helper_module, 'time', fake_time)
 
         helper.set_rate_limit(5)
-        helper._last_request_time = 100.0
+        helper._last_request_monotonic = 100.0
         helper._enforce_rate_limit()
 
         fake_time.sleep.assert_not_called()
-        assert helper._last_request_time == 201.0
+        assert helper._last_request_monotonic == 201.0
+
+    def test_wall_clock_step_back_does_not_stall_requests(self, helper, monkeypatch):
+        # NTP stepping the wall clock back an hour between two requests must
+        # not turn into an hour-long "remaining interval" sleep.
+        wall = {"now": 10_000.0}
+        mono = {"now": 50.0}
+        sleeps = []
+        fake_time = MagicMock()
+        fake_time.time.side_effect = lambda: wall["now"]
+        fake_time.monotonic.side_effect = lambda: mono["now"]
+        fake_time.sleep.side_effect = sleeps.append
+        monkeypatch.setattr(api_helper_module, 'time', fake_time)
+
+        helper.set_rate_limit(5)
+        helper._enforce_rate_limit()
+        wall["now"] -= 3600
+        mono["now"] += 10
+        helper._enforce_rate_limit()
+
+        assert all(s <= 5 for s in sleeps)
 
 
 # ---------------------------------------------------------------------------

test/test_auto_update_setup.py

@@ -75,6 +75,25 @@ def test_does_nothing_while_updates_are_off(tmp_path):
     assert not (root / aus.RESULT_REL).exists()
 
 
+@pytest.mark.parametrize('value', [True, 'yes', ['enabled']])
+def test_a_non_object_auto_update_value_reads_as_off(value):
+    """A hand-edited "auto_update": true raised AttributeError and aborted
+    setup; the web UI's save treats a non-object as {} (off), so this does too."""
+    assert aus.is_enabled({'auto_update': value}) is False
+
+
+def test_a_failed_result_write_leaves_no_temp_file(tmp_path, monkeypatch):
+    root, etc = project(tmp_path)
+
+    def broken_dump(*args, **kwargs):
+        raise OSError('disk full')
+    monkeypatch.setattr(aus.json, 'dump', broken_dump)
+    setup(root, etc, FakeSystemctl())._report('failed', 'x')
+    leftovers = [p.name for p in (root / aus.RESULT_REL).parent.iterdir()
+                 if p.name.startswith('.auto_update_setup_')]
+    assert leftovers == []
+
+
 def test_installs_both_units_for_the_web_user(tmp_path):
     root, etc = project(tmp_path)
     systemctl = FakeSystemctl()

test/test_background_data_service.py

@@ -93,6 +93,33 @@ class TestCacheHit:
         stats = service.get_statistics()
         assert stats["cached_hits"] == 1
 
+    def test_cache_hit_callback_runs_outside_the_service_lock(
+            self, service, mock_cache_manager):
+        """The worker path calls plugin callbacks after releasing the lock;
+        the cache-hit path held it, so a slow callback stalled every other
+        thread's submit and result bookkeeping."""
+        import threading
+        mock_cache_manager.get.return_value = {"events": []}
+        seen = {}
+
+        def callback(result):
+            # The result is already filed, as before.
+            seen["filed"] = service.get_result(result.request_id) is result
+            other = {}
+
+            def try_lock():
+                other["got"] = service._lock.acquire(timeout=1)
+                if other["got"]:
+                    service._lock.release()
+            t = threading.Thread(target=try_lock)
+            t.start()
+            t.join()
+            seen["lock_free"] = other["got"]
+
+        service.submit_fetch_request(sport="nba", year=2024, url="https://x.com",
+                                     cache_key="k", callback=callback)
+        assert seen == {"filed": True, "lock_free": True}
+
 
 # ---------------------------------------------------------------------------
 # Actual fetch path (mocked HTTP)

test/test_dynamic_team_resolver.py

@@ -48,9 +48,11 @@ def reset_class_cache():
     """Reset the CLASS-level shared cache between tests."""
     DynamicTeamResolver._rankings_cache = {}
     DynamicTeamResolver._cache_timestamp = 0
+    DynamicTeamResolver._failure_timestamp = 0
     yield
     DynamicTeamResolver._rankings_cache = {}
     DynamicTeamResolver._cache_timestamp = 0
+    DynamicTeamResolver._failure_timestamp = 0
 
 
 @pytest.fixture
@@ -158,12 +160,20 @@ class TestRankingsParsing:
         assert list(rankings.keys()) == ['A1', 'B2', 'C3']
         assert list(rankings.values()) == [1, 2, 3]
 
-    def test_empty_rankings_returns_empty_and_caches_nothing(
-            self, resolver, mock_get):
+    def test_empty_rankings_returns_empty_and_backs_off(
+            self, resolver, mock_get, monkeypatch):
         mock_get.return_value = _make_response({'rankings': []})
 
         assert resolver._fetch_ncaa_fb_rankings() == {}
-        # Nothing was cached, so the next call hits HTTP again.
+        # No rankings were cached, but the failure is: the next call inside
+        # the back-off window doesn't hit HTTP again.
+        assert resolver._fetch_ncaa_fb_rankings() == {}
+        assert mock_get.call_count == 1
+        assert DynamicTeamResolver._rankings_cache == {}
+
+        stamp = DynamicTeamResolver._failure_timestamp
+        monkeypatch.setattr(
+            dtr_module, 'time', types.SimpleNamespace(time=lambda: stamp + 301))
         assert resolver._fetch_ncaa_fb_rankings() == {}
         assert mock_get.call_count == 2
 
@@ -226,7 +236,7 @@ class TestSharedCache:
 
 class TestFailureHandling:
     def test_network_failure_drops_dynamic_keeps_static_caches_nothing(
-            self, resolver, mock_get):
+            self, resolver, mock_get, monkeypatch):
         mock_get.side_effect = [
             requests.exceptions.RequestException('boom'),
             _make_response(_rankings_payload()),
@@ -237,12 +247,29 @@ class TestFailureHandling:
         # Dynamic name silently dropped, static name kept, nothing raises.
         assert result == ['UGA']
 
-        # Nothing was cached on failure: a subsequent call refetches and
-        # succeeds.
+        # Inside the back-off window the outage isn't retried: each resolve
+        # would otherwise wait out the full request timeout again.
+        assert resolver.resolve_teams(['UGA', 'AP_TOP_5']) == ['UGA']
+        assert mock_get.call_count == 1
+
+        # No rankings were cached on failure: once the window passes, the
+        # next call refetches and succeeds.
+        stamp = DynamicTeamResolver._failure_timestamp
+        monkeypatch.setattr(
+            dtr_module, 'time', types.SimpleNamespace(time=lambda: stamp + 301))
         result = resolver.resolve_teams(['UGA', 'AP_TOP_5'])
         assert result == ['UGA', 'MICH', 'OSU', 'TEX', 'ALA']
         assert mock_get.call_count == 2
 
+    def test_clear_cache_forgets_a_failure(self, resolver, mock_get):
+        mock_get.side_effect = [
+            requests.exceptions.RequestException('boom'),
+            _make_response(_rankings_payload()),
+        ]
+        assert resolver.resolve_teams(['AP_TOP_5']) == []
+        resolver.clear_cache()
+        assert resolver.resolve_teams(['AP_TOP_5']) == TOP_TEAMS[:5]
+
 
 # ---------------------------------------------------------------------------
 # sport argument

test/test_element_style.py

@@ -1115,6 +1115,14 @@ class TestAliasedLookup:
         ).style("score_text")
         assert st.scale == 2.0
 
+    def test_scale_default_is_found_through_an_alias(self):
+        """The saved value equals the default filed under the bare-noun
+        layout key, so it is the save flow's write-in, not a choice."""
+        defaults = {"customization": {"layout": {"score": {"scale": 1.5}}}}
+        config = {"customization": {"layout": {"score": {"scale": 1.5}}}}
+        st = ElementStyleResolver(config, defaults).style("score_text")
+        assert st.scale == 1.0
+
 HANDWRITTEN = {
     "type": "object",
     "properties": {

test/test_error_handling.py

@@ -31,3 +31,18 @@ class TestCustomExceptions:
         # DisplayError includes context in string representation
         assert "Display not found" in str(error)
         assert error.context.get('display_mode') == 'adafruit'
+
+    def test_callers_context_dict_is_not_mutated(self):
+        """A caller reusing one context dict across raises used to have
+        every error's own key written into it."""
+        cases = [
+            (CacheError, {'cache_key': 'k'}),
+            (ConfigError, {'config_path': 'c.json', 'field': 'f'}),
+            (PluginError, {'plugin_id': 'weather'}),
+            (DisplayError, {'display_mode': 'm'}),
+        ]
+        for cls, kwargs in cases:
+            shared = {'attempt': 1}
+            error = cls("boom", context=shared, **kwargs)
+            assert shared == {'attempt': 1}, cls.__name__
+            assert error.context == {'attempt': 1, **kwargs}, cls.__name__

test/test_font_manager.py

@@ -8,13 +8,18 @@ test here asserts observable behavior: returned font types, cache identity,
 fallback selection, and BDF native-size reading.
 """
 
+import hashlib
+import io
 import json
 import shutil
+import zipfile
+from unittest.mock import MagicMock
 
 import freetype
 import pytest
 from PIL import ImageFont
 
+import src.font_manager as fm_module
 from src.common.font_layout import resolve_asset_path
 from src.font_manager import FontManager
 
@@ -188,3 +193,63 @@ class TestPluginFonts:
         assert fm.register_plugin_fonts("my-plugin", self.MANIFEST)
 
         assert fm.font_catalog["my-plugin::bundled"] == str(plugin_dir / "fonts" / "Bundled.ttf")
+
+
+class TestDownloadFont:
+    """_download_font: plugin fonts declared by URL, cached in temp_font_dir."""
+
+    URL = "https://fonts.example/pack.zip"
+
+    @staticmethod
+    def _zip_bytes():
+        buf = io.BytesIO()
+        with zipfile.ZipFile(buf, "w") as zf:
+            zf.writestr("MyFont.ttf", b"not really a font")
+        return buf.getvalue()
+
+    @staticmethod
+    def _response(chunks):
+        response = MagicMock()
+        response.raise_for_status.return_value = None
+        response.iter_content.side_effect = lambda chunk_size: iter(chunks)
+        return response
+
+    def test_a_zip_is_served_as_its_extracted_font_after_a_restart(self, fm, tmp_path):
+        # The state a previous run leaves: the .zip and its extracted font.
+        # The cache check used to find the .zip first and register the
+        # archive itself as the font.
+        fm.temp_font_dir = tmp_path
+        url_hash = hashlib.sha256(self.URL.encode()).hexdigest()[:16]
+        zip_path = tmp_path / f"pack_{url_hash}.zip"
+        zip_path.write_bytes(self._zip_bytes())
+        extract_dir = tmp_path / f"pack_{url_hash}"
+        with zipfile.ZipFile(zip_path) as zf:
+            zf.extractall(extract_dir)
+
+        path = fm._download_font(self.URL, {"family": "pack"})
+
+        assert path == str(extract_dir / "MyFont.ttf")
+
+    def test_download_has_a_timeout_and_lands_atomically(self, fm, tmp_path, monkeypatch):
+        fm.temp_font_dir = tmp_path
+        get = MagicMock(return_value=self._response([self._zip_bytes()]))
+        monkeypatch.setattr(fm_module.requests, "get", get)
+
+        path = fm._download_font(self.URL, {"family": "pack"})
+
+        assert path is not None and path.endswith("MyFont.ttf")
+        assert get.call_args.kwargs.get("timeout")
+        assert not list(tmp_path.glob("*.part"))
+
+    def test_an_interrupted_download_leaves_nothing_to_be_served(self, fm, tmp_path, monkeypatch):
+        fm.temp_font_dir = tmp_path
+
+        def chunks():
+            yield b"partial"
+            raise OSError("connection reset")
+        response = self._response([])
+        response.iter_content.side_effect = lambda chunk_size: chunks()
+        monkeypatch.setattr(fm_module.requests, "get", MagicMock(return_value=response))
+
+        assert fm._download_font("https://fonts.example/Font.ttf", {"family": "f"}) is None
+        assert list(tmp_path.iterdir()) == []

test/test_logging_config.py

@@ -57,6 +57,17 @@ class TestStructuredFormatter:
         assert out["plugin_id"] == "clock"
         assert out["operation_id"] == "op-1"
 
+    def test_non_json_context_values_are_stringified(self):
+        # A datetime/Path in the context used to raise TypeError from
+        # json.dumps, and the handler dropped the whole record.
+        from datetime import datetime
+        from pathlib import Path
+        record = make_record(context={"at": datetime(2026, 1, 2, 3, 4, 5),
+                                      "path": Path("a/b")})
+        out = json.loads(StructuredFormatter().format(record))
+        assert out["context"]["at"] == "2026-01-02 03:04:05"
+        assert out["context"]["path"] == str(Path("a/b"))
+
     def test_exception_key_when_exc_info_present(self):
         try:
             raise ValueError("kaboom")

test/test_logo_helper.py

@@ -235,6 +235,14 @@ class TestLoadLogoWithDownload:
             "PHI", tmp_path / "missing.png", None, max_width=16, max_height=16)
         assert logo is not None and logo.size == (16, 16)
 
+    def test_placeholder_uses_the_scaled_box(self, helper, tmp_path):
+        # A real logo at scale 2 is fitted into 32x32; the stand-in for a
+        # missing one must be the same size, not the unscaled 16x16.
+        logo = helper.load_logo_with_download(
+            "PHI", tmp_path / "missing.png", None, max_width=16, max_height=16,
+            scale=2.0)
+        assert logo is not None and logo.size == (32, 32)
+
 
 class TestDownloadLogo:
     def test_writes_file_and_sets_permissions(self, helper, tmp_path):

test/test_permission_utils.py

@@ -19,6 +19,7 @@ from src.common.permission_utils import (
     _redact_url_credentials,
     ensure_shared_group_ownership,
     install_requirements_file,
+    sudo_remove_directory,
 )
 
 
@@ -46,6 +47,42 @@ class TestRedactUrlCredentials:
         assert _redact_url_credentials(text) == text
 
 
+class TestSudoRemoveDirectory:
+    """sudoers matches the exact argv, so the bash path the rule names has
+    to be found by trying each candidate, as install_requirements_file does."""
+
+    def _target(self, tmp_path):
+        target = tmp_path / "some-plugin"
+        target.mkdir()
+        return target
+
+    @patch('src.common.permission_utils.subprocess.run')
+    def test_tries_the_next_bash_path_when_sudo_refuses(self, mock_run, tmp_path):
+        target = self._target(tmp_path)
+
+        def fake_run(argv, **kwargs):
+            if mock_run.call_count == 1:
+                return MagicMock(returncode=1, stdout="",
+                                 stderr="sudo: a password is required")
+            target.rmdir()
+            return MagicMock(returncode=0, stdout="", stderr="")
+        mock_run.side_effect = fake_run
+
+        assert sudo_remove_directory(target, allowed_bases=[tmp_path]) is True
+        assert mock_run.call_count == 2
+        first, second = (c.args[0][2] for c in mock_run.call_args_list)
+        assert first != second
+
+    @patch('src.common.permission_utils.subprocess.run')
+    def test_stops_when_the_helper_itself_fails(self, mock_run, tmp_path):
+        target = self._target(tmp_path)
+        mock_run.return_value = MagicMock(returncode=1, stdout="",
+                                          stderr="refusing: not a plugin dir")
+
+        assert sudo_remove_directory(target, allowed_bases=[tmp_path]) is False
+        assert mock_run.call_count == 1
+
+
 class TestInstallRequirementsFileRedaction:
     @patch('src.common.permission_utils.subprocess.run')
     def test_wrapper_path_redacts_stderr_and_stdout(self, mock_run, tmp_path):

test/test_sync_manager.py

@@ -120,7 +120,7 @@ def raise_n_then_stop(mgr, exc, count):
     return _side_effect
 
 
-def fake_clock(monkeypatch, *, time_fn=None, sleep_fn=None):
+def fake_clock(monkeypatch, *, time_fn=None, sleep_fn=None, monotonic_fn=None):
     """Swap sync_manager's own `time` reference for a private stand-in.
 
     sync_manager.time IS the stdlib module, so patching attributes on it
@@ -129,9 +129,14 @@ def fake_clock(monkeypatch, *, time_fn=None, sleep_fn=None):
     hard-to-trace source of cross-test flakiness. Rebinding the module's
     reference keeps the patch scoped to the code under test. Anything not
     overridden falls through to the real functions.
+
+    ``time_fn`` drives both clocks unless ``monotonic_fn`` is given: the
+    timers read time.monotonic(), and a test that only needs "a frozen
+    clock" shouldn't care which one.
     """
     monkeypatch.setattr(sync_manager, "time", SimpleNamespace(
         time=time_fn or time.time,
+        monotonic=monotonic_fn or time_fn or time.monotonic,
         sleep=sleep_fn or time.sleep,
     ))
 
@@ -310,6 +315,34 @@ class TestWatchdogs:
         assert mgr._leader_state is LeaderState.CONNECTED
         assert mgr._peer_ip == "10.0.0.1"
 
+    def test_wall_clock_jump_does_not_time_out_the_peer(self, monkeypatch):
+        # A Pi has no RTC: NTP can step the wall clock by hours after the
+        # peer connected. Only elapsed (monotonic) time counts toward the
+        # heartbeat timeout.
+        mgr = make_manager(role=SyncRole.LEADER)
+        mgr._leader_state = LeaderState.CONNECTED
+        mgr._peer_ip = "10.0.0.1"
+        mgr._last_heartbeat_time = 100.0
+        fake_clock(monkeypatch,
+                   time_fn=lambda: 100.0 + 3600,
+                   monotonic_fn=lambda: 101.0,
+                   sleep_fn=lambda _: setattr(mgr, "_running", False))
+        mgr._running = True
+        mgr._leader_watchdog()
+        assert mgr._leader_state is LeaderState.CONNECTED
+
+    def test_wall_clock_jump_does_not_drop_the_leader(self, monkeypatch):
+        mgr = make_manager(role=SyncRole.FOLLOWER)
+        mgr._follower_state = FollowerState.FOLLOWER
+        mgr._last_leader_frame_time = 100.0
+        fake_clock(monkeypatch,
+                   time_fn=lambda: 100.0 + 3600,
+                   monotonic_fn=lambda: 101.0,
+                   sleep_fn=lambda _: setattr(mgr, "_running", False))
+        mgr._running = True
+        mgr._follower_watchdog()
+        assert mgr._follower_state is FollowerState.FOLLOWER
+
     def test_leader_watchdog_ignores_disconnected_state(self, monkeypatch):
         mgr = make_manager(role=SyncRole.LEADER)
         mgr._leader_state = LeaderState.INCOMPATIBLE
@@ -496,7 +529,8 @@ class TestFollowerRecvLoop:
         mgr = make_manager(role=SyncRole.FOLLOWER)
         sleeps = MagicMock()
         with patch.object(sync_manager, "time",
-                          SimpleNamespace(time=time.time, sleep=sleeps)):
+                          SimpleNamespace(time=time.time, monotonic=time.monotonic,
+                                          sleep=sleeps)):
             self._drive(mgr, b"12345")
         assert mgr.get_latest_frame() is None
         sleeps.assert_not_called()
@@ -507,7 +541,8 @@ class TestFollowerRecvLoop:
             mgr = make_manager(role=SyncRole.FOLLOWER)
             sleeps = MagicMock()
             with patch.object(sync_manager, "time",
-                              SimpleNamespace(time=time.time, sleep=sleeps)):
+                              SimpleNamespace(time=time.time, monotonic=time.monotonic,
+                                              sleep=sleeps)):
                 self._drive(mgr, json.dumps(payload).encode())
             assert mgr.get_latest_scroll_x() is None
             sleeps.assert_not_called()

test/web_interface/test_config_arrays.py

@@ -30,6 +30,22 @@ def test_element_types_are_left_to_normalization():
     assert config["color"] == ["1", "2", "3"]
 
 
+def test_a_nullable_array_union_is_still_an_array():
+    """["array", "null"] is how the per-mode style overrides are typed (null
+    means inherit); their indexed colour inputs must still recombine."""
+    config = {"text_color": {"0": 1, "1": 2, "2": 3}}
+    coerce_array_shapes(config, {"text_color": {"type": ["array", "null"]}})
+    assert config["text_color"] == [1, 2, 3]
+
+
+def test_a_nullable_object_union_is_walked_into():
+    config = {"live": {"tags": {"0": "a"}}}
+    coerce_array_shapes(config, {"live": {
+        "type": ["object", "null"],
+        "properties": {"tags": {"type": "array"}}}})
+    assert config["live"]["tags"] == ["a"]
+
+
 def test_nested_objects_and_array_items_are_walked():
     schema = {"feeds": {"type": "object", "properties": {
         "custom_feeds": {"type": "array", "items": {"type": "object", "properties": {