fix(core): font zip cache, monotonic timers, resolver back-off, and other core/common fixes (#654)

* fix(core): font zip cache, monotonic timers, resolver back-off, and other core/common fixes

- font_manager: a .zip font URL is served as its extracted font after a
  restart (the cached-file check returned the archive first); downloads
  use requests with a 30s timeout into a temp file + os.replace.
- api_helper / sync_manager: rate-limit and heartbeat/leader timeouts use
  time.monotonic(); last_request_time and the status file's ts stay
  wall-clock. set_on_new_cycle docstring no longer claims core uses it.
- logo_helper: the placeholder uses the same scaled box as a real logo.
- permission_utils: one _sudo_bash_candidates() helper (with the sudoers
  exact-argv rationale) shared by sudo_remove_directory, which now retries
  the next bash path on a sudo refusal, and install_requirements_file.
- dynamic_team_resolver: failed/empty fetch backs off 5 min; duplicate
  INFO log and contradictory docstring example fixed.
- element_style: scale default looked up through element aliases.
- background_data_service: cache-hit callback runs outside the lock.
- config_arrays: union-aware type check (["array","null"]); stale
  dotToNested() reference removed.
- auto_update_setup: non-dict auto_update reads as off; temp result file
  unlinked when the write fails.
- exceptions: constructors copy the caller's context dict.
- logging_config: StructuredFormatter json.dumps(default=str).
- error_aggregator: removed unused export_path/export_to_file/_auto_export.
- Docstrings: validate_file_upload max_size_mb, raise_on_errors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix(sync): retry the status-file rename like the other atomic writers

On Windows os.replace can fail with "Access is denied" while a scanner
briefly holds the target open; config_manager_atomic._replace already
retries that (and re-raises at once on other platforms). The sync status
writer called os.replace directly, which made
test_concurrent_writers_each_use_their_own_temp_file flaky on Windows.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

src/common/api_helper.py

@@ -84,7 +84,11 @@ class APIHelper:
         self.session.headers.update({**DEFAULT_HTTP_HEADERS, 'Connection': 'keep-alive'})
         
         # Rate limiting
-        self._last_request_time = 0
+        self._last_request_time = 0  # wall clock, reported by get_request_stats()
+        # The interval is measured on time.monotonic(): a wall-clock step
+        # back (NTP correcting a Pi with no RTC) made time_since_last
+        # negative and the "remaining interval" sleep as long as the step.
+        self._last_request_monotonic: Optional[float] = None
         self._min_request_interval = 1.0  # Minimum seconds between requests
     
     def get(self, url: str, params: Optional[Dict] = None, 
@@ -333,13 +337,14 @@ class APIHelper:
     
     def _enforce_rate_limit(self) -> None:
         """Enforce rate limiting between requests."""
-        current_time = time.time()
-        time_since_last = current_time - self._last_request_time
-        
-        if time_since_last < self._min_request_interval:
-            sleep_time = self._min_request_interval - time_since_last
-            time.sleep(sleep_time)
-        
+        if self._last_request_monotonic is not None:
+            time_since_last = time.monotonic() - self._last_request_monotonic
+
+            if time_since_last < self._min_request_interval:
+                sleep_time = self._min_request_interval - time_since_last
+                time.sleep(sleep_time)
+
+        self._last_request_monotonic = time.monotonic()
         self._last_request_time = time.time()
     
     def set_rate_limit(self, min_interval: float) -> None:
@@ -362,5 +367,8 @@ class APIHelper:
         return {
             'min_request_interval': self._min_request_interval,
             'last_request_time': self._last_request_time,
-            'time_since_last_request': time.time() - self._last_request_time
+            'time_since_last_request': (
+                time.monotonic() - self._last_request_monotonic
+                if self._last_request_monotonic is not None
+                else time.time() - self._last_request_time),
         }

src/common/logo_helper.py

@@ -8,7 +8,7 @@ Extracted from LEDMatrix core to provide reusable functionality for plugins.
 import logging
 import time
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import requests
 from PIL import Image, ImageDraw
@@ -125,17 +125,7 @@ class LogoHelper:
         # Resolve the effective target size BEFORE the cache lookup so the
         # key is size-qualified — a panel-size change must not return a
         # logo resized for the old dimensions.
-        if max_width is None:
-            max_width = int(self.display_width * DEFAULT_LOGO_BOX_FACTOR)
-        if max_height is None:
-            max_height = int(self.display_height * DEFAULT_LOGO_BOX_FACTOR)
-        # Imported here: src.element_style imports src.common (for bdf_font),
-        # whose __init__ imports this module.
-        from src.element_style import coerce_scale
-        scale = coerce_scale(scale, 1.0)
-        if scale != 1.0:
-            max_width = max(1, int(round(max_width * scale)))
-            max_height = max(1, int(round(max_height * scale)))
+        max_width, max_height, scale = self._scaled_box(max_width, max_height, scale)
         # The key carries the scaled box, so two elements scaled differently
         # cannot be served each other's image.
         cache_key = f"{team_abbr}_{logo_path}_{max_width}x{max_height}"
@@ -236,8 +226,30 @@ class LogoHelper:
                 # exists to prevent.
                 self._refresh_stale_placeholder(logo_path)
         
-        # Create placeholder if all else fails
-        return self._create_placeholder_logo(team_abbr, max_width, max_height)
+        # Create placeholder if all else fails. Sized to the same scaled box
+        # a real logo gets, so a scaled element doesn't jump in size while
+        # its logo is missing.
+        box_width, box_height, _ = self._scaled_box(max_width, max_height, scale)
+        return self._create_placeholder_logo(team_abbr, box_width, box_height)
+
+    def _scaled_box(self, max_width: Optional[int], max_height: Optional[int],
+                    scale: float) -> Tuple[int, int, float]:
+        """The logo box after defaults and the user's scale are applied.
+
+        Returns ``(width, height, coerced_scale)``.
+        """
+        if max_width is None:
+            max_width = int(self.display_width * DEFAULT_LOGO_BOX_FACTOR)
+        if max_height is None:
+            max_height = int(self.display_height * DEFAULT_LOGO_BOX_FACTOR)
+        # Imported here: src.element_style imports src.common (for bdf_font),
+        # whose __init__ imports this module.
+        from src.element_style import coerce_scale
+        scale = coerce_scale(scale, 1.0)
+        if scale != 1.0:
+            max_width = max(1, int(round(max_width * scale)))
+            max_height = max(1, int(round(max_height * scale)))
+        return max_width, max_height, scale
     
     def _invalidate_cached_logo(self, team_abbr: str, logo_path: Path) -> None:
         """Drop every cached size of one logo after its file changed on disk."""

src/common/permission_utils.py

@@ -290,6 +290,26 @@ def get_cache_dir_mode() -> int:
     return 0o2775  # rwxrwsr-x (setgid + group writable)
 
 
+def _sudo_bash_candidates() -> list:
+    """Bash paths to try, in order, when running a vetted helper via sudo.
+
+    sudoers matches the exact argv, so ``sudo -n <bash> <helper> ...`` only
+    works if <bash> is the same path configure_web_sudo.sh wrote into the
+    rule -- whatever ``command -v bash`` said on the machine that ran it.
+    On merged-/usr systems /usr/bin/bash and /bin/bash are the same file but
+    different strings to sudo, and the web user's PATH can differ from the
+    installer's, so no single guess is reliable. Callers try each in turn and
+    move on only when sudo refused the command line (SUDO_REFUSAL_PHRASES).
+    The helper is invoked through bash rather than its shebang for the same
+    reason: the rule names bash, not the script.
+    """
+    candidates = []
+    for candidate in ("/usr/bin/bash", "/bin/bash", _shutil.which("bash")):
+        if candidate and candidate not in candidates:
+            candidates.append(candidate)
+    return candidates
+
+
 def sudo_remove_directory(path: Path, allowed_bases: Optional[list] = None) -> bool:
     """
     Remove a directory using sudo as a last resort.
@@ -350,22 +370,25 @@ def sudo_remove_directory(path: Path, allowed_bases: Optional[list] = None) -> b
         logger.error(f"Safe removal helper not found: {helper_script}")
         return False
 
-    bash_path = _shutil.which('bash') or '/bin/bash'
-
     try:
-        result = subprocess.run(
-            ['sudo', '-n', bash_path, str(helper_script), str(resolved)],
-            capture_output=True,
-            text=True,
-            timeout=30
-        )
-        if result.returncode == 0 and not resolved.exists():
-            logger.info(f"Successfully removed {path} via sudo helper")
-            return True
-        else:
-            stderr = result.stderr.strip()
-            logger.error(f"sudo helper failed for {path}: {stderr}")
-            return False
+        for bash_path in _sudo_bash_candidates():
+            result = subprocess.run(
+                ['sudo', '-n', bash_path, str(helper_script), str(resolved)],
+                capture_output=True,
+                text=True,
+                timeout=30
+            )
+            if result.returncode == 0 and not resolved.exists():
+                logger.info(f"Successfully removed {path} via sudo helper")
+                return True
+            # Only a refused command line is worth another bash path; if the
+            # helper itself ran and failed, a retry would just repeat it.
+            if result.returncode == 0 or not any(
+                    phrase in (result.stderr or '') for phrase in SUDO_REFUSAL_PHRASES):
+                break
+        stderr = (result.stderr or '').strip()
+        logger.error(f"sudo helper failed for {path}: {stderr}")
+        return False
     except subprocess.TimeoutExpired:
         logger.error(f"sudo helper timed out for {path}")
         return False
@@ -417,16 +440,10 @@ def install_requirements_file(req_file: Path, timeout: int = 300) -> subprocess.
     wrapper = project_root / "scripts" / "fix_perms" / "safe_pip_install.sh"
 
     if wrapper.exists():
-        # See sudo_remove_directory / configure_web_sudo.sh for why bash must
-        # be invoked with an explicit, known path rather than relying on the
-        # wrapper's shebang: sudoers matches the exact command line.
-        bash_candidates = []
-        for candidate in ("/usr/bin/bash", "/bin/bash", _shutil.which("bash")):
-            if candidate and candidate not in bash_candidates:
-                bash_candidates.append(candidate)
-
+        # See _sudo_bash_candidates for why bash is invoked by explicit path
+        # and why there is more than one to try.
         result = None
-        for bash_path in bash_candidates:
+        for bash_path in _sudo_bash_candidates():
             # bash_path and wrapper are fixed, known-good paths, and
             # safe_pip_install.sh independently re-validates req_file is an
             # allowed requirements.txt before installing anything as root.

src/common/sync_manager.py

@@ -32,6 +32,7 @@ from typing import Callable, Optional
 import numpy as np
 from PIL import Image
 
+from src.config_manager_atomic import _replace
 from src.display_geometry import DEFAULT_CHAIN_LENGTH, DEFAULT_COLS, DEFAULT_ROWS
 
 # Raw-frame wire format: 8-byte magic + 4-byte header + raw RGB pixels
@@ -127,6 +128,9 @@ class DisplaySyncManager:
         self._peer_ip: Optional[str] = None
         self._peer_compatible: bool = False
         self._peer_chain: int = 0
+        # time.monotonic() readings, like _last_leader_frame_time: these only
+        # feed the timeout watchdogs, and a wall-clock step (NTP correcting a
+        # Pi with no RTC) would otherwise fake or mask a timeout.
         self._last_heartbeat_time: float = 0.0
         self._leader_width: int = 0  # set by display_controller after init
         self._oversized_frame_warned: bool = False
@@ -206,7 +210,7 @@ class DisplaySyncManager:
                     self._handle_hello(msg, sender_ip)
                 elif t == "hb":
                     if self._peer_ip == sender_ip:
-                        self._last_heartbeat_time = time.time()
+                        self._last_heartbeat_time = time.monotonic()
             except socket.timeout:
                 continue
             except Exception as exc:
@@ -229,7 +233,7 @@ class DisplaySyncManager:
         self._peer_ip = sender_ip
         self._peer_compatible = compatible
         self._peer_chain = peer_chain
-        self._last_heartbeat_time = time.time()
+        self._last_heartbeat_time = time.monotonic()
 
         prev_state = self._leader_state
         if compatible:
@@ -273,7 +277,7 @@ class DisplaySyncManager:
         while self._running:
             time.sleep(1.0)
             if self._leader_state == LeaderState.CONNECTED:
-                if time.time() - self._last_heartbeat_time > PEER_TIMEOUT:
+                if time.monotonic() - self._last_heartbeat_time > PEER_TIMEOUT:
                     self.logger.info(
                         "Sync: follower heartbeat timeout — peer disconnected"
                     )
@@ -501,7 +505,7 @@ class DisplaySyncManager:
         """Note that the leader at ``sender_ip`` just sent something, and
         switch from standalone to follower mode if not already following.
         Returns True if this call made the switch."""
-        self._last_leader_frame_time = time.time()
+        self._last_leader_frame_time = time.monotonic()
         self._leader_ip = sender_ip
         if self._follower_state != FollowerState.STANDALONE:
             return False
@@ -621,11 +625,13 @@ class DisplaySyncManager:
         heartbeat = json.dumps({"t": "hb"}).encode("utf-8")
         dest = ("<broadcast>", self.port)
 
-        last_hello = 0.0
-        last_hb = 0.0
+        # -inf, not 0.0: monotonic time starts near boot, so "now - 0.0" can
+        # be under the interval and would delay the first announcement.
+        last_hello = float("-inf")
+        last_hb = float("-inf")
 
         while self._running:
-            now = time.time()
+            now = time.monotonic()
             if now - last_hello >= HELLO_INTERVAL:
                 try:
                     self._send_sock.sendto(hello, dest)
@@ -644,7 +650,7 @@ class DisplaySyncManager:
         while self._running:
             time.sleep(1.0)
             if self._follower_state == FollowerState.FOLLOWER:
-                if time.time() - self._last_leader_frame_time > LEADER_TIMEOUT:
+                if time.monotonic() - self._last_leader_frame_time > LEADER_TIMEOUT:
                     self.logger.info(
                         "Sync: leader frame timeout — returning to standalone mode"
                     )
@@ -670,7 +676,10 @@ class DisplaySyncManager:
 
     def set_on_new_cycle(self, callback: Callable[[], None]) -> None:
         """Follower: register a callback fired when the leader starts a new scroll cycle.
-        Used to trigger a local start_new_cycle() so both Pis rebuild from same fresh data.
+
+        Nothing in core registers one: display_controller follows the leader
+        through set_on_scroll_image() and the scroll position instead of
+        rebuilding locally. The hook stays for callers that want the signal.
         """
         self._on_new_cycle = callback
 
@@ -734,7 +743,9 @@ class DisplaySyncManager:
                 # mkstemp makes it owner-only; the web UI may run as a
                 # different user from the display service.
                 os.chmod(tmp, 0o644)
-                os.replace(tmp, STATUS_FILE)
+                # _replace: on Windows a rename can briefly fail with
+                # "Access is denied" while a scanner holds the target open.
+                _replace(tmp, STATUS_FILE)
                 tmp = None
         except Exception as exc:
             self.logger.debug("Sync: status file write error: %s", exc)