Compare commits

..
Author SHA1 Message Date
ChuckandClaude Fable 5 ced173e95a test: sync mock cache manager signature with CacheManager.get
test_circuit_breaker has been failing on main: plugin_health passes
memory_ttl= to cache_manager.get(), and the conftest mock's signature
was never updated — the same component/double drift class as the
monitored_update bug (#392).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FqzC1nzTWL4kaqgMaQZFam
2026-07-12 10:31:52 -04:00
ChuckandClaude Fable 5 acfd704d13 fix(core): harden text-measurement caches; surface snapshot failures
Deep-dive findings, all three latent on every 24/7 install:

- font_manager.metrics_cache and display_manager._text_width_cache were
  unbounded dicts keyed by (text, id(font)). Two problems: keys embed
  the measured TEXT, so ever-changing strings (a clock, a live score, a
  ticker) grow them without limit; and id()-keying without holding a
  reference means a garbage-collected font's id can be recycled by a
  DIFFERENT font, silently returning wrong widths/metrics (classic
  plugins create fonts per render, so this is reachable). Both caches
  are now LRU-bounded (1024) and pin the font in the entry so its id
  stays valid. metrics_cache also keyed on the text itself instead of
  hash(text), removing a collision path.

- _write_snapshot_if_due logged failures at DEBUG — invisible at the
  default level. The snapshot's mtime is the web UI's display mirror
  AND its hardware-liveness proxy, so a quiet failure freezes the
  mirror and makes health checks lie (seen in the field: a stale
  root-owned /tmp file froze it for a day). Failures now WARN, rate-
  limited to once per 5 minutes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FqzC1nzTWL4kaqgMaQZFam
2026-07-12 10:29:20 -04:00
5 changed files with 67 additions and 50 deletions
+5 -20
View File
@@ -199,10 +199,6 @@ class DisplayController:
self.wifi_status_file = WIFI_STATUS_FILE self.wifi_status_file = WIFI_STATUS_FILE
self.wifi_status_active = False self.wifi_status_active = False
self.wifi_status_expires_at: Optional[float] = None self.wifi_status_expires_at: Optional[float] = None
# _check_wifi_status_message throttle state (checked at frame rate,
# stat'd at most once per second)
self._wifi_status_check_ts = 0.0
self._wifi_status_last_result: Optional[Dict[str, Any]] = None
# Plugin display() signature cache — must be initialised before the plugin # Plugin display() signature cache — must be initialised before the plugin
# loading loop below so the .pop() invalidation at load time is always safe. # loading loop below so the .pop() invalidation at load time is always safe.
@@ -1639,7 +1635,7 @@ class DisplayController:
self._sleep_with_plugin_updates(60) self._sleep_with_plugin_updates(60)
continue continue
logger.debug("Display active, processing mode: %s", self.current_display_mode) logger.info(f"Display active, processing mode: {self.current_display_mode}")
# Plugins update on their own schedules - no forced sync updates needed # Plugins update on their own schedules - no forced sync updates needed
# Each plugin has its own update_interval and background services # Each plugin has its own update_interval and background services
@@ -1807,7 +1803,7 @@ class DisplayController:
if self.plugin_manager and hasattr(self.plugin_manager, 'health_tracker') and self.plugin_manager.health_tracker: if self.plugin_manager and hasattr(self.plugin_manager, 'health_tracker') and self.plugin_manager.health_tracker:
should_skip = self.plugin_manager.health_tracker.should_skip_plugin(plugin_id) should_skip = self.plugin_manager.health_tracker.should_skip_plugin(plugin_id)
if should_skip: if should_skip:
logger.info("Skipping plugin %s due to circuit breaker (mode: %s)", plugin_id, active_mode) logger.info(f"Skipping plugin {plugin_id} due to circuit breaker (mode: {active_mode})")
display_result = False display_result = False
# Skip to next mode - let existing logic handle it # Skip to next mode - let existing logic handle it
manager_to_display = None manager_to_display = None
@@ -1865,7 +1861,7 @@ class DisplayController:
if isinstance(result, bool): if isinstance(result, bool):
display_result = result display_result = result
if not display_result: if not display_result:
logger.info("Plugin %s display() returned False for mode %s", plugin_id, active_mode) logger.info(f"Plugin {plugin_id} display() returned False for mode {active_mode}")
# Record success if display completed without exception # Record success if display completed without exception
if self.plugin_manager and hasattr(self.plugin_manager, 'health_tracker') and self.plugin_manager.health_tracker: if self.plugin_manager and hasattr(self.plugin_manager, 'health_tracker') and self.plugin_manager.health_tracker:
@@ -2358,16 +2354,6 @@ class DisplayController:
Returns None on any error or if message is expired/invalid. Returns None on any error or if message is expired/invalid.
""" """
try: try:
# Throttle the existence stat to ~1 Hz: this runs on every render
# iteration (60+ fps), and the file usually doesn't exist — the
# status message's lifetime is measured in seconds anyway.
# Both attributes are initialised in __init__.
now = time.time()
if (now - self._wifi_status_check_ts) < 1.0:
return self._wifi_status_last_result
self._wifi_status_check_ts = now
self._wifi_status_last_result = None
# Check if file exists # Check if file exists
if not self.wifi_status_file or not self.wifi_status_file.exists(): if not self.wifi_status_file or not self.wifi_status_file.exists():
return None return None
@@ -2418,14 +2404,13 @@ class DisplayController:
pass pass
return None return None
# Message is valid and not expired — cache for the throttle window # Message is valid and not expired
self._wifi_status_last_result = { return {
'message': message, 'message': message,
'timestamp': timestamp, 'timestamp': timestamp,
'duration': duration, 'duration': duration,
'expires_at': expires_at 'expires_at': expires_at
} }
return self._wifi_status_last_result
except Exception as e: except Exception as e:
# Catch-all for any unexpected errors - log but don't break the display # Catch-all for any unexpected errors - log but don't break the display
+35 -8
View File
@@ -33,7 +33,8 @@ else:
from contextlib import contextmanager from contextlib import contextmanager
from PIL import Image, ImageDraw, ImageFont from PIL import Image, ImageDraw, ImageFont
import time import time
from typing import Dict, Any, List, Optional from collections import OrderedDict
from typing import Dict, Any, List, Optional, Tuple
import logging import logging
import math import math
import freetype import freetype
@@ -180,14 +181,25 @@ class DisplayManager:
# the logical image is blitted to the matrix unchanged. # the logical image is blitted to the matrix unchanged.
self._double_sided = None # dict {copies, axis, logical_width, logical_height} or None self._double_sided = None # dict {copies, axis, logical_width, logical_height} or None
self._physical_image = None # full-chain buffer reused each frame when tiling self._physical_image = None # full-chain buffer reused each frame when tiling
# Text-width measurement cache: (text, id(font)) -> pixel_width # Text-width measurement cache: (text, id(font)) -> (width, font_ref)
# Avoids re-measuring the same string+font on every display() call. # Avoids re-measuring the same string+font on every display() call.
# LRU-bounded: keys embed the TEXT, so changing strings (a clock, a
# live score) would otherwise grow it forever on a 24/7 service.
# Entries hold a strong reference to the font so its id() can't be
# recycled by a different font object — an id-keyed cache without
# the reference can return the WRONG width after garbage collection.
# Cleared on _load_fonts() so stale entries don't survive a font reload. # Cleared on _load_fonts() so stale entries don't survive a font reload.
self._text_width_cache: Dict[tuple, int] = {} self._text_width_cache: "OrderedDict[tuple, Tuple[int, Any]]" = OrderedDict()
self._TEXT_WIDTH_CACHE_MAX = 1024
# Snapshot settings for web preview integration (service writes, web reads) # Snapshot settings for web preview integration (service writes, web reads)
self._snapshot_path = "/tmp/led_matrix_preview.png" # nosec B108 - fixed path intentional; web UI reads same path self._snapshot_path = "/tmp/led_matrix_preview.png" # nosec B108 - fixed path intentional; web UI reads same path
self._snapshot_min_interval_sec = 0.2 # max ~5 fps self._snapshot_min_interval_sec = 0.2 # max ~5 fps
self._last_snapshot_ts = 0.0 self._last_snapshot_ts = 0.0
# Snapshot failures are logged as warnings, rate-limited so a
# persistent failure (e.g. an unwritable file) can't spam the log —
# but is never silent: the snapshot's mtime doubles as the web UI's
# hardware-liveness signal, so a quiet failure makes health checks lie.
self._snapshot_fail_log_ts = 0.0
# Scrolling state tracking for graceful updates # Scrolling state tracking for graceful updates
self._scrolling_state = { self._scrolling_state = {
@@ -699,12 +711,15 @@ class DisplayManager:
Results are cached by (text, font identity) so plugins that measure Results are cached by (text, font identity) so plugins that measure
the same string every frame (e.g. to centre a score) pay only one the same string every frame (e.g. to centre a score) pay only one
measurement per unique (text, font) pair. measurement per unique (text, font) pair. The entry keeps the font
alive so its id() can't be recycled, and the cache is LRU-bounded so
ever-changing text (clocks, tickers) can't grow it without limit.
""" """
cache_key = (text, id(font)) cache_key = (text, id(font))
cached = self._text_width_cache.get(cache_key) cached = self._text_width_cache.get(cache_key)
if cached is not None: if cached is not None:
return cached self._text_width_cache.move_to_end(cache_key)
return cached[0]
try: try:
if isinstance(font, freetype.Face): if isinstance(font, freetype.Face):
@@ -719,7 +734,9 @@ class DisplayManager:
logger.error("Error getting text width: %s", e) logger.error("Error getting text width: %s", e)
return 0 return 0
self._text_width_cache[cache_key] = width self._text_width_cache[cache_key] = (width, font)
while len(self._text_width_cache) > self._TEXT_WIDTH_CACHE_MAX:
self._text_width_cache.popitem(last=False)
return width return width
def get_font_height(self, font): def get_font_height(self, font):
@@ -1164,5 +1181,15 @@ class DisplayManager:
pass pass
self._last_snapshot_ts = now self._last_snapshot_ts = now
except Exception as e: except Exception as e:
# Snapshot failures should never break display; log at debug to avoid noise # Snapshot failures must never break display — but they must not
logger.debug(f"Snapshot write skipped: {e}") # be silent either: the snapshot's mtime is the web UI's display
# mirror AND its hardware-liveness proxy, so a quietly failing
# write freezes the mirror and makes health checks lie (seen in
# the field: a stale root-owned /tmp file froze it for a day).
# Warn at most once per 5 minutes to avoid log spam.
if (now - self._snapshot_fail_log_ts) > 300:
self._snapshot_fail_log_ts = now
logger.warning("Snapshot write failing (web preview/health "
"mirror is stale): %s", e)
else:
logger.debug(f"Snapshot write skipped: {e}")
+18 -5
View File
@@ -35,6 +35,7 @@ import urllib.request
import zipfile import zipfile
import tempfile import tempfile
import time import time
from collections import OrderedDict
from pathlib import Path from pathlib import Path
from PIL import ImageFont from PIL import ImageFont
from typing import Dict, Tuple, Optional, Union, Any, List from typing import Dict, Tuple, Optional, Union, Any, List
@@ -58,7 +59,13 @@ class FontManager:
# Font discovery and catalog # Font discovery and catalog
self.font_catalog: Dict[str, str] = {} # family_name -> file_path self.font_catalog: Dict[str, str] = {} # family_name -> file_path
self.font_cache: Dict[str, Union[ImageFont.FreeTypeFont, freetype.Face]] = {} # (family, size) -> font self.font_cache: Dict[str, Union[ImageFont.FreeTypeFont, freetype.Face]] = {} # (family, size) -> font
self.metrics_cache: Dict[str, Tuple[int, int, int]] = {} # (text, font_id) -> (width, height, baseline) # (text, id(font)) -> ((width, height, baseline), font_ref).
# LRU-bounded — keys embed the measured TEXT, so changing strings
# (clocks, live scores) would otherwise grow it forever. Entries
# keep the font alive so its id() can't be recycled by a different
# font object (which would silently return wrong metrics).
self.metrics_cache: "OrderedDict[Any, Tuple[Tuple[int, int, int], Any]]" = OrderedDict()
self._METRICS_CACHE_MAX = 1024
# Plugin font management # Plugin font management
self.plugin_fonts: Dict[str, Dict[str, Any]] = {} # plugin_id -> font_manifest self.plugin_fonts: Dict[str, Dict[str, Any]] = {} # plugin_id -> font_manifest
@@ -507,10 +514,14 @@ class FontManager:
Returns: Returns:
Tuple of (width, height, baseline_offset) Tuple of (width, height, baseline_offset)
""" """
cache_key = f"{hash(text)}_{id(font)}" # Key on the text itself (hash(text) could collide) + font identity;
# the entry below keeps the font referenced so the id stays valid.
cache_key = (text, id(font))
if cache_key in self.metrics_cache: cached = self.metrics_cache.get(cache_key)
return self.metrics_cache[cache_key] if cached is not None:
self.metrics_cache.move_to_end(cache_key)
return cached[0]
try: try:
if isinstance(font, freetype.Face): if isinstance(font, freetype.Face):
@@ -547,7 +558,9 @@ class FontManager:
baseline = 10 baseline = 10
result = (width, height, baseline) result = (width, height, baseline)
self.metrics_cache[cache_key] = result self.metrics_cache[cache_key] = (result, font)
while len(self.metrics_cache) > self._METRICS_CACHE_MAX:
self.metrics_cache.popitem(last=False)
return result return result
def get_font_height(self, font: Union[ImageFont.FreeTypeFont, freetype.Face]) -> int: def get_font_height(self, font: Union[ImageFont.FreeTypeFont, freetype.Face]) -> int:
+4 -16
View File
@@ -66,10 +66,6 @@ class RenderPipeline:
else display_manager.height else display_manager.height
) )
# Reusable blank frame for cycle-end pushes (allocated lazily,
# re-blacked before each reuse)
self._blank_frame = None
# ScrollHelper for optimized scrolling # ScrollHelper for optimized scrolling
self.scroll_helper = ScrollHelper( self.scroll_helper = ScrollHelper(
self.display_width, self.display_width,
@@ -238,19 +234,11 @@ class RenderPipeline:
) )
# Push blank immediately so the hardware never shows any # Push blank immediately so the hardware never shows any
# post-wrap content while the coordinator recomposes the # post-wrap content while the coordinator recomposes the
# next cycle (~100 ms). The blank is allocated once and # next cycle (~100 ms).
# reused across cycle wraps (fresh paste each time in case
# a consumer drew on the previous one).
try: try:
if self._blank_frame is None or self._blank_frame.size != ( from PIL import Image as _Image
self.display_width, self.display_height): blank = _Image.new('RGB', (self.display_width, self.display_height))
self._blank_frame = Image.new( self.display_manager.image = blank
'RGB', (self.display_width, self.display_height))
else:
self._blank_frame.paste(
(0, 0, 0),
(0, 0, self.display_width, self.display_height))
self.display_manager.image = self._blank_frame
self.display_manager.update_display() self.display_manager.update_display()
except Exception: except Exception:
logger.exception("Failed to write blank frame to display at cycle end") logger.exception("Failed to write blank frame to display at cycle end")
+5 -1
View File
@@ -38,7 +38,11 @@ def mock_cache_manager():
mock._memory_cache_timestamps = {} mock._memory_cache_timestamps = {}
mock.cache_dir = "/tmp/test_cache" mock.cache_dir = "/tmp/test_cache"
def mock_get(key: str, max_age: int = 300) -> Optional[Dict]: def mock_get(key: str, max_age: Optional[int] = 300,
memory_ttl: Optional[int] = None) -> Optional[Dict]:
# Signature mirrors CacheManager.get — keep in sync or callers
# passing keyword args (health tracker, resource monitor) break
# only in tests, hiding real-API compatibility.
return mock._memory_cache.get(key) return mock._memory_cache.get(key)
def mock_set(key: str, data: Dict, ttl: Optional[int] = None) -> None: def mock_set(key: str, data: Dict, ttl: Optional[int] = None) -> None: