mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-08-21 10:29:06 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a4290e8a28 |
Binary file not shown.
|
After Width: | Height: | Size: 76 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 128 KiB |
@@ -328,7 +328,11 @@ class ScrollHelper:
|
|||||||
elapsed_time = current_time - (self.scroll_start_time or current_time)
|
elapsed_time = current_time - (self.scroll_start_time or current_time)
|
||||||
# The image already includes display_width padding, so we only need total_scroll_width
|
# The image already includes display_width padding, so we only need total_scroll_width
|
||||||
required_total_distance = self.total_scroll_width
|
required_total_distance = self.total_scroll_width
|
||||||
self.logger.info(
|
# Progress telemetry, emitted every few seconds for the whole of
|
||||||
|
# every scroll. It says how far along a marquee is, which is what
|
||||||
|
# you turn debug on to watch and not something an operator needs
|
||||||
|
# in the journal on a device that scrolls all day.
|
||||||
|
self.logger.debug(
|
||||||
"Scroll progress: elapsed=%.2fs, target=%.2fs, total_scrolled=%.0f/%d px (%.1f%%)",
|
"Scroll progress: elapsed=%.2fs, target=%.2fs, total_scrolled=%.0f/%d px (%.1f%%)",
|
||||||
elapsed_time,
|
elapsed_time,
|
||||||
self.calculated_duration,
|
self.calculated_duration,
|
||||||
|
|||||||
@@ -178,20 +178,10 @@ class PluginHealthTracker:
|
|||||||
)
|
)
|
||||||
return self._health_state[plugin_id]
|
return self._health_state[plugin_id]
|
||||||
|
|
||||||
# Fields the circuit breaker is rebuilt from after a restart. Everything
|
|
||||||
# else in a health record is reporting, read only for display.
|
|
||||||
_DURABLE_FIELDS = ('consecutive_failures', 'circuit_state',
|
|
||||||
'circuit_opened_time', 'half_open_start_time')
|
|
||||||
|
|
||||||
def _durable(self, state: Dict[str, Any]) -> tuple:
|
|
||||||
"""The part of a health record whose loss would change behaviour."""
|
|
||||||
return tuple(state.get(field) for field in self._DURABLE_FIELDS)
|
|
||||||
|
|
||||||
def record_success(self, plugin_id: str) -> None:
|
def record_success(self, plugin_id: str) -> None:
|
||||||
"""Record a successful plugin execution."""
|
"""Record a successful plugin execution."""
|
||||||
state = self.get_health_state(plugin_id)
|
state = self.get_health_state(plugin_id)
|
||||||
current_time = time.time()
|
current_time = time.time()
|
||||||
durable_before = self._durable(state)
|
|
||||||
|
|
||||||
# Reset consecutive failures
|
# Reset consecutive failures
|
||||||
state['consecutive_failures'] = 0
|
state['consecutive_failures'] = 0
|
||||||
@@ -209,17 +199,6 @@ class PluginHealthTracker:
|
|||||||
state['circuit_state'] = CircuitState.CLOSED.value
|
state['circuit_state'] = CircuitState.CLOSED.value
|
||||||
state['circuit_opened_time'] = None
|
state['circuit_opened_time'] = None
|
||||||
|
|
||||||
# A healthy plugin reports success every cycle, and in that steady state
|
|
||||||
# the only fields changed above are a counter and a timestamp that
|
|
||||||
# nothing reads back after a restart. Persisting them anyway rewrites a
|
|
||||||
# small file per plugin per cycle: on a rig running 24 plugins, a
|
|
||||||
# five-minute sample measured 22 rewrites, about 4.4 a minute or 6,300 a
|
|
||||||
# day. Those land on an SD card, where the cost is an erase-block cycle
|
|
||||||
# rather than the 400 bytes involved, and where wear is what eventually
|
|
||||||
# kills the card.
|
|
||||||
# In-memory state is still updated every time, so the health API and web
|
|
||||||
# UI show exactly what they did before; only the write is skipped.
|
|
||||||
if self._durable(state) != durable_before:
|
|
||||||
self._save_health_state(plugin_id, state)
|
self._save_health_state(plugin_id, state)
|
||||||
|
|
||||||
def record_failure(self, plugin_id: str, error: Optional[Exception] = None) -> None:
|
def record_failure(self, plugin_id: str, error: Optional[Exception] = None) -> None:
|
||||||
|
|||||||
@@ -31,6 +31,14 @@ if TYPE_CHECKING:
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
#: A frame rate this close to target is not news; below it is.
|
||||||
|
_FPS_HEALTHY_FRACTION = 0.9
|
||||||
|
|
||||||
|
#: A healthy marquee still reports this often, so silence means stopped
|
||||||
|
#: rather than fine.
|
||||||
|
_FPS_HEARTBEAT_INTERVAL = 300.0
|
||||||
|
|
||||||
|
|
||||||
def _percentile(ordered: List[float], fraction: float) -> float:
|
def _percentile(ordered: List[float], fraction: float) -> float:
|
||||||
"""Nearest-rank percentile of an already-sorted list.
|
"""Nearest-rank percentile of an already-sorted list.
|
||||||
|
|
||||||
@@ -395,7 +403,9 @@ class VegasModeCoordinator:
|
|||||||
duration = self.render_pipeline.get_dynamic_duration()
|
duration = self.render_pipeline.get_dynamic_duration()
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
frame_count = 0
|
frame_count = 0
|
||||||
fps_log_interval = 5.0 # Log FPS every 5 seconds
|
fps_log_interval = 5.0 # Sample FPS every 5 seconds
|
||||||
|
last_fps_health_log = 0.0 # last INFO-level report
|
||||||
|
was_degraded = False # so the recovery is reported too
|
||||||
last_fps_log_time = start_time
|
last_fps_log_time = start_time
|
||||||
fps_frame_count = 0
|
fps_frame_count = 0
|
||||||
# A mean hides stutter completely. At 120fps a five-second window is
|
# A mean hides stutter completely. At 120fps a five-second window is
|
||||||
@@ -448,16 +458,36 @@ class VegasModeCoordinator:
|
|||||||
frame_count += 1
|
frame_count += 1
|
||||||
fps_frame_count += 1
|
fps_frame_count += 1
|
||||||
|
|
||||||
# Periodic FPS logging
|
# Periodic FPS logging. Reported at INFO only when the frame rate
|
||||||
|
# is actually worth an operator's attention -- a shortfall against
|
||||||
|
# target, or the recovery from one -- with a slow heartbeat so a
|
||||||
|
# healthy marquee still shows a pulse.
|
||||||
|
#
|
||||||
|
# Measured over two hours on a running rig: 1410 samples, 98.5%
|
||||||
|
# of them within 10% of target. The 1.5% that were not included a
|
||||||
|
# reading of 8.6fps against a target of 60 -- a real stall, and
|
||||||
|
# completely invisible inside 1389 lines reading "59.6".
|
||||||
current_time = time.time()
|
current_time = time.time()
|
||||||
if current_time - last_fps_log_time >= fps_log_interval:
|
if current_time - last_fps_log_time >= fps_log_interval:
|
||||||
fps = fps_frame_count / (current_time - last_fps_log_time)
|
fps = fps_frame_count / (current_time - last_fps_log_time)
|
||||||
p99 = _percentile(sorted(frame_times), 0.99)
|
p99 = _percentile(sorted(frame_times), 0.99)
|
||||||
|
target = self.vegas_config.target_fps
|
||||||
|
degraded = target > 0 and fps < target * _FPS_HEALTHY_FRACTION
|
||||||
|
due = current_time - last_fps_health_log >= _FPS_HEARTBEAT_INTERVAL
|
||||||
|
if degraded or was_degraded or due:
|
||||||
logger.info(
|
logger.info(
|
||||||
"Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms",
|
"Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms",
|
||||||
fps, self.vegas_config.target_fps, fps_frame_count,
|
fps, target, fps_frame_count,
|
||||||
p99 * 1000.0, frame_worst * 1000.0
|
p99 * 1000.0, frame_worst * 1000.0
|
||||||
)
|
)
|
||||||
|
last_fps_health_log = current_time
|
||||||
|
else:
|
||||||
|
logger.debug(
|
||||||
|
"Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms",
|
||||||
|
fps, target, fps_frame_count,
|
||||||
|
p99 * 1000.0, frame_worst * 1000.0
|
||||||
|
)
|
||||||
|
was_degraded = degraded
|
||||||
last_fps_log_time = current_time
|
last_fps_log_time = current_time
|
||||||
fps_frame_count = 0
|
fps_frame_count = 0
|
||||||
frame_worst = 0.0
|
frame_worst = 0.0
|
||||||
|
|||||||
@@ -1,110 +0,0 @@
|
|||||||
"""A healthy plugin must not rewrite its health record every cycle.
|
|
||||||
|
|
||||||
Every successful plugin update called record_success(), which persisted the
|
|
||||||
record unconditionally. In steady state the only fields that had changed were
|
|
||||||
total_successes and last_success_time -- a counter and a timestamp that
|
|
||||||
health_monitor reads for display and that nothing reads back after a restart.
|
|
||||||
|
|
||||||
Measured on a rig running 24 plugins: about 17 health-file rewrites a minute,
|
|
||||||
roughly 25,000 a day. Each is ~400 bytes, but they land on an SD card where
|
|
||||||
the unit of cost is an erase-block cycle, not the byte count, and where wear is
|
|
||||||
what eventually kills the card.
|
|
||||||
|
|
||||||
The circuit breaker still needs its own state to survive a restart, so the
|
|
||||||
write is kept for exactly the fields it is rebuilt from -- and a failure, a
|
|
||||||
circuit opening, or a recovery must still be written the moment it happens.
|
|
||||||
"""
|
|
||||||
import time
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from src.plugin_system.plugin_health import PluginHealthTracker, CircuitState
|
|
||||||
|
|
||||||
|
|
||||||
class _Cache:
|
|
||||||
"""Counts writes; serves back whatever was last written."""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.store = {}
|
|
||||||
self.writes = 0
|
|
||||||
|
|
||||||
def set(self, key, data, ttl=None, **kwargs):
|
|
||||||
self.writes += 1
|
|
||||||
self.store[key] = data
|
|
||||||
|
|
||||||
def get(self, key, max_age=None, memory_ttl=None, **kwargs):
|
|
||||||
return self.store.get(key)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def tracker():
|
|
||||||
cache = _Cache()
|
|
||||||
t = PluginHealthTracker(cache_manager=cache)
|
|
||||||
return t, cache
|
|
||||||
|
|
||||||
|
|
||||||
def test_steady_state_success_stops_writing(tracker):
|
|
||||||
"""The regression: 100 healthy cycles used to be 100 SD writes."""
|
|
||||||
t, cache = tracker
|
|
||||||
t.record_success("weather")
|
|
||||||
first = cache.writes
|
|
||||||
for _ in range(100):
|
|
||||||
t.record_success("weather")
|
|
||||||
assert cache.writes == first, (
|
|
||||||
f"{cache.writes - first} redundant writes across 100 healthy cycles"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_counters_are_still_accurate_in_memory(tracker):
|
|
||||||
"""Skipping the write must not skip the bookkeeping."""
|
|
||||||
t, _ = tracker
|
|
||||||
for _ in range(10):
|
|
||||||
t.record_success("weather")
|
|
||||||
state = t.get_health_state("weather")
|
|
||||||
assert state["total_successes"] == 10
|
|
||||||
assert state["last_success_time"] is not None
|
|
||||||
assert state["last_success_time"] <= time.time()
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_failure_is_written_immediately(tracker):
|
|
||||||
t, cache = tracker
|
|
||||||
t.record_success("weather")
|
|
||||||
before = cache.writes
|
|
||||||
t.record_failure("weather", RuntimeError("boom"))
|
|
||||||
assert cache.writes > before, "a failure must reach disk"
|
|
||||||
|
|
||||||
|
|
||||||
def test_recovery_after_failure_is_written(tracker):
|
|
||||||
"""consecutive_failures returning to 0 is durable state changing."""
|
|
||||||
t, cache = tracker
|
|
||||||
t.record_failure("weather", RuntimeError("boom"))
|
|
||||||
before = cache.writes
|
|
||||||
t.record_success("weather")
|
|
||||||
assert cache.writes > before, "recovery must reach disk"
|
|
||||||
assert t.get_health_state("weather")["consecutive_failures"] == 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_closing_circuit_is_written(tracker):
|
|
||||||
"""Success in half-open closes the circuit -- that must survive a restart."""
|
|
||||||
t, cache = tracker
|
|
||||||
state = t.get_health_state("weather")
|
|
||||||
state["circuit_state"] = CircuitState.HALF_OPEN.value
|
|
||||||
state["half_open_start_time"] = time.time()
|
|
||||||
before = cache.writes
|
|
||||||
t.record_success("weather")
|
|
||||||
assert cache.writes > before, "a circuit transition must reach disk"
|
|
||||||
assert t.get_health_state("weather")["circuit_state"] == CircuitState.CLOSED.value
|
|
||||||
|
|
||||||
|
|
||||||
def test_durable_state_survives_a_restart(tracker):
|
|
||||||
"""What is skipped must genuinely not matter to the breaker."""
|
|
||||||
t, cache = tracker
|
|
||||||
for _ in range(3):
|
|
||||||
t.record_failure("weather", RuntimeError("boom"))
|
|
||||||
for _ in range(50):
|
|
||||||
t.record_success("weather")
|
|
||||||
|
|
||||||
revived = PluginHealthTracker(cache_manager=cache)
|
|
||||||
state = revived.get_health_state("weather")
|
|
||||||
assert state["consecutive_failures"] == 0
|
|
||||||
assert state["circuit_state"] == CircuitState.CLOSED.value
|
|
||||||
Reference in New Issue
Block a user