mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-08-21 10:29:06 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a4290e8a28 |
Binary file not shown.
|
After Width: | Height: | Size: 76 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 128 KiB |
@@ -328,7 +328,11 @@ class ScrollHelper:
|
|||||||
elapsed_time = current_time - (self.scroll_start_time or current_time)
|
elapsed_time = current_time - (self.scroll_start_time or current_time)
|
||||||
# The image already includes display_width padding, so we only need total_scroll_width
|
# The image already includes display_width padding, so we only need total_scroll_width
|
||||||
required_total_distance = self.total_scroll_width
|
required_total_distance = self.total_scroll_width
|
||||||
self.logger.info(
|
# Progress telemetry, emitted every few seconds for the whole of
|
||||||
|
# every scroll. It says how far along a marquee is, which is what
|
||||||
|
# you turn debug on to watch and not something an operator needs
|
||||||
|
# in the journal on a device that scrolls all day.
|
||||||
|
self.logger.debug(
|
||||||
"Scroll progress: elapsed=%.2fs, target=%.2fs, total_scrolled=%.0f/%d px (%.1f%%)",
|
"Scroll progress: elapsed=%.2fs, target=%.2fs, total_scrolled=%.0f/%d px (%.1f%%)",
|
||||||
elapsed_time,
|
elapsed_time,
|
||||||
self.calculated_duration,
|
self.calculated_duration,
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import time
|
|||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
from typing import Dict, Optional, Any, Callable
|
from typing import Dict, Optional, Any, Callable
|
||||||
from dataclasses import dataclass, field, fields
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import psutil
|
import psutil
|
||||||
@@ -102,50 +102,6 @@ class PluginResourceMonitor:
|
|||||||
"psutil not available - resource monitoring will be limited to execution time only"
|
"psutil not available - resource monitoring will be limited to execution time only"
|
||||||
)
|
)
|
||||||
|
|
||||||
def _metrics_from_cache(self, plugin_id: str, cached: Any) -> "ResourceMetrics":
|
|
||||||
"""Build metrics from a cached record, ignoring anything unrecognised.
|
|
||||||
|
|
||||||
ResourceMetrics(**cached) raises TypeError on a single unexpected key,
|
|
||||||
and that exception escapes into plugin_manager, which reports it as
|
|
||||||
"plugin <id> operation failed". Every plugin fails, and the plugin
|
|
||||||
system never finishes initialising.
|
|
||||||
|
|
||||||
Seen on a live rig: every plugin failing with
|
|
||||||
|
|
||||||
ResourceMetrics.__init__() got an unexpected keyword argument
|
|
||||||
'consecutive_failures'
|
|
||||||
|
|
||||||
which is a plugin_health field, not a metrics one. How a health-shaped
|
|
||||||
record came to sit under a plugin_metrics key on that machine is not
|
|
||||||
established -- a restored backup that mixed two machines' caches is the
|
|
||||||
likeliest explanation -- but the loader should not be brittle enough for
|
|
||||||
it to matter. plugin_health already repairs its records field by field
|
|
||||||
rather than trusting whatever is on disk; this does the same.
|
|
||||||
|
|
||||||
Unknown keys are dropped and named once, so a genuine schema change is
|
|
||||||
visible in the log instead of silently discarded.
|
|
||||||
"""
|
|
||||||
if not isinstance(cached, dict):
|
|
||||||
self.logger.warning(
|
|
||||||
"Ignoring cached metrics for %s: expected a mapping, got %s",
|
|
||||||
plugin_id, type(cached).__name__)
|
|
||||||
return ResourceMetrics()
|
|
||||||
|
|
||||||
known = {f.name for f in fields(ResourceMetrics)}
|
|
||||||
unknown = sorted(set(cached) - known)
|
|
||||||
if unknown:
|
|
||||||
self.logger.warning(
|
|
||||||
"Dropping unrecognised field(s) from cached metrics for %s: %s",
|
|
||||||
plugin_id, ", ".join(unknown))
|
|
||||||
usable = {k: v for k, v in cached.items() if k in known}
|
|
||||||
try:
|
|
||||||
return ResourceMetrics(**usable)
|
|
||||||
except (TypeError, ValueError) as e:
|
|
||||||
self.logger.warning(
|
|
||||||
"Cached metrics for %s unusable (%s); starting fresh",
|
|
||||||
plugin_id, e)
|
|
||||||
return ResourceMetrics()
|
|
||||||
|
|
||||||
def _get_metrics_key(self, plugin_id: str) -> str:
|
def _get_metrics_key(self, plugin_id: str) -> str:
|
||||||
"""Get cache key for plugin metrics."""
|
"""Get cache key for plugin metrics."""
|
||||||
return f"plugin_metrics:{plugin_id}"
|
return f"plugin_metrics:{plugin_id}"
|
||||||
@@ -170,7 +126,7 @@ class PluginResourceMonitor:
|
|||||||
cache_key, max_age=None, memory_ttl=0 if force_reload else None
|
cache_key, max_age=None, memory_ttl=0 if force_reload else None
|
||||||
)
|
)
|
||||||
if cached:
|
if cached:
|
||||||
metrics = self._metrics_from_cache(plugin_id, cached)
|
metrics = ResourceMetrics(**cached)
|
||||||
else:
|
else:
|
||||||
metrics = ResourceMetrics()
|
metrics = ResourceMetrics()
|
||||||
self._metrics[plugin_id] = metrics
|
self._metrics[plugin_id] = metrics
|
||||||
|
|||||||
@@ -31,6 +31,14 @@ if TYPE_CHECKING:
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
#: A frame rate this close to target is not news; below it is.
|
||||||
|
_FPS_HEALTHY_FRACTION = 0.9
|
||||||
|
|
||||||
|
#: A healthy marquee still reports this often, so silence means stopped
|
||||||
|
#: rather than fine.
|
||||||
|
_FPS_HEARTBEAT_INTERVAL = 300.0
|
||||||
|
|
||||||
|
|
||||||
def _percentile(ordered: List[float], fraction: float) -> float:
|
def _percentile(ordered: List[float], fraction: float) -> float:
|
||||||
"""Nearest-rank percentile of an already-sorted list.
|
"""Nearest-rank percentile of an already-sorted list.
|
||||||
|
|
||||||
@@ -395,7 +403,9 @@ class VegasModeCoordinator:
|
|||||||
duration = self.render_pipeline.get_dynamic_duration()
|
duration = self.render_pipeline.get_dynamic_duration()
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
frame_count = 0
|
frame_count = 0
|
||||||
fps_log_interval = 5.0 # Log FPS every 5 seconds
|
fps_log_interval = 5.0 # Sample FPS every 5 seconds
|
||||||
|
last_fps_health_log = 0.0 # last INFO-level report
|
||||||
|
was_degraded = False # so the recovery is reported too
|
||||||
last_fps_log_time = start_time
|
last_fps_log_time = start_time
|
||||||
fps_frame_count = 0
|
fps_frame_count = 0
|
||||||
# A mean hides stutter completely. At 120fps a five-second window is
|
# A mean hides stutter completely. At 120fps a five-second window is
|
||||||
@@ -448,16 +458,36 @@ class VegasModeCoordinator:
|
|||||||
frame_count += 1
|
frame_count += 1
|
||||||
fps_frame_count += 1
|
fps_frame_count += 1
|
||||||
|
|
||||||
# Periodic FPS logging
|
# Periodic FPS logging. Reported at INFO only when the frame rate
|
||||||
|
# is actually worth an operator's attention -- a shortfall against
|
||||||
|
# target, or the recovery from one -- with a slow heartbeat so a
|
||||||
|
# healthy marquee still shows a pulse.
|
||||||
|
#
|
||||||
|
# Measured over two hours on a running rig: 1410 samples, 98.5%
|
||||||
|
# of them within 10% of target. The 1.5% that were not included a
|
||||||
|
# reading of 8.6fps against a target of 60 -- a real stall, and
|
||||||
|
# completely invisible inside 1389 lines reading "59.6".
|
||||||
current_time = time.time()
|
current_time = time.time()
|
||||||
if current_time - last_fps_log_time >= fps_log_interval:
|
if current_time - last_fps_log_time >= fps_log_interval:
|
||||||
fps = fps_frame_count / (current_time - last_fps_log_time)
|
fps = fps_frame_count / (current_time - last_fps_log_time)
|
||||||
p99 = _percentile(sorted(frame_times), 0.99)
|
p99 = _percentile(sorted(frame_times), 0.99)
|
||||||
|
target = self.vegas_config.target_fps
|
||||||
|
degraded = target > 0 and fps < target * _FPS_HEALTHY_FRACTION
|
||||||
|
due = current_time - last_fps_health_log >= _FPS_HEARTBEAT_INTERVAL
|
||||||
|
if degraded or was_degraded or due:
|
||||||
logger.info(
|
logger.info(
|
||||||
"Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms",
|
"Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms",
|
||||||
fps, self.vegas_config.target_fps, fps_frame_count,
|
fps, target, fps_frame_count,
|
||||||
p99 * 1000.0, frame_worst * 1000.0
|
p99 * 1000.0, frame_worst * 1000.0
|
||||||
)
|
)
|
||||||
|
last_fps_health_log = current_time
|
||||||
|
else:
|
||||||
|
logger.debug(
|
||||||
|
"Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms",
|
||||||
|
fps, target, fps_frame_count,
|
||||||
|
p99 * 1000.0, frame_worst * 1000.0
|
||||||
|
)
|
||||||
|
was_degraded = degraded
|
||||||
last_fps_log_time = current_time
|
last_fps_log_time = current_time
|
||||||
fps_frame_count = 0
|
fps_frame_count = 0
|
||||||
frame_worst = 0.0
|
frame_worst = 0.0
|
||||||
|
|||||||
@@ -1,100 +0,0 @@
|
|||||||
"""A malformed metrics cache entry must not take every plugin down with it.
|
|
||||||
|
|
||||||
`ResourceMetrics(**cached)` raises TypeError on a single unexpected key, and
|
|
||||||
that exception escapes into plugin_manager, which reports it per plugin as
|
|
||||||
"plugin <id> operation failed". Every plugin fails and the plugin system never
|
|
||||||
finishes initialising -- the health endpoint reports
|
|
||||||
`plugin_system: not_initialized` while the display itself keeps running.
|
|
||||||
|
|
||||||
Seen on a live rig, once per plugin, continuously:
|
|
||||||
|
|
||||||
ERROR - src.plugin_system.plugin_manager - plugin geochron operation failed:
|
|
||||||
ResourceMetrics.__init__() got an unexpected keyword argument
|
|
||||||
'consecutive_failures'
|
|
||||||
|
|
||||||
`consecutive_failures` belongs to plugin_health, not to metrics. How a
|
|
||||||
health-shaped record came to sit under a plugin_metrics key on that machine is
|
|
||||||
not established -- a restored backup that mixed two machines' caches is the
|
|
||||||
likeliest explanation, and the same rig had one restored onto it -- but a
|
|
||||||
loader that turns one bad cache entry into a total outage is the part worth
|
|
||||||
fixing. plugin_health already repairs its own records field by field rather
|
|
||||||
than trusting what is on disk.
|
|
||||||
"""
|
|
||||||
import logging
|
|
||||||
from dataclasses import fields
|
|
||||||
from unittest.mock import MagicMock
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from src.plugin_system.resource_monitor import PluginResourceMonitor, ResourceMetrics
|
|
||||||
|
|
||||||
|
|
||||||
class _Cache:
|
|
||||||
def __init__(self, payload=None):
|
|
||||||
self.payload = payload
|
|
||||||
|
|
||||||
def get(self, key, max_age=None, memory_ttl=None, **kwargs):
|
|
||||||
return self.payload
|
|
||||||
|
|
||||||
def set(self, key, data, ttl=None, **kwargs):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def _monitor(payload):
|
|
||||||
m = PluginResourceMonitor(cache_manager=_Cache(payload))
|
|
||||||
m.logger = logging.getLogger("test")
|
|
||||||
return m
|
|
||||||
|
|
||||||
|
|
||||||
#: What the rig actually had under the metrics key.
|
|
||||||
HEALTH_SHAPED = {
|
|
||||||
"consecutive_failures": 0, "circuit_state": "closed",
|
|
||||||
"circuit_opened_time": None, "half_open_start_time": None,
|
|
||||||
"last_error": None, "last_failure_time": None,
|
|
||||||
"last_success_time": 1_700_000_000.0, "total_failures": 0,
|
|
||||||
"total_successes": 42,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_health_record_under_the_metrics_key_does_not_raise():
|
|
||||||
"""The exact failure: it must degrade, not take the plugin system down."""
|
|
||||||
monitor = _monitor(HEALTH_SHAPED)
|
|
||||||
metrics = monitor.get_metrics(" plugin-a".strip())
|
|
||||||
assert isinstance(metrics, ResourceMetrics)
|
|
||||||
|
|
||||||
|
|
||||||
def test_recognised_fields_in_a_mixed_record_are_kept():
|
|
||||||
"""Dropping the record wholesale would lose real history unnecessarily."""
|
|
||||||
mixed = dict(HEALTH_SHAPED, call_count=7, memory_mb=12.5)
|
|
||||||
metrics = _monitor(mixed).get_metrics("plugin-b")
|
|
||||||
assert metrics.call_count == 7
|
|
||||||
assert metrics.memory_mb == 12.5
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_clean_record_still_loads_unchanged():
|
|
||||||
clean = {f.name: 3 for f in fields(ResourceMetrics)}
|
|
||||||
metrics = _monitor(clean).get_metrics("plugin-c")
|
|
||||||
for name in (f.name for f in fields(ResourceMetrics)):
|
|
||||||
assert getattr(metrics, name) == 3
|
|
||||||
|
|
||||||
|
|
||||||
def test_unknown_fields_are_named_in_the_log(caplog):
|
|
||||||
"""Silently discarding them would hide a real schema change."""
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
_monitor(HEALTH_SHAPED).get_metrics("plugin-d")
|
|
||||||
# getMessage(), not .message: the latter is only populated once a handler
|
|
||||||
# formats the record, so the obvious spelling silently never matches.
|
|
||||||
assert any("consecutive_failures" in r.getMessage() for r in caplog.records), \
|
|
||||||
caplog.text
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("payload", ["a string", 42, ["a", "list"]])
|
|
||||||
def test_a_non_mapping_cache_entry_does_not_raise(payload):
|
|
||||||
metrics = _monitor(payload).get_metrics("plugin-e")
|
|
||||||
assert isinstance(metrics, ResourceMetrics)
|
|
||||||
|
|
||||||
|
|
||||||
def test_values_of_the_wrong_type_do_not_raise():
|
|
||||||
"""A dataclass will accept these, but a later float() on them would not."""
|
|
||||||
metrics = _monitor({"call_count": "not a number"}).get_metrics("plugin-f")
|
|
||||||
assert isinstance(metrics, ResourceMetrics)
|
|
||||||
Reference in New Issue
Block a user