mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-08-21 02:19:07 +00:00
perf(plugins): stop rewriting a plugin's metrics file on every call
Plugin metrics were persisted to the cache inside monitor_call, so every call by every plugin rewrote a small JSON file. Measured on a running rig: one plugin's plugin_metrics file changed nine times a minute, with fourteen such files active. Each is around 350 bytes, which on ext4 costs a 4KB block plus a journal entry, so the cost is dominated by the write itself rather than the payload. Cache writes accounted for essentially all of that device's 2.4 MB/min of SD traffic, on a card that wears out and has already failed twice on the other rig. Metrics cannot be de-duplicated the way health state can, because call_count changes on every call and the timings usually do too. So they are rate-limited instead: at most one write per plugin per 30 seconds. The in-memory copy stays authoritative and exact -- a plugin's call_count is still precise the instant after it runs. Only the cross-process snapshot the web UI reads is delayed, and telemetry up to half a minute old is still a fair description of a long-running plugin. reset_metrics clears the throttle timestamp, so a reset is not left showing a deleted key for the rest of the interval. Extrapolating the sampled rate, this takes metric writes from roughly 126 a minute to 28. Health persistence, the other half of the churn, is handled separately in #475. Verified by reverting the throttle: the churn test then reports 50 writes for 50 calls. 88 tests pass across resource monitor, plugin system and web API. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01STMbQE4YctTacQXfbYqKuW
This commit is contained in:
co-authored by
Claude Opus 5
parent
cf0a551f7b
commit
0fd2bfae99
@@ -127,3 +127,50 @@ class TestForceReload:
|
||||
fresh = mon.get_metrics_summary("p", force_reload=True)
|
||||
assert fresh["call_count"] == 7
|
||||
assert any(c.kwargs.get("memory_ttl") == 0 for c in cache.get.call_args_list)
|
||||
|
||||
|
||||
class TestMetricsPersistenceChurn:
|
||||
"""Metrics are telemetry; writing them on every call wore the SD card.
|
||||
|
||||
Each write is a ~350-byte file, which on ext4 costs a 4KB block plus a
|
||||
journal entry. At roughly nine calls a minute per plugin across fourteen
|
||||
plugins it dominated the device's write volume.
|
||||
"""
|
||||
|
||||
def test_repeated_calls_persist_once_per_interval(self):
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
for _ in range(50):
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 1, (
|
||||
f"50 calls produced {len(writes)} metric writes; expected 1")
|
||||
|
||||
def test_the_interval_elapsing_allows_the_next_write(self, monkeypatch):
|
||||
import src.plugin_system.resource_monitor as rm
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
mon.monitor_call("p", lambda: None)
|
||||
# pretend the interval has passed
|
||||
mon._metrics_persisted_at["p"] -= rm._METRICS_PERSIST_INTERVAL + 1
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 2
|
||||
|
||||
def test_in_memory_metrics_stay_exact_while_writes_are_skipped(self):
|
||||
mon = PluginResourceMonitor(_cache(), enable_monitoring=False)
|
||||
for _ in range(20):
|
||||
mon.monitor_call("p", lambda: None)
|
||||
assert mon.get_metrics("p").call_count == 20
|
||||
|
||||
def test_reset_lets_the_next_call_persist_immediately(self):
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
mon.monitor_call("p", lambda: None)
|
||||
mon.reset_metrics("p")
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 2, "reset should clear the throttle timestamp"
|
||||
|
||||
Reference in New Issue
Block a user