mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-08-21 10:29:06 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6642a78d0e |
@@ -49,20 +49,6 @@ class ResourceMetrics:
|
||||
self.total_execution_time = self.total_execution_time / self.call_count
|
||||
|
||||
|
||||
#: How often a plugin's metrics are written to the cache, in seconds.
|
||||
#:
|
||||
#: Persisting on every call meant a small file rewritten roughly nine times a
|
||||
#: minute per plugin. On a rig with fourteen active plugins that was ~126
|
||||
#: writes a minute for metrics alone, and since each ~350-byte file costs a
|
||||
#: 4KB block plus an ext4 journal entry, it dominated the device's write
|
||||
#: volume -- on an SD card, which wears out.
|
||||
#:
|
||||
#: The in-memory copy stays authoritative and exact; only the cross-process
|
||||
#: snapshot the web UI reads is delayed, and telemetry up to half a minute old
|
||||
#: is still a fair description of a long-running plugin.
|
||||
_METRICS_PERSIST_INTERVAL = 30.0
|
||||
|
||||
|
||||
class PluginResourceMonitor:
|
||||
"""
|
||||
Monitors resource usage for plugins.
|
||||
@@ -89,10 +75,6 @@ class PluginResourceMonitor:
|
||||
# Resource metrics per plugin
|
||||
self._metrics: Dict[str, ResourceMetrics] = {}
|
||||
self._limits: Dict[str, ResourceLimits] = {}
|
||||
# When each plugin's metrics last reached the cache. Metrics change on
|
||||
# every call, so they cannot be de-duplicated the way health state can;
|
||||
# they are rate-limited instead. See _METRICS_PERSIST_INTERVAL.
|
||||
self._metrics_persisted_at: Dict[str, float] = {}
|
||||
|
||||
# Thread-local storage for execution tracking
|
||||
self._local = threading.local()
|
||||
@@ -250,8 +232,18 @@ class PluginResourceMonitor:
|
||||
# CPU is harder to measure per-call, so we track it separately
|
||||
metrics.cpu_percent = self._get_process_cpu_percent()
|
||||
|
||||
# Persist metrics, at most once per interval per plugin.
|
||||
self._persist_metrics(plugin_id, metrics)
|
||||
# Persist metrics
|
||||
cache_key = self._get_metrics_key(plugin_id)
|
||||
self.cache_manager.set(cache_key, {
|
||||
'memory_mb': metrics.memory_mb,
|
||||
'cpu_percent': metrics.cpu_percent,
|
||||
'execution_time': metrics.execution_time,
|
||||
'call_count': metrics.call_count,
|
||||
'total_execution_time': metrics.total_execution_time,
|
||||
'max_execution_time': metrics.max_execution_time,
|
||||
'min_execution_time': metrics.min_execution_time if metrics.min_execution_time != float('inf') else 0.0,
|
||||
'last_update_time': metrics.last_update_time
|
||||
})
|
||||
|
||||
# Check limits
|
||||
if limits:
|
||||
@@ -371,37 +363,6 @@ class PluginResourceMonitor:
|
||||
summaries[plugin_id] = self.get_metrics_summary(plugin_id)
|
||||
return summaries
|
||||
|
||||
def _persist_metrics(self, plugin_id: str, metrics: ResourceMetrics,
|
||||
force: bool = False) -> None:
|
||||
"""Write a plugin's metrics to the cache, at most once per interval.
|
||||
|
||||
Caller must hold ``self._lock``.
|
||||
"""
|
||||
# Monotonic, not wall clock: these devices have no RTC, so the clock
|
||||
# jumps by however far off boot-time was the moment NTP first syncs.
|
||||
# A forward jump would allow an early write, a backward one would
|
||||
# stall the snapshot well past the interval.
|
||||
now = time.monotonic()
|
||||
if not force and now - self._metrics_persisted_at.get(plugin_id, 0.0) \
|
||||
< _METRICS_PERSIST_INTERVAL:
|
||||
return
|
||||
cache_key = self._get_metrics_key(plugin_id)
|
||||
self.cache_manager.set(cache_key, {
|
||||
'memory_mb': metrics.memory_mb,
|
||||
'cpu_percent': metrics.cpu_percent,
|
||||
'execution_time': metrics.execution_time,
|
||||
'call_count': metrics.call_count,
|
||||
'total_execution_time': metrics.total_execution_time,
|
||||
'max_execution_time': metrics.max_execution_time,
|
||||
'min_execution_time': (metrics.min_execution_time
|
||||
if metrics.min_execution_time != float('inf')
|
||||
else 0.0),
|
||||
'last_update_time': metrics.last_update_time,
|
||||
})
|
||||
# Only after the write lands. Marking it first would mean a failed
|
||||
# set() bought the next interval's silence without leaving a snapshot.
|
||||
self._metrics_persisted_at[plugin_id] = now
|
||||
|
||||
def reset_metrics(self, plugin_id: str) -> None:
|
||||
"""Reset metrics for a plugin."""
|
||||
with self._lock:
|
||||
@@ -409,7 +370,4 @@ class PluginResourceMonitor:
|
||||
self._metrics[plugin_id] = ResourceMetrics()
|
||||
cache_key = self._get_metrics_key(plugin_id)
|
||||
self.cache_manager.delete(cache_key)
|
||||
# Let the next call persist immediately rather than leaving the
|
||||
# deleted key absent for the rest of the interval.
|
||||
self._metrics_persisted_at.pop(plugin_id, None)
|
||||
|
||||
|
||||
@@ -127,67 +127,3 @@ class TestForceReload:
|
||||
fresh = mon.get_metrics_summary("p", force_reload=True)
|
||||
assert fresh["call_count"] == 7
|
||||
assert any(c.kwargs.get("memory_ttl") == 0 for c in cache.get.call_args_list)
|
||||
|
||||
|
||||
class TestMetricsPersistenceChurn:
|
||||
"""Metrics are telemetry; writing them on every call wore the SD card.
|
||||
|
||||
Each write is a ~350-byte file, which on ext4 costs a 4KB block plus a
|
||||
journal entry. At roughly nine calls a minute per plugin across fourteen
|
||||
plugins it dominated the device's write volume.
|
||||
"""
|
||||
|
||||
def test_repeated_calls_persist_once_per_interval(self):
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
for _ in range(50):
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 1, (
|
||||
f"50 calls produced {len(writes)} metric writes; expected 1")
|
||||
|
||||
def test_the_interval_elapsing_allows_the_next_write(self, monkeypatch):
|
||||
import src.plugin_system.resource_monitor as rm
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
mon.monitor_call("p", lambda: None)
|
||||
# pretend the interval has passed
|
||||
mon._metrics_persisted_at["p"] -= rm._METRICS_PERSIST_INTERVAL + 1
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 2
|
||||
|
||||
def test_in_memory_metrics_stay_exact_while_writes_are_skipped(self):
|
||||
mon = PluginResourceMonitor(_cache(), enable_monitoring=False)
|
||||
for _ in range(20):
|
||||
mon.monitor_call("p", lambda: None)
|
||||
assert mon.get_metrics("p").call_count == 20
|
||||
|
||||
def test_reset_lets_the_next_call_persist_immediately(self):
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
mon.monitor_call("p", lambda: None)
|
||||
mon.reset_metrics("p")
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 2, "reset should clear the throttle timestamp"
|
||||
|
||||
def test_a_failed_write_does_not_buy_the_next_interval_of_silence(self):
|
||||
"""A set() that raises must not count as having persisted.
|
||||
|
||||
Marking the timestamp before the write would leave no snapshot in the
|
||||
cache and still suppress the next 30 seconds of attempts.
|
||||
"""
|
||||
cache = _cache()
|
||||
cache.set.side_effect = [OSError("disk full"), None]
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
with pytest.raises(OSError):
|
||||
mon.monitor_call("p", lambda: None)
|
||||
# the very next call must try again rather than skip the interval
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 2, "a failed write should be retried, not skipped"
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""A pull that changed nothing on the running system is not an applied update.
|
||||
|
||||
git_pull replaces files on disk and restarts nothing -- there is no systemctl
|
||||
call anywhere in the handler. The display and web services keep running the
|
||||
code they loaded at boot, so the user is told "Code updated successfully" and
|
||||
sees no change until they happen to reboot. The response now says whether a
|
||||
restart is owed, and the UI raises the existing restart-pending banner.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from web_interface.blueprints import api_v3 as mod # noqa: E402
|
||||
from web_interface.blueprints.api_v3 import api_v3 # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
app.register_blueprint(api_v3, url_prefix='/api/v3')
|
||||
# The handler consults these after a successful pull; None is the
|
||||
# "not wired up" case it already guards for.
|
||||
api_v3.plugin_store_manager = None
|
||||
api_v3.config_manager = None
|
||||
return app.test_client()
|
||||
|
||||
|
||||
def _git(heads, pull_rc=0, pull_out='Updating a1b2c3..d4e5f6\n'):
|
||||
"""Fake git. `heads` are the successive answers to rev-parse HEAD."""
|
||||
seq = list(heads)
|
||||
|
||||
def run(args, **kwargs):
|
||||
def ok(stdout='', rc=0, b=False):
|
||||
return subprocess.CompletedProcess(
|
||||
args, rc, stdout=(stdout.encode() if b else stdout),
|
||||
stderr=(b'' if b else ''))
|
||||
if args[:2] == ['git', 'rev-parse'] and args[-1] == 'HEAD':
|
||||
return ok(seq.pop(0) + '\n' if seq else 'deadbeef\n')
|
||||
if 'symbolic-full-name' in args or '@{u}' in args:
|
||||
return ok('origin/main\n')
|
||||
if args[:2] == ['git', 'status']:
|
||||
return ok('')
|
||||
if args[:2] == ['git', 'diff']:
|
||||
return ok('')
|
||||
if args[:2] == ['git', 'pull']:
|
||||
return ok(pull_out, pull_rc)
|
||||
return ok('')
|
||||
return run
|
||||
|
||||
|
||||
def _pull(client):
|
||||
return client.post('/api/v3/system/action',
|
||||
json={'action': 'git_pull'}).get_json()
|
||||
|
||||
|
||||
class TestRestartIsRequestedWhenCodeChanged:
|
||||
def test_a_pull_that_moved_head_asks_for_a_restart(self, client):
|
||||
with patch.object(mod.subprocess, 'run', _git(['aaa111', 'bbb222'])):
|
||||
data = _pull(client)
|
||||
assert data['status'] == 'success'
|
||||
assert data['restart_required'] is True, (
|
||||
"new code on disk, services still running the old code, and "
|
||||
"nothing told the user to restart")
|
||||
|
||||
def test_already_up_to_date_does_not(self, client):
|
||||
with patch.object(mod.subprocess, 'run',
|
||||
_git(['aaa111', 'aaa111'], pull_out='Already up to date.\n')):
|
||||
data = _pull(client)
|
||||
assert data['status'] == 'success'
|
||||
assert data['restart_required'] is False, (
|
||||
"prompting after a no-op update trains users to ignore the prompt")
|
||||
|
||||
def test_a_failed_pull_does_not(self, client):
|
||||
with patch.object(mod.subprocess, 'run', _git(['aaa111'], pull_rc=1)):
|
||||
data = _pull(client)
|
||||
assert data['status'] == 'error'
|
||||
assert data['restart_required'] is False
|
||||
@@ -1996,6 +1996,11 @@ def execute_system_action():
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("git rev-parse timed out before pull")
|
||||
|
||||
# Whether the pull actually brought new code in. "Already up to
|
||||
# date" is a success too, and prompting for a restart then would
|
||||
# train users to ignore the prompt.
|
||||
code_changed = False
|
||||
|
||||
# Perform the git pull. Branches without an upstream were given
|
||||
# an explicit "origin <branch>" above so the update still works.
|
||||
result = subprocess.run(
|
||||
@@ -2039,6 +2044,7 @@ def execute_system_action():
|
||||
capture_output=True, text=True, timeout=10, cwd=project_dir)
|
||||
new_head = _post.stdout.strip() if _post.returncode == 0 else None
|
||||
if old_head and new_head and old_head != new_head:
|
||||
code_changed = True
|
||||
diff = subprocess.run(
|
||||
['git', 'diff', '--name-only', f'{old_head}..{new_head}'],
|
||||
capture_output=True, text=True, timeout=15, cwd=project_dir)
|
||||
@@ -2098,9 +2104,14 @@ def execute_system_action():
|
||||
if ln.strip()), '')
|
||||
pull_message = f"Update failed: {detail}" if detail else "Update failed; check logs for details"
|
||||
|
||||
# Nothing here restarts anything: the pull replaces files on
|
||||
# disk while the display and web services keep running the code
|
||||
# they loaded at boot. Without this the user is told the update
|
||||
# succeeded and sees no change until they happen to reboot.
|
||||
return jsonify({
|
||||
'status': 'success' if result.returncode == 0 else 'error',
|
||||
'message': pull_message,
|
||||
'restart_required': bool(result.returncode == 0 and code_changed),
|
||||
})
|
||||
elif action == 'checkout_branch':
|
||||
# Switch branches from the Tools tab. Needed because a checkout
|
||||
|
||||
@@ -116,14 +116,25 @@ document.body.addEventListener('htmx:afterRequest', function(event) {
|
||||
// ===== Restart-pending banner =====
|
||||
// Shown after restart-requiring saves; persists across tab switches (and
|
||||
// reloads, via sessionStorage) until the display restarts or it's dismissed.
|
||||
window.showRestartPending = function() {
|
||||
try { sessionStorage.setItem('ledmatrix-restart-pending', '1'); } catch { /* private browsing */ }
|
||||
window.showRestartPending = function(message) {
|
||||
try {
|
||||
sessionStorage.setItem('ledmatrix-restart-pending', '1');
|
||||
// Persisted alongside the flag: a code update and a config save want
|
||||
// different wording, and the banner outlives the page that raised it.
|
||||
if (message) sessionStorage.setItem('ledmatrix-restart-pending-text', message);
|
||||
else sessionStorage.removeItem('ledmatrix-restart-pending-text');
|
||||
} catch { /* private browsing */ }
|
||||
const banner = document.getElementById('restart-pending-banner');
|
||||
const text = document.getElementById('restart-pending-text');
|
||||
if (text && message) text.textContent = message;
|
||||
if (banner) banner.style.display = 'block';
|
||||
};
|
||||
|
||||
window.dismissRestartPending = function() {
|
||||
try { sessionStorage.removeItem('ledmatrix-restart-pending'); } catch { /* no-op */ }
|
||||
try {
|
||||
sessionStorage.removeItem('ledmatrix-restart-pending');
|
||||
sessionStorage.removeItem('ledmatrix-restart-pending-text');
|
||||
} catch { /* no-op */ }
|
||||
const banner = document.getElementById('restart-pending-banner');
|
||||
if (banner) banner.style.display = 'none';
|
||||
};
|
||||
@@ -151,6 +162,9 @@ document.addEventListener('DOMContentLoaded', function() {
|
||||
try {
|
||||
if (sessionStorage.getItem('ledmatrix-restart-pending') === '1') {
|
||||
const banner = document.getElementById('restart-pending-banner');
|
||||
const saved = sessionStorage.getItem('ledmatrix-restart-pending-text');
|
||||
const text = document.getElementById('restart-pending-text');
|
||||
if (text && saved) text.textContent = saved;
|
||||
if (banner) banner.style.display = 'block';
|
||||
}
|
||||
} catch { /* no-op */ }
|
||||
|
||||
@@ -413,7 +413,8 @@
|
||||
<div class="flex items-center justify-between">
|
||||
<div class="flex items-center space-x-3">
|
||||
<i class="fas fa-rotate text-lg"></i>
|
||||
<span class="text-sm font-medium" aria-live="polite">
|
||||
<span class="text-sm font-medium" aria-live="polite"
|
||||
id="restart-pending-text">
|
||||
Configuration saved — restart the display to apply the changes
|
||||
</span>
|
||||
</div>
|
||||
@@ -1146,6 +1147,13 @@
|
||||
if (data.status === 'success') {
|
||||
document.getElementById('update-banner').style.display = 'none';
|
||||
try { sessionStorage.removeItem('update-sha-dismissed'); } catch(e) {}
|
||||
// The pull replaced files on disk; the running services still
|
||||
// hold the code they loaded at boot. Ask for the restart that
|
||||
// makes the update actually take effect.
|
||||
if (data.restart_required && typeof window.showRestartPending === 'function') {
|
||||
window.showRestartPending(
|
||||
'Update installed \u2014 restart the display to run the new code');
|
||||
}
|
||||
}
|
||||
if (typeof showNotification === 'function') {
|
||||
showNotification(data.message || 'Update complete', data.status || 'success');
|
||||
|
||||
Reference in New Issue
Block a user