mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-08-21 02:19:07 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6642a78d0e |
@@ -178,20 +178,10 @@ class PluginHealthTracker:
|
|||||||
)
|
)
|
||||||
return self._health_state[plugin_id]
|
return self._health_state[plugin_id]
|
||||||
|
|
||||||
# Fields the circuit breaker is rebuilt from after a restart. Everything
|
|
||||||
# else in a health record is reporting, read only for display.
|
|
||||||
_DURABLE_FIELDS = ('consecutive_failures', 'circuit_state',
|
|
||||||
'circuit_opened_time', 'half_open_start_time')
|
|
||||||
|
|
||||||
def _durable(self, state: Dict[str, Any]) -> tuple:
|
|
||||||
"""The part of a health record whose loss would change behaviour."""
|
|
||||||
return tuple(state.get(field) for field in self._DURABLE_FIELDS)
|
|
||||||
|
|
||||||
def record_success(self, plugin_id: str) -> None:
|
def record_success(self, plugin_id: str) -> None:
|
||||||
"""Record a successful plugin execution."""
|
"""Record a successful plugin execution."""
|
||||||
state = self.get_health_state(plugin_id)
|
state = self.get_health_state(plugin_id)
|
||||||
current_time = time.time()
|
current_time = time.time()
|
||||||
durable_before = self._durable(state)
|
|
||||||
|
|
||||||
# Reset consecutive failures
|
# Reset consecutive failures
|
||||||
state['consecutive_failures'] = 0
|
state['consecutive_failures'] = 0
|
||||||
@@ -209,18 +199,7 @@ class PluginHealthTracker:
|
|||||||
state['circuit_state'] = CircuitState.CLOSED.value
|
state['circuit_state'] = CircuitState.CLOSED.value
|
||||||
state['circuit_opened_time'] = None
|
state['circuit_opened_time'] = None
|
||||||
|
|
||||||
# A healthy plugin reports success every cycle, and in that steady state
|
self._save_health_state(plugin_id, state)
|
||||||
# the only fields changed above are a counter and a timestamp that
|
|
||||||
# nothing reads back after a restart. Persisting them anyway rewrites a
|
|
||||||
# small file per plugin per cycle: on a rig running 24 plugins, a
|
|
||||||
# five-minute sample measured 22 rewrites, about 4.4 a minute or 6,300 a
|
|
||||||
# day. Those land on an SD card, where the cost is an erase-block cycle
|
|
||||||
# rather than the 400 bytes involved, and where wear is what eventually
|
|
||||||
# kills the card.
|
|
||||||
# In-memory state is still updated every time, so the health API and web
|
|
||||||
# UI show exactly what they did before; only the write is skipped.
|
|
||||||
if self._durable(state) != durable_before:
|
|
||||||
self._save_health_state(plugin_id, state)
|
|
||||||
|
|
||||||
def record_failure(self, plugin_id: str, error: Optional[Exception] = None) -> None:
|
def record_failure(self, plugin_id: str, error: Optional[Exception] = None) -> None:
|
||||||
"""Record a failed plugin execution."""
|
"""Record a failed plugin execution."""
|
||||||
|
|||||||
@@ -1,110 +0,0 @@
|
|||||||
"""A healthy plugin must not rewrite its health record every cycle.
|
|
||||||
|
|
||||||
Every successful plugin update called record_success(), which persisted the
|
|
||||||
record unconditionally. In steady state the only fields that had changed were
|
|
||||||
total_successes and last_success_time -- a counter and a timestamp that
|
|
||||||
health_monitor reads for display and that nothing reads back after a restart.
|
|
||||||
|
|
||||||
Measured on a rig running 24 plugins: about 17 health-file rewrites a minute,
|
|
||||||
roughly 25,000 a day. Each is ~400 bytes, but they land on an SD card where
|
|
||||||
the unit of cost is an erase-block cycle, not the byte count, and where wear is
|
|
||||||
what eventually kills the card.
|
|
||||||
|
|
||||||
The circuit breaker still needs its own state to survive a restart, so the
|
|
||||||
write is kept for exactly the fields it is rebuilt from -- and a failure, a
|
|
||||||
circuit opening, or a recovery must still be written the moment it happens.
|
|
||||||
"""
|
|
||||||
import time
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from src.plugin_system.plugin_health import PluginHealthTracker, CircuitState
|
|
||||||
|
|
||||||
|
|
||||||
class _Cache:
|
|
||||||
"""Counts writes; serves back whatever was last written."""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.store = {}
|
|
||||||
self.writes = 0
|
|
||||||
|
|
||||||
def set(self, key, data, ttl=None, **kwargs):
|
|
||||||
self.writes += 1
|
|
||||||
self.store[key] = data
|
|
||||||
|
|
||||||
def get(self, key, max_age=None, memory_ttl=None, **kwargs):
|
|
||||||
return self.store.get(key)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def tracker():
|
|
||||||
cache = _Cache()
|
|
||||||
t = PluginHealthTracker(cache_manager=cache)
|
|
||||||
return t, cache
|
|
||||||
|
|
||||||
|
|
||||||
def test_steady_state_success_stops_writing(tracker):
|
|
||||||
"""The regression: 100 healthy cycles used to be 100 SD writes."""
|
|
||||||
t, cache = tracker
|
|
||||||
t.record_success("weather")
|
|
||||||
first = cache.writes
|
|
||||||
for _ in range(100):
|
|
||||||
t.record_success("weather")
|
|
||||||
assert cache.writes == first, (
|
|
||||||
f"{cache.writes - first} redundant writes across 100 healthy cycles"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_the_counters_are_still_accurate_in_memory(tracker):
|
|
||||||
"""Skipping the write must not skip the bookkeeping."""
|
|
||||||
t, _ = tracker
|
|
||||||
for _ in range(10):
|
|
||||||
t.record_success("weather")
|
|
||||||
state = t.get_health_state("weather")
|
|
||||||
assert state["total_successes"] == 10
|
|
||||||
assert state["last_success_time"] is not None
|
|
||||||
assert state["last_success_time"] <= time.time()
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_failure_is_written_immediately(tracker):
|
|
||||||
t, cache = tracker
|
|
||||||
t.record_success("weather")
|
|
||||||
before = cache.writes
|
|
||||||
t.record_failure("weather", RuntimeError("boom"))
|
|
||||||
assert cache.writes > before, "a failure must reach disk"
|
|
||||||
|
|
||||||
|
|
||||||
def test_recovery_after_failure_is_written(tracker):
|
|
||||||
"""consecutive_failures returning to 0 is durable state changing."""
|
|
||||||
t, cache = tracker
|
|
||||||
t.record_failure("weather", RuntimeError("boom"))
|
|
||||||
before = cache.writes
|
|
||||||
t.record_success("weather")
|
|
||||||
assert cache.writes > before, "recovery must reach disk"
|
|
||||||
assert t.get_health_state("weather")["consecutive_failures"] == 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_a_closing_circuit_is_written(tracker):
|
|
||||||
"""Success in half-open closes the circuit -- that must survive a restart."""
|
|
||||||
t, cache = tracker
|
|
||||||
state = t.get_health_state("weather")
|
|
||||||
state["circuit_state"] = CircuitState.HALF_OPEN.value
|
|
||||||
state["half_open_start_time"] = time.time()
|
|
||||||
before = cache.writes
|
|
||||||
t.record_success("weather")
|
|
||||||
assert cache.writes > before, "a circuit transition must reach disk"
|
|
||||||
assert t.get_health_state("weather")["circuit_state"] == CircuitState.CLOSED.value
|
|
||||||
|
|
||||||
|
|
||||||
def test_durable_state_survives_a_restart(tracker):
|
|
||||||
"""What is skipped must genuinely not matter to the breaker."""
|
|
||||||
t, cache = tracker
|
|
||||||
for _ in range(3):
|
|
||||||
t.record_failure("weather", RuntimeError("boom"))
|
|
||||||
for _ in range(50):
|
|
||||||
t.record_success("weather")
|
|
||||||
|
|
||||||
revived = PluginHealthTracker(cache_manager=cache)
|
|
||||||
state = revived.get_health_state("weather")
|
|
||||||
assert state["consecutive_failures"] == 0
|
|
||||||
assert state["circuit_state"] == CircuitState.CLOSED.value
|
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
"""A pull that changed nothing on the running system is not an applied update.
|
||||||
|
|
||||||
|
git_pull replaces files on disk and restarts nothing -- there is no systemctl
|
||||||
|
call anywhere in the handler. The display and web services keep running the
|
||||||
|
code they loaded at boot, so the user is told "Code updated successfully" and
|
||||||
|
sees no change until they happen to reboot. The response now says whether a
|
||||||
|
restart is owed, and the UI raises the existing restart-pending banner.
|
||||||
|
"""
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from flask import Flask
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||||
|
|
||||||
|
from web_interface.blueprints import api_v3 as mod # noqa: E402
|
||||||
|
from web_interface.blueprints.api_v3 import api_v3 # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def client():
|
||||||
|
app = Flask(__name__)
|
||||||
|
app.config['TESTING'] = True
|
||||||
|
app.register_blueprint(api_v3, url_prefix='/api/v3')
|
||||||
|
# The handler consults these after a successful pull; None is the
|
||||||
|
# "not wired up" case it already guards for.
|
||||||
|
api_v3.plugin_store_manager = None
|
||||||
|
api_v3.config_manager = None
|
||||||
|
return app.test_client()
|
||||||
|
|
||||||
|
|
||||||
|
def _git(heads, pull_rc=0, pull_out='Updating a1b2c3..d4e5f6\n'):
|
||||||
|
"""Fake git. `heads` are the successive answers to rev-parse HEAD."""
|
||||||
|
seq = list(heads)
|
||||||
|
|
||||||
|
def run(args, **kwargs):
|
||||||
|
def ok(stdout='', rc=0, b=False):
|
||||||
|
return subprocess.CompletedProcess(
|
||||||
|
args, rc, stdout=(stdout.encode() if b else stdout),
|
||||||
|
stderr=(b'' if b else ''))
|
||||||
|
if args[:2] == ['git', 'rev-parse'] and args[-1] == 'HEAD':
|
||||||
|
return ok(seq.pop(0) + '\n' if seq else 'deadbeef\n')
|
||||||
|
if 'symbolic-full-name' in args or '@{u}' in args:
|
||||||
|
return ok('origin/main\n')
|
||||||
|
if args[:2] == ['git', 'status']:
|
||||||
|
return ok('')
|
||||||
|
if args[:2] == ['git', 'diff']:
|
||||||
|
return ok('')
|
||||||
|
if args[:2] == ['git', 'pull']:
|
||||||
|
return ok(pull_out, pull_rc)
|
||||||
|
return ok('')
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
def _pull(client):
|
||||||
|
return client.post('/api/v3/system/action',
|
||||||
|
json={'action': 'git_pull'}).get_json()
|
||||||
|
|
||||||
|
|
||||||
|
class TestRestartIsRequestedWhenCodeChanged:
|
||||||
|
def test_a_pull_that_moved_head_asks_for_a_restart(self, client):
|
||||||
|
with patch.object(mod.subprocess, 'run', _git(['aaa111', 'bbb222'])):
|
||||||
|
data = _pull(client)
|
||||||
|
assert data['status'] == 'success'
|
||||||
|
assert data['restart_required'] is True, (
|
||||||
|
"new code on disk, services still running the old code, and "
|
||||||
|
"nothing told the user to restart")
|
||||||
|
|
||||||
|
def test_already_up_to_date_does_not(self, client):
|
||||||
|
with patch.object(mod.subprocess, 'run',
|
||||||
|
_git(['aaa111', 'aaa111'], pull_out='Already up to date.\n')):
|
||||||
|
data = _pull(client)
|
||||||
|
assert data['status'] == 'success'
|
||||||
|
assert data['restart_required'] is False, (
|
||||||
|
"prompting after a no-op update trains users to ignore the prompt")
|
||||||
|
|
||||||
|
def test_a_failed_pull_does_not(self, client):
|
||||||
|
with patch.object(mod.subprocess, 'run', _git(['aaa111'], pull_rc=1)):
|
||||||
|
data = _pull(client)
|
||||||
|
assert data['status'] == 'error'
|
||||||
|
assert data['restart_required'] is False
|
||||||
@@ -1996,6 +1996,11 @@ def execute_system_action():
|
|||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
logger.warning("git rev-parse timed out before pull")
|
logger.warning("git rev-parse timed out before pull")
|
||||||
|
|
||||||
|
# Whether the pull actually brought new code in. "Already up to
|
||||||
|
# date" is a success too, and prompting for a restart then would
|
||||||
|
# train users to ignore the prompt.
|
||||||
|
code_changed = False
|
||||||
|
|
||||||
# Perform the git pull. Branches without an upstream were given
|
# Perform the git pull. Branches without an upstream were given
|
||||||
# an explicit "origin <branch>" above so the update still works.
|
# an explicit "origin <branch>" above so the update still works.
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
@@ -2039,6 +2044,7 @@ def execute_system_action():
|
|||||||
capture_output=True, text=True, timeout=10, cwd=project_dir)
|
capture_output=True, text=True, timeout=10, cwd=project_dir)
|
||||||
new_head = _post.stdout.strip() if _post.returncode == 0 else None
|
new_head = _post.stdout.strip() if _post.returncode == 0 else None
|
||||||
if old_head and new_head and old_head != new_head:
|
if old_head and new_head and old_head != new_head:
|
||||||
|
code_changed = True
|
||||||
diff = subprocess.run(
|
diff = subprocess.run(
|
||||||
['git', 'diff', '--name-only', f'{old_head}..{new_head}'],
|
['git', 'diff', '--name-only', f'{old_head}..{new_head}'],
|
||||||
capture_output=True, text=True, timeout=15, cwd=project_dir)
|
capture_output=True, text=True, timeout=15, cwd=project_dir)
|
||||||
@@ -2098,9 +2104,14 @@ def execute_system_action():
|
|||||||
if ln.strip()), '')
|
if ln.strip()), '')
|
||||||
pull_message = f"Update failed: {detail}" if detail else "Update failed; check logs for details"
|
pull_message = f"Update failed: {detail}" if detail else "Update failed; check logs for details"
|
||||||
|
|
||||||
|
# Nothing here restarts anything: the pull replaces files on
|
||||||
|
# disk while the display and web services keep running the code
|
||||||
|
# they loaded at boot. Without this the user is told the update
|
||||||
|
# succeeded and sees no change until they happen to reboot.
|
||||||
return jsonify({
|
return jsonify({
|
||||||
'status': 'success' if result.returncode == 0 else 'error',
|
'status': 'success' if result.returncode == 0 else 'error',
|
||||||
'message': pull_message,
|
'message': pull_message,
|
||||||
|
'restart_required': bool(result.returncode == 0 and code_changed),
|
||||||
})
|
})
|
||||||
elif action == 'checkout_branch':
|
elif action == 'checkout_branch':
|
||||||
# Switch branches from the Tools tab. Needed because a checkout
|
# Switch branches from the Tools tab. Needed because a checkout
|
||||||
|
|||||||
@@ -116,14 +116,25 @@ document.body.addEventListener('htmx:afterRequest', function(event) {
|
|||||||
// ===== Restart-pending banner =====
|
// ===== Restart-pending banner =====
|
||||||
// Shown after restart-requiring saves; persists across tab switches (and
|
// Shown after restart-requiring saves; persists across tab switches (and
|
||||||
// reloads, via sessionStorage) until the display restarts or it's dismissed.
|
// reloads, via sessionStorage) until the display restarts or it's dismissed.
|
||||||
window.showRestartPending = function() {
|
window.showRestartPending = function(message) {
|
||||||
try { sessionStorage.setItem('ledmatrix-restart-pending', '1'); } catch { /* private browsing */ }
|
try {
|
||||||
|
sessionStorage.setItem('ledmatrix-restart-pending', '1');
|
||||||
|
// Persisted alongside the flag: a code update and a config save want
|
||||||
|
// different wording, and the banner outlives the page that raised it.
|
||||||
|
if (message) sessionStorage.setItem('ledmatrix-restart-pending-text', message);
|
||||||
|
else sessionStorage.removeItem('ledmatrix-restart-pending-text');
|
||||||
|
} catch { /* private browsing */ }
|
||||||
const banner = document.getElementById('restart-pending-banner');
|
const banner = document.getElementById('restart-pending-banner');
|
||||||
|
const text = document.getElementById('restart-pending-text');
|
||||||
|
if (text && message) text.textContent = message;
|
||||||
if (banner) banner.style.display = 'block';
|
if (banner) banner.style.display = 'block';
|
||||||
};
|
};
|
||||||
|
|
||||||
window.dismissRestartPending = function() {
|
window.dismissRestartPending = function() {
|
||||||
try { sessionStorage.removeItem('ledmatrix-restart-pending'); } catch { /* no-op */ }
|
try {
|
||||||
|
sessionStorage.removeItem('ledmatrix-restart-pending');
|
||||||
|
sessionStorage.removeItem('ledmatrix-restart-pending-text');
|
||||||
|
} catch { /* no-op */ }
|
||||||
const banner = document.getElementById('restart-pending-banner');
|
const banner = document.getElementById('restart-pending-banner');
|
||||||
if (banner) banner.style.display = 'none';
|
if (banner) banner.style.display = 'none';
|
||||||
};
|
};
|
||||||
@@ -151,6 +162,9 @@ document.addEventListener('DOMContentLoaded', function() {
|
|||||||
try {
|
try {
|
||||||
if (sessionStorage.getItem('ledmatrix-restart-pending') === '1') {
|
if (sessionStorage.getItem('ledmatrix-restart-pending') === '1') {
|
||||||
const banner = document.getElementById('restart-pending-banner');
|
const banner = document.getElementById('restart-pending-banner');
|
||||||
|
const saved = sessionStorage.getItem('ledmatrix-restart-pending-text');
|
||||||
|
const text = document.getElementById('restart-pending-text');
|
||||||
|
if (text && saved) text.textContent = saved;
|
||||||
if (banner) banner.style.display = 'block';
|
if (banner) banner.style.display = 'block';
|
||||||
}
|
}
|
||||||
} catch { /* no-op */ }
|
} catch { /* no-op */ }
|
||||||
|
|||||||
@@ -413,7 +413,8 @@
|
|||||||
<div class="flex items-center justify-between">
|
<div class="flex items-center justify-between">
|
||||||
<div class="flex items-center space-x-3">
|
<div class="flex items-center space-x-3">
|
||||||
<i class="fas fa-rotate text-lg"></i>
|
<i class="fas fa-rotate text-lg"></i>
|
||||||
<span class="text-sm font-medium" aria-live="polite">
|
<span class="text-sm font-medium" aria-live="polite"
|
||||||
|
id="restart-pending-text">
|
||||||
Configuration saved — restart the display to apply the changes
|
Configuration saved — restart the display to apply the changes
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
@@ -1146,6 +1147,13 @@
|
|||||||
if (data.status === 'success') {
|
if (data.status === 'success') {
|
||||||
document.getElementById('update-banner').style.display = 'none';
|
document.getElementById('update-banner').style.display = 'none';
|
||||||
try { sessionStorage.removeItem('update-sha-dismissed'); } catch(e) {}
|
try { sessionStorage.removeItem('update-sha-dismissed'); } catch(e) {}
|
||||||
|
// The pull replaced files on disk; the running services still
|
||||||
|
// hold the code they loaded at boot. Ask for the restart that
|
||||||
|
// makes the update actually take effect.
|
||||||
|
if (data.restart_required && typeof window.showRestartPending === 'function') {
|
||||||
|
window.showRestartPending(
|
||||||
|
'Update installed \u2014 restart the display to run the new code');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (typeof showNotification === 'function') {
|
if (typeof showNotification === 'function') {
|
||||||
showNotification(data.message || 'Update complete', data.status || 'success');
|
showNotification(data.message || 'Update complete', data.status || 'success');
|
||||||
|
|||||||
Reference in New Issue
Block a user