mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-08-21 10:29:06 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbfb0e035d |
Regular → Executable
Regular → Executable
Regular → Executable
@@ -49,20 +49,6 @@ class ResourceMetrics:
|
|||||||
self.total_execution_time = self.total_execution_time / self.call_count
|
self.total_execution_time = self.total_execution_time / self.call_count
|
||||||
|
|
||||||
|
|
||||||
#: How often a plugin's metrics are written to the cache, in seconds.
|
|
||||||
#:
|
|
||||||
#: Persisting on every call meant a small file rewritten roughly nine times a
|
|
||||||
#: minute per plugin. On a rig with fourteen active plugins that was ~126
|
|
||||||
#: writes a minute for metrics alone, and since each ~350-byte file costs a
|
|
||||||
#: 4KB block plus an ext4 journal entry, it dominated the device's write
|
|
||||||
#: volume -- on an SD card, which wears out.
|
|
||||||
#:
|
|
||||||
#: The in-memory copy stays authoritative and exact; only the cross-process
|
|
||||||
#: snapshot the web UI reads is delayed, and telemetry up to half a minute old
|
|
||||||
#: is still a fair description of a long-running plugin.
|
|
||||||
_METRICS_PERSIST_INTERVAL = 30.0
|
|
||||||
|
|
||||||
|
|
||||||
class PluginResourceMonitor:
|
class PluginResourceMonitor:
|
||||||
"""
|
"""
|
||||||
Monitors resource usage for plugins.
|
Monitors resource usage for plugins.
|
||||||
@@ -89,10 +75,6 @@ class PluginResourceMonitor:
|
|||||||
# Resource metrics per plugin
|
# Resource metrics per plugin
|
||||||
self._metrics: Dict[str, ResourceMetrics] = {}
|
self._metrics: Dict[str, ResourceMetrics] = {}
|
||||||
self._limits: Dict[str, ResourceLimits] = {}
|
self._limits: Dict[str, ResourceLimits] = {}
|
||||||
# When each plugin's metrics last reached the cache. Metrics change on
|
|
||||||
# every call, so they cannot be de-duplicated the way health state can;
|
|
||||||
# they are rate-limited instead. See _METRICS_PERSIST_INTERVAL.
|
|
||||||
self._metrics_persisted_at: Dict[str, float] = {}
|
|
||||||
|
|
||||||
# Thread-local storage for execution tracking
|
# Thread-local storage for execution tracking
|
||||||
self._local = threading.local()
|
self._local = threading.local()
|
||||||
@@ -250,8 +232,18 @@ class PluginResourceMonitor:
|
|||||||
# CPU is harder to measure per-call, so we track it separately
|
# CPU is harder to measure per-call, so we track it separately
|
||||||
metrics.cpu_percent = self._get_process_cpu_percent()
|
metrics.cpu_percent = self._get_process_cpu_percent()
|
||||||
|
|
||||||
# Persist metrics, at most once per interval per plugin.
|
# Persist metrics
|
||||||
self._persist_metrics(plugin_id, metrics)
|
cache_key = self._get_metrics_key(plugin_id)
|
||||||
|
self.cache_manager.set(cache_key, {
|
||||||
|
'memory_mb': metrics.memory_mb,
|
||||||
|
'cpu_percent': metrics.cpu_percent,
|
||||||
|
'execution_time': metrics.execution_time,
|
||||||
|
'call_count': metrics.call_count,
|
||||||
|
'total_execution_time': metrics.total_execution_time,
|
||||||
|
'max_execution_time': metrics.max_execution_time,
|
||||||
|
'min_execution_time': metrics.min_execution_time if metrics.min_execution_time != float('inf') else 0.0,
|
||||||
|
'last_update_time': metrics.last_update_time
|
||||||
|
})
|
||||||
|
|
||||||
# Check limits
|
# Check limits
|
||||||
if limits:
|
if limits:
|
||||||
@@ -371,37 +363,6 @@ class PluginResourceMonitor:
|
|||||||
summaries[plugin_id] = self.get_metrics_summary(plugin_id)
|
summaries[plugin_id] = self.get_metrics_summary(plugin_id)
|
||||||
return summaries
|
return summaries
|
||||||
|
|
||||||
def _persist_metrics(self, plugin_id: str, metrics: ResourceMetrics,
|
|
||||||
force: bool = False) -> None:
|
|
||||||
"""Write a plugin's metrics to the cache, at most once per interval.
|
|
||||||
|
|
||||||
Caller must hold ``self._lock``.
|
|
||||||
"""
|
|
||||||
# Monotonic, not wall clock: these devices have no RTC, so the clock
|
|
||||||
# jumps by however far off boot-time was the moment NTP first syncs.
|
|
||||||
# A forward jump would allow an early write, a backward one would
|
|
||||||
# stall the snapshot well past the interval.
|
|
||||||
now = time.monotonic()
|
|
||||||
if not force and now - self._metrics_persisted_at.get(plugin_id, 0.0) \
|
|
||||||
< _METRICS_PERSIST_INTERVAL:
|
|
||||||
return
|
|
||||||
cache_key = self._get_metrics_key(plugin_id)
|
|
||||||
self.cache_manager.set(cache_key, {
|
|
||||||
'memory_mb': metrics.memory_mb,
|
|
||||||
'cpu_percent': metrics.cpu_percent,
|
|
||||||
'execution_time': metrics.execution_time,
|
|
||||||
'call_count': metrics.call_count,
|
|
||||||
'total_execution_time': metrics.total_execution_time,
|
|
||||||
'max_execution_time': metrics.max_execution_time,
|
|
||||||
'min_execution_time': (metrics.min_execution_time
|
|
||||||
if metrics.min_execution_time != float('inf')
|
|
||||||
else 0.0),
|
|
||||||
'last_update_time': metrics.last_update_time,
|
|
||||||
})
|
|
||||||
# Only after the write lands. Marking it first would mean a failed
|
|
||||||
# set() bought the next interval's silence without leaving a snapshot.
|
|
||||||
self._metrics_persisted_at[plugin_id] = now
|
|
||||||
|
|
||||||
def reset_metrics(self, plugin_id: str) -> None:
|
def reset_metrics(self, plugin_id: str) -> None:
|
||||||
"""Reset metrics for a plugin."""
|
"""Reset metrics for a plugin."""
|
||||||
with self._lock:
|
with self._lock:
|
||||||
@@ -409,7 +370,4 @@ class PluginResourceMonitor:
|
|||||||
self._metrics[plugin_id] = ResourceMetrics()
|
self._metrics[plugin_id] = ResourceMetrics()
|
||||||
cache_key = self._get_metrics_key(plugin_id)
|
cache_key = self._get_metrics_key(plugin_id)
|
||||||
self.cache_manager.delete(cache_key)
|
self.cache_manager.delete(cache_key)
|
||||||
# Let the next call persist immediately rather than leaving the
|
|
||||||
# deleted key absent for the rest of the interval.
|
|
||||||
self._metrics_persisted_at.pop(plugin_id, None)
|
|
||||||
|
|
||||||
|
|||||||
Regular → Executable
Regular → Executable
@@ -58,7 +58,7 @@ def repos(tmp_path):
|
|||||||
def test_branch_with_upstream_uses_a_plain_pull(repos):
|
def test_branch_with_upstream_uses_a_plain_pull(repos):
|
||||||
args, note, error = resolve_pull_command(str(repos))
|
args, note, error = resolve_pull_command(str(repos))
|
||||||
assert error is None
|
assert error is None
|
||||||
assert args == ['git', 'pull', '--rebase']
|
assert args == ['git', 'pull', '--rebase', '--autostash']
|
||||||
assert note == ''
|
assert note == ''
|
||||||
|
|
||||||
|
|
||||||
@@ -73,7 +73,7 @@ def test_branch_without_upstream_falls_back_to_origin_branch(repos):
|
|||||||
|
|
||||||
args, note, error = resolve_pull_command(str(repos))
|
args, note, error = resolve_pull_command(str(repos))
|
||||||
assert error is None
|
assert error is None
|
||||||
assert args == ['git', 'pull', '--rebase', 'origin', 'audit']
|
assert args == ['git', 'pull', '--rebase', '--autostash', 'origin', 'audit']
|
||||||
assert 'audit' in note
|
assert 'audit' in note
|
||||||
|
|
||||||
|
|
||||||
@@ -155,7 +155,7 @@ def test_switching_attaches_tracking_so_pull_needs_no_fallback(repos):
|
|||||||
|
|
||||||
args, note, error = resolve_pull_command(str(repos))
|
args, note, error = resolve_pull_command(str(repos))
|
||||||
assert error is None
|
assert error is None
|
||||||
assert args == ['git', 'pull', '--rebase']
|
assert args == ['git', 'pull', '--rebase', '--autostash']
|
||||||
assert note == ''
|
assert note == ''
|
||||||
|
|
||||||
|
|
||||||
@@ -200,3 +200,37 @@ def test_stash_option_lets_the_switch_through_and_keeps_the_work(repos):
|
|||||||
assert _git('branch', '--show-current', cwd=repos).stdout.strip() == 'other'
|
assert _git('branch', '--show-current', cwd=repos).stdout.strip() == 'other'
|
||||||
# The edit is not lost — it is on the stash.
|
# The edit is not lost — it is on the stash.
|
||||||
assert 'switch to other' in _git('stash', 'list', cwd=repos).stdout
|
assert 'switch to other' in _git('stash', 'list', cwd=repos).stdout
|
||||||
|
|
||||||
|
|
||||||
|
class TestInstallerDoesNotBlockTheUpdateButton:
|
||||||
|
"""first_time_install.sh chmods scripts that git tracked as 644.
|
||||||
|
|
||||||
|
With core.fileMode true -- the default on Linux -- that leaves five
|
||||||
|
permanently modified tracked files on every machine that ran the
|
||||||
|
installer, and `git pull --rebase` refuses to start:
|
||||||
|
|
||||||
|
error: cannot pull with rebase: You have unstaged changes.
|
||||||
|
|
||||||
|
Tracking them as executable makes the installer's chmod a no-op.
|
||||||
|
"""
|
||||||
|
|
||||||
|
CHMODDED = [
|
||||||
|
'first_time_install.sh',
|
||||||
|
'start_display.sh',
|
||||||
|
'stop_display.sh',
|
||||||
|
'scripts/install/install_service.sh',
|
||||||
|
'scripts/install/install_web_service.sh',
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_scripts_the_installer_chmods_are_tracked_executable(self):
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
root = Path(__file__).resolve().parent.parent
|
||||||
|
out = subprocess.run(['git', 'ls-files', '-s', *self.CHMODDED],
|
||||||
|
capture_output=True, text=True, cwd=str(root)).stdout
|
||||||
|
modes = {line.split()[3]: line.split()[0] for line in out.strip().split('\n') if line}
|
||||||
|
non_exec = sorted(f for f, m in modes.items() if m != '100755')
|
||||||
|
assert not non_exec, (
|
||||||
|
f"{non_exec} are chmodded by the installer but tracked non-executable, "
|
||||||
|
"so every install leaves the working tree dirty and the update "
|
||||||
|
"button cannot pull")
|
||||||
|
|||||||
@@ -127,67 +127,3 @@ class TestForceReload:
|
|||||||
fresh = mon.get_metrics_summary("p", force_reload=True)
|
fresh = mon.get_metrics_summary("p", force_reload=True)
|
||||||
assert fresh["call_count"] == 7
|
assert fresh["call_count"] == 7
|
||||||
assert any(c.kwargs.get("memory_ttl") == 0 for c in cache.get.call_args_list)
|
assert any(c.kwargs.get("memory_ttl") == 0 for c in cache.get.call_args_list)
|
||||||
|
|
||||||
|
|
||||||
class TestMetricsPersistenceChurn:
|
|
||||||
"""Metrics are telemetry; writing them on every call wore the SD card.
|
|
||||||
|
|
||||||
Each write is a ~350-byte file, which on ext4 costs a 4KB block plus a
|
|
||||||
journal entry. At roughly nine calls a minute per plugin across fourteen
|
|
||||||
plugins it dominated the device's write volume.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def test_repeated_calls_persist_once_per_interval(self):
|
|
||||||
cache = _cache()
|
|
||||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
|
||||||
for _ in range(50):
|
|
||||||
mon.monitor_call("p", lambda: None)
|
|
||||||
writes = [c for c in cache.set.call_args_list
|
|
||||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
|
||||||
assert len(writes) == 1, (
|
|
||||||
f"50 calls produced {len(writes)} metric writes; expected 1")
|
|
||||||
|
|
||||||
def test_the_interval_elapsing_allows_the_next_write(self, monkeypatch):
|
|
||||||
import src.plugin_system.resource_monitor as rm
|
|
||||||
cache = _cache()
|
|
||||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
|
||||||
mon.monitor_call("p", lambda: None)
|
|
||||||
# pretend the interval has passed
|
|
||||||
mon._metrics_persisted_at["p"] -= rm._METRICS_PERSIST_INTERVAL + 1
|
|
||||||
mon.monitor_call("p", lambda: None)
|
|
||||||
writes = [c for c in cache.set.call_args_list
|
|
||||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
|
||||||
assert len(writes) == 2
|
|
||||||
|
|
||||||
def test_in_memory_metrics_stay_exact_while_writes_are_skipped(self):
|
|
||||||
mon = PluginResourceMonitor(_cache(), enable_monitoring=False)
|
|
||||||
for _ in range(20):
|
|
||||||
mon.monitor_call("p", lambda: None)
|
|
||||||
assert mon.get_metrics("p").call_count == 20
|
|
||||||
|
|
||||||
def test_reset_lets_the_next_call_persist_immediately(self):
|
|
||||||
cache = _cache()
|
|
||||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
|
||||||
mon.monitor_call("p", lambda: None)
|
|
||||||
mon.reset_metrics("p")
|
|
||||||
mon.monitor_call("p", lambda: None)
|
|
||||||
writes = [c for c in cache.set.call_args_list
|
|
||||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
|
||||||
assert len(writes) == 2, "reset should clear the throttle timestamp"
|
|
||||||
|
|
||||||
def test_a_failed_write_does_not_buy_the_next_interval_of_silence(self):
|
|
||||||
"""A set() that raises must not count as having persisted.
|
|
||||||
|
|
||||||
Marking the timestamp before the write would leave no snapshot in the
|
|
||||||
cache and still suppress the next 30 seconds of attempts.
|
|
||||||
"""
|
|
||||||
cache = _cache()
|
|
||||||
cache.set.side_effect = [OSError("disk full"), None]
|
|
||||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
|
||||||
with pytest.raises(OSError):
|
|
||||||
mon.monitor_call("p", lambda: None)
|
|
||||||
# the very next call must try again rather than skip the interval
|
|
||||||
mon.monitor_call("p", lambda: None)
|
|
||||||
writes = [c for c in cache.set.call_args_list
|
|
||||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
|
||||||
assert len(writes) == 2, "a failed write should be retried, not skipped"
|
|
||||||
|
|||||||
@@ -1657,13 +1657,22 @@ def resolve_pull_command(project_dir):
|
|||||||
backup, or following an install guide that names one. The update button
|
backup, or following an install guide that names one. The update button
|
||||||
then reports a failure the user cannot act on.
|
then reports a failure the user cannot act on.
|
||||||
|
|
||||||
|
``--autostash`` is passed for the same reason. Rebase refuses to start
|
||||||
|
when any tracked file is modified, and on these installs something always
|
||||||
|
is: first_time_install.sh chmods five scripts that git tracked as 644, so
|
||||||
|
every machine that ran the installer carries five permanent mode changes
|
||||||
|
and the update button reports "cannot pull with rebase: You have unstaged
|
||||||
|
changes". Those modes are corrected in this commit, but a user cannot pull
|
||||||
|
the correction while the pull is what is blocked, and any other local edit
|
||||||
|
would reproduce it anyway. Autostash reapplies the changes afterwards.
|
||||||
|
|
||||||
Returns ``(args, note, error)``. When ``origin/<branch>`` exists the pull
|
Returns ``(args, note, error)``. When ``origin/<branch>`` exists the pull
|
||||||
is made explicit against it, so the update proceeds and the branch is
|
is made explicit against it, so the update proceeds and the branch is
|
||||||
given tracking information afterwards.
|
given tracking information afterwards.
|
||||||
"""
|
"""
|
||||||
upstream = _git_upstream(project_dir)
|
upstream = _git_upstream(project_dir)
|
||||||
if upstream:
|
if upstream:
|
||||||
return ['git', 'pull', '--rebase'], '', None
|
return ['git', 'pull', '--rebase', '--autostash'], '', None
|
||||||
|
|
||||||
branch = _git_current_branch(project_dir)
|
branch = _git_current_branch(project_dir)
|
||||||
if not branch:
|
if not branch:
|
||||||
@@ -1673,7 +1682,7 @@ def resolve_pull_command(project_dir):
|
|||||||
)
|
)
|
||||||
if _git_remote_branch_exists(project_dir, branch):
|
if _git_remote_branch_exists(project_dir, branch):
|
||||||
return (
|
return (
|
||||||
['git', 'pull', '--rebase', 'origin', branch],
|
['git', 'pull', '--rebase', '--autostash', 'origin', branch],
|
||||||
f"Branch '{branch}' had no upstream; pulled from origin/{branch} and set it as the upstream.",
|
f"Branch '{branch}' had no upstream; pulled from origin/{branch} and set it as the upstream.",
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user