From 8927a1b6b103c6b5ad225f0ab50a5d6db877420f Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:13:30 -0400 Subject: [PATCH] perf(memory): size the cache to the board and stop reinstalling deps On a 1GB Pi 3B+ the display process settles around 600MB RSS of 905MB total. When the remaining headroom runs out the failure is not a clean crash: fork() starts returning ENOMEM, so sshd accepts connections and closes them before its banner, timer jobs stop running, and the panel goes dark, while already-resident processes keep serving normally. The board looks healthy from outside and cannot be logged into. Only a power cycle clears it. Three contributing causes: - MemoryCache had a fixed 1000-entry ceiling. Entries are parsed API payloads of tens of KB, so one ceiling cannot serve both a 512MB Zero 2 W and an 8GB Pi 5. Now scaled from MemTotal (150 entries at <=1GB, 1500 at >=8GB), overridable with LEDMATRIX_CACHE_MAX_ENTRIES. - requirements_are_satisfied() returned False for any requirement with extras, so a plugin depending on python-socketio[client] re-ran pip on every single start: ~8s, a network dependency, and a 100-200MB spike at the least convenient moment. During a restart loop it repeats for each restart. Extras are now resolved one level deep against installed metadata, keeping the conservative "anything unverifiable falls through to pip" contract. - ledmatrix.service had no memory ceiling. MemoryMax=85% expressed as a percentage so one unit file suits every board. Note this needs the memory cgroup controller, which Pi firmware disables by default; first_time_install.sh now adds cgroup_enable=memory to cmdline.txt, and the unit file documents how to verify it took effect. first_time_install.sh also enables persistent journald storage (capped at 64M). Default storage is volatile, so every reboot destroys the logs that would explain why the board rebooted. Co-Authored-By: Claude Opus 5 --- first_time_install.sh | 37 ++++++++++++++++++ src/cache/memory_cache.py | 47 +++++++++++++++++++++++ src/cache_manager.py | 6 ++- src/plugin_system/plugin_loader.py | 60 ++++++++++++++++++++++++++++-- systemd/ledmatrix.service | 13 +++++++ 5 files changed, 157 insertions(+), 6 deletions(-) diff --git a/first_time_install.sh b/first_time_install.sh index 2256be52..b3cb190b 100644 --- a/first_time_install.sh +++ b/first_time_install.sh @@ -1688,6 +1688,43 @@ else echo "✗ $CMDLINE_FILE not found; skipping isolcpus optimization" fi +# Enable the memory cgroup controller (idempotent). +# The Pi firmware boots with cgroup_disable=memory, so systemd's MemoryMax= is +# accepted and silently ignored — the display service then has no ceiling, and +# a runaway takes the whole board down (sshd can no longer fork, the panel goes +# dark) rather than just restarting the one service. +if [ "$SKIP_PERF" != "1" ] && [ -f "$CMDLINE_FILE" ]; then + if grep -q 'cgroup_enable=memory' "$CMDLINE_FILE"; then + echo "cgroup_enable=memory already present in $CMDLINE_FILE" + else + echo "Adding cgroup_enable=memory to $CMDLINE_FILE..." + cp "$CMDLINE_FILE" "$CMDLINE_FILE.bak" 2>/dev/null || true + sed -i '1 s/$/ cgroup_enable=memory cgroup_memory=1/' "$CMDLINE_FILE" + echo " Takes effect after reboot. Verify with:" + echo " grep memory /sys/fs/cgroup/cgroup.controllers" + fi +fi + +# Persist the journal (idempotent). +# These images default to volatile storage: journald keeps everything in /run +# (tmpfs), so every reboot destroys the logs — including the ones that would +# explain why the board rebooted. Capped so an SD card is not worn out by logs. +if [ -d /var/log/journal ] && [ -n "$(ls -A /var/log/journal 2>/dev/null)" ]; then + echo "Persistent journald storage already enabled" +else + echo "Enabling persistent journald storage..." + mkdir -p /etc/systemd/journald.conf.d + cat > /etc/systemd/journald.conf.d/ledmatrix-persistent.conf <<'JOURNALD' +# Installed by LEDMatrix first_time_install.sh +[Journal] +Storage=persistent +SystemMaxUse=64M +JOURNALD + mkdir -p /var/log/journal + systemd-tmpfiles --create --prefix /var/log/journal >/dev/null 2>&1 || true + systemctl restart systemd-journald >/dev/null 2>&1 || true +fi + # Ensure dtparam=audio=off in config.txt (idempotent) if [ "$SKIP_PERF" = "1" ]; then : # skipped diff --git a/src/cache/memory_cache.py b/src/cache/memory_cache.py index a13e3ccf..60993427 100644 --- a/src/cache/memory_cache.py +++ b/src/cache/memory_cache.py @@ -4,11 +4,58 @@ Memory Cache Handles in-memory caching with TTL support, size limits, and automatic cleanup. """ +import os import time import threading import logging from typing import Dict, Any, Optional +# Historical fixed ceiling, kept as the fallback when RAM cannot be read. +DEFAULT_MAX_SIZE = 1000 + + +def _total_memory_mb() -> Optional[float]: + """Physical RAM in MB, or None where /proc/meminfo is unavailable.""" + try: + with open('/proc/meminfo', 'r', encoding='utf-8') as fh: + for line in fh: + if line.startswith('MemTotal:'): + return int(line.split()[1]) / 1024 + except (OSError, ValueError, IndexError): + return None + return None + + +def default_max_size() -> int: + """Entry ceiling scaled to this machine's RAM. + + One fixed ceiling cannot serve both a 512 MB Pi Zero 2 W and an 8 GB Pi 5. + Entries here are parsed API payloads that routinely run tens of kilobytes + each, so a thousand of them is a comfortable cache on a large board and a + substantial fraction of total RAM on a small one — where the process + competing for that RAM is also driving the panel. Set + LEDMATRIX_CACHE_MAX_ENTRIES to override. + """ + override = os.environ.get('LEDMATRIX_CACHE_MAX_ENTRIES') + if override: + try: + value = int(override) + if value > 0: + return value + except ValueError: + pass + + total_mb = _total_memory_mb() + if total_mb is None: + return DEFAULT_MAX_SIZE + if total_mb < 1536: # 512 MB and 1 GB boards + return 150 + if total_mb < 3072: # 2 GB + return 400 + if total_mb < 6144: # 4 GB + return 800 + return 1500 # 8 GB and up + class MemoryCache: """Manages in-memory cache with TTL and size limits.""" diff --git a/src/cache_manager.py b/src/cache_manager.py index bef98784..67624de1 100644 --- a/src/cache_manager.py +++ b/src/cache_manager.py @@ -33,7 +33,7 @@ import logging import threading import tempfile from src.exceptions import CacheError -from src.cache.memory_cache import MemoryCache +from src.cache.memory_cache import MemoryCache, default_max_size from src.cache.disk_cache import DiskCache from src.cache.cache_strategy import CacheStrategy from src.cache.cache_metrics import CacheMetrics @@ -84,7 +84,9 @@ class CacheManager: self.logger.warning("ConfigManager not available, using default cache intervals") # Initialize cache components using composition - self._memory_cache_component = MemoryCache(max_size=1000, cleanup_interval=300.0) + self._memory_cache_component = MemoryCache( + max_size=default_max_size(), cleanup_interval=300.0 + ) self._disk_cache_component = DiskCache(cache_dir=self.cache_dir, logger=self.logger) self._strategy_component = CacheStrategy(config_manager=self.config_manager, logger=self.logger) self._metrics_component = CacheMetrics(logger=self.logger) diff --git a/src/plugin_system/plugin_loader.py b/src/plugin_system/plugin_loader.py index bd748ef7..e2e2d11c 100644 --- a/src/plugin_system/plugin_loader.py +++ b/src/plugin_system/plugin_loader.py @@ -14,7 +14,7 @@ import sys import subprocess import threading from pathlib import Path -from typing import Dict, Any, Optional, Tuple, Type +from typing import Dict, Any, List, Optional, Tuple, Type import logging from packaging.requirements import InvalidRequirement, Requirement @@ -45,6 +45,58 @@ def requirements_has_real_deps(requirements_file: str) -> bool: return False +def _extra_dependencies(dist_name: str, extras) -> Optional[List[Requirement]]: + """Dependencies a distribution declares *only* behind the given extras. + + Returns None when the installed metadata cannot be read or parsed, so the + caller can fall back to running pip rather than assuming anything. + """ + try: + meta = importlib.metadata.metadata(dist_name) + except importlib.metadata.PackageNotFoundError: + return None + + gated: List[Requirement] = [] + for raw in meta.get_all('Requires-Dist') or []: + try: + dep = Requirement(raw) + except InvalidRequirement: + return None + if dep.marker is None: + continue + # Keep only what the distribution gates behind an extra we asked for: + # satisfied when `extra` is that name, but not when no extra is + # requested. A marker that holds either way (python_version, sys_platform) + # belongs to the base install and is already covered by the version check. + if dep.marker.evaluate({'extra': ''}): + continue + if any(dep.marker.evaluate({'extra': extra}) for extra in extras): + gated.append(dep) + return gated + + +def _extras_are_satisfied(req: Requirement) -> bool: + """Check the dependencies pulled in by req's extras are installed. + + One level deep, not transitive: enough to tell "the extra was installed" + from "the extra was never installed", which is all the caller needs to + decide whether pip has work to do. Anything unreadable returns False, so + the caller still falls through to pip. + """ + gated = _extra_dependencies(req.name, req.extras) + if gated is None: + return False + + for dep in gated: + try: + dep_version = importlib.metadata.version(dep.name) + except importlib.metadata.PackageNotFoundError: + return False + if dep.specifier and not dep.specifier.contains(dep_version, prereleases=True): + return False + return True + + def requirements_are_satisfied(requirements_file: str) -> bool: """ Check whether every real requirement line in requirements.txt is already @@ -76,9 +128,6 @@ def requirements_are_satisfied(requirements_file: str) -> bool: except InvalidRequirement: return False - if req.extras: - return False # verifying extras' sub-dependencies isn't worth it here - if req.marker is not None and not req.marker.evaluate(): continue # not applicable on this platform/interpreter @@ -90,6 +139,9 @@ def requirements_are_satisfied(requirements_file: str) -> bool: if req.specifier and not req.specifier.contains(installed_version, prereleases=True): return False + if req.extras and not _extras_are_satisfied(req): + return False + return True diff --git a/systemd/ledmatrix.service b/systemd/ledmatrix.service index d2aba454..d5f064b6 100644 --- a/systemd/ledmatrix.service +++ b/systemd/ledmatrix.service @@ -15,6 +15,19 @@ ExecStart=/usr/bin/python3 __PROJECT_ROOT_DIR__/run.py # that a successful outcome and never bringing it back. Restart=always RestartSec=10 +# Memory ceiling as a share of physical RAM, so one unit file suits a 512 MB +# Pi Zero 2 W and an 8 GB Pi 5 alike. This is a backstop, not a tuning knob: it +# turns "the board runs out of memory, stops being able to fork, and takes sshd +# and the panel down together until someone pulls the plug" into "this one +# service restarts". +# +# NOTE: Raspberry Pi firmware boots the kernel with cgroup_disable=memory, and +# systemd accepts this setting and then silently ignores it. Verify with: +# grep memory /sys/fs/cgroup/cgroup.controllers +# If that prints nothing, add "cgroup_enable=memory cgroup_memory=1" to +# /boot/firmware/cmdline.txt (all on line 1) and reboot. first_time_install.sh +# does this for you. +MemoryMax=85% StandardOutput=journal StandardError=journal SyslogIdentifier=ledmatrix