From e745ae8060b6381939860ca874c57cf7822d1bf3 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:47:29 -0400 Subject: [PATCH] feat(display): cap malloc arenas in-process and malloc_trim between screens (#774) * feat(display): cap malloc arenas in-process and malloc_trim between screens Co-Authored-By: Claude Opus 5.5 * test: malloc_tuning with ctypes mocked; add to the mypy ratchet Co-Authored-By: Claude Opus 5.5 * docs(changelog): malloc arena cap and malloc_trim between screens Co-Authored-By: Claude Opus 5.5 * docs(changelog): spacing Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 23 +++++ mypy-clean.txt | 1 + run.py | 6 ++ src/display_controller.py | 7 ++ src/malloc_tuning.py | 123 ++++++++++++++++++++++ test/test_malloc_tuning.py | 206 +++++++++++++++++++++++++++++++++++++ 6 files changed, 366 insertions(+) create mode 100644 src/malloc_tuning.py create mode 100644 test/test_malloc_tuning.py diff --git a/CHANGELOG.md b/CHANGELOG.md index abeee525..44724d01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,29 @@ accepts both, but the store flags the old spelling as deprecated ## Unreleased +### The display hands freed memory back to the OS + +The display process's resident memory climbed in steps for hours while the +data it held stayed flat: glibc keeps what Python frees in per-thread malloc +arenas and returns little of it. `src/malloc_tuning.py` (new, standard library +only, a no-op off Linux/glibc) does two things in-process, so it reaches +devices without re-running the installer: + +- **Arena cap at start-up.** `run.py` calls `mallopt(M_ARENA_MAX, 2)` before any + thread exists, the same cap as the unit's `Environment=MALLOC_ARENA_MAX=2`. + Units installed before that line never got it (systemd runs the copy in + `/etc/systemd/system`); a `MALLOC_ARENA_MAX` in the environment still wins. +- **`malloc_trim(0)` between screens**, at most every 5 minutes, from the top of + the render loop where no frame is being drawn. Measured on a Pi 4: 2-11 ms + per call. + +On ledpi (Pi 4, 192x48, Vegas on, nine plugins, a unit without +`MALLOC_ARENA_MAX`), alternated main / branch / branch / main arms of 2.5 h: +two hours in, resident memory was 551 MB on main (the second main arm was +already at 651 MB after 1 h 44 min) against 412 and 386 MB with this change, +and the 20-minute frame soaks came out at 0.147-0.165% late against main's +0.151-0.188%. + ## 3.8.1 Smooth scrolling at the slower speeds, and the fixes and performance work diff --git a/mypy-clean.txt b/mypy-clean.txt index ffc7a6c5..aa3ba212 100644 --- a/mypy-clean.txt +++ b/mypy-clean.txt @@ -59,6 +59,7 @@ src/ipc/contract.py src/ipc/server.py src/logging_config.py src/logo_downloader.py +src/malloc_tuning.py src/matrix_support.py src/pi5_matrix_support.py src/plugin_system/__init__.py diff --git a/run.py b/run.py index c327d549..af7d2c6c 100755 --- a/run.py +++ b/run.py @@ -14,6 +14,12 @@ project_dir = os.path.dirname(os.path.abspath(__file__)) if project_dir not in sys.path: sys.path.insert(0, project_dir) +# Cap glibc's malloc arenas before any thread exists (arenas already made +# stay): the in-process twin of the unit's MALLOC_ARENA_MAX=2, for units +# installed before that line. A no-op off glibc. See src/malloc_tuning.py. +from src import malloc_tuning +malloc_tuning.cap_arenas() + # Under systemd the watchdog clock is already running, and start-up (plugin # loads, initial updates) takes far longer than the render loop's limit. Widen # it before anything slow is imported; the render loop narrows it again once diff --git a/src/display_controller.py b/src/display_controller.py index 84bfbe8c..44a70f84 100644 --- a/src/display_controller.py +++ b/src/display_controller.py @@ -37,6 +37,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed # pylint: disab import pytz from src import display_watchdog +from src.malloc_tuning import MallocTrimmer from src.display_arbiter import ( Arbiter, ArbiterInputs, ArbiterState, FramePolicy, ScreenPlan, Source, WifiNotice, live_pick, live_takeover, on_demand_bound, rotation_plan, @@ -4217,6 +4218,7 @@ class DisplayController: logger.info(f"Initial mode set to: {self.current_display_mode} (index: {self.current_mode_index}, total modes: {len(self.available_modes)})") self._publish_current_mode_state() runner = ScreenRunner(_MODULE_CLOCK, _ScreenHost(self), logger) + trimmer = MallocTrimmer() while True: # Arms the watchdog after the first frame -- or after the @@ -4224,6 +4226,11 @@ class DisplayController: # it from then on. display_watchdog.watchdog.loop_pass() + # Between screens, nothing being drawn: every few minutes hand + # the memory glibc is holding for freed images back to the OS + # (src/malloc_tuning.py). A clock read when none is due. + trimmer.maybe_trim() + # Apply plugin enable/disable edits saved via the web UI. The # config-watcher thread only sets the flag; loading/unloading and # rebuilding available_modes happens here on the render thread so diff --git a/src/malloc_tuning.py b/src/malloc_tuning.py new file mode 100644 index 00000000..a2fcf632 --- /dev/null +++ b/src/malloc_tuning.py @@ -0,0 +1,123 @@ +"""Keep glibc's malloc from holding on to memory the display has freed. + +The display process allocates and frees PIL images and numpy buffers all day +from a dozen threads. glibc gives each allocating thread its own malloc arena +(up to 8 x CPU count) and returns little of what is freed inside them to the +OS, so resident memory climbs for hours while the live data stays flat. Two +in-process remedies, both standard library only (ctypes) and both no-ops off +Linux/glibc: + +* :func:`cap_arenas` -- ``mallopt(M_ARENA_MAX, 2)``, the in-process twin of the + unit's ``Environment=MALLOC_ARENA_MAX=2``. Units installed before that line + existed never got it (systemd runs the copy in /etc/systemd/system), so the + process applies it itself. Call it before any other thread starts: arenas + already created stay. A ``MALLOC_ARENA_MAX`` set in the environment wins. +* :class:`MallocTrimmer` -- ``malloc_trim(0)`` at most every few minutes, + called from the render loop between screens, where no frame is being drawn. + glibc 2.8+ releases free pages from the middle of every arena, not only the + top of the main heap. + +Without glibc (macOS, Windows, musl, the dev server on any of them) nothing is +loaded and every call returns False. +""" +import ctypes +import logging +import os +import sys +import time +from typing import Any, Callable, Optional + +logger = logging.getLogger(__name__) + +#: glibc's mallopt() parameter number for the arena cap (malloc.h). +M_ARENA_MAX = -8 + +#: The arena cap applied when the environment does not set one; the same value +#: as the unit's ``MALLOC_ARENA_MAX``. +DEFAULT_ARENA_MAX = 2 + +#: Seconds between malloc_trim() calls. A trim takes about 1-20 ms on a Pi 4, +#: so this keeps it far from frame timing while still returning memory long +#: before it piles up. +TRIM_INTERVAL_SECONDS = 300.0 + +_UNLOADED = object() +_libc: Any = _UNLOADED + + +def _load_libc() -> Optional[Any]: + """The process's C library if it is glibc with malloc_trim, else None.""" + global _libc + if _libc is _UNLOADED: + _libc = None + if sys.platform.startswith('linux'): + try: + libc = ctypes.CDLL(None) + # gnu_get_libc_version is glibc-only, so musl (which has + # mallopt but no malloc_trim) is left alone as a whole. + if all(hasattr(libc, name) for name in + ('gnu_get_libc_version', 'malloc_trim', 'mallopt')): + libc.malloc_trim.argtypes = [ctypes.c_size_t] + libc.malloc_trim.restype = ctypes.c_int + libc.mallopt.argtypes = [ctypes.c_int, ctypes.c_int] + libc.mallopt.restype = ctypes.c_int + _libc = libc + except (OSError, AttributeError, TypeError): + logger.debug("glibc malloc controls unavailable", exc_info=True) + return _libc + + +def cap_arenas(max_arenas: int = DEFAULT_ARENA_MAX) -> bool: + """Cap glibc's malloc arenas at ``max_arenas``. True when the cap was set. + + Skipped when ``MALLOC_ARENA_MAX`` is in the environment: glibc has read it + already, and an operator who set it chose that value. + """ + if os.environ.get('MALLOC_ARENA_MAX'): + return False + libc = _load_libc() + if libc is None: + return False + try: + return bool(libc.mallopt(M_ARENA_MAX, int(max_arenas))) + except Exception: # pylint: disable=broad-except + logger.debug("mallopt(M_ARENA_MAX) failed", exc_info=True) + return False + + +class MallocTrimmer: + """Calls ``malloc_trim(0)`` at most once per ``interval`` seconds. + + :meth:`maybe_trim` is meant for an idle point of the render loop; it costs + one clock read when no trim is due. The first trim comes one interval + after construction, so start-up's allocations have settled. + """ + + def __init__(self, interval: float = TRIM_INTERVAL_SECONDS, + clock: Callable[[], float] = time.monotonic) -> None: + self._interval = interval + self._clock = clock + self._libc = _load_libc() + self._next = clock() + interval + + @property + def available(self) -> bool: + return self._libc is not None + + def maybe_trim(self) -> bool: + """Trim if one is due. True when malloc_trim ran and released memory.""" + if self._libc is None: + return False + now = self._clock() + if now < self._next: + return False + self._next = now + self._interval + try: + released = bool(self._libc.malloc_trim(0)) + except Exception: # pylint: disable=broad-except + logger.debug("malloc_trim failed; not trying again", exc_info=True) + self._libc = None + return False + logger.debug("malloc_trim(0) took %.1f ms, released=%s", + (self._clock() - now) * 1000.0, released) + return released diff --git a/test/test_malloc_tuning.py b/test/test_malloc_tuning.py new file mode 100644 index 00000000..438cce48 --- /dev/null +++ b/test/test_malloc_tuning.py @@ -0,0 +1,206 @@ +"""src/malloc_tuning.py: glibc arena cap and periodic malloc_trim, ctypes mocked.""" +import ctypes +from pathlib import Path +from unittest import mock + +import pytest + +from src import malloc_tuning as mt + + +class FakeLibc: + """Stands in for ctypes.CDLL(None) on glibc: records calls.""" + + def __init__(self, trim_result=1, glibc=True): + self.trims = [] + self.mallopts = [] + self._trim_result = trim_result + if glibc: + self.gnu_get_libc_version = lambda: b'2.41' + self.malloc_trim = mock.Mock(side_effect=self._trim) + self.mallopt = mock.Mock(side_effect=self._mallopt) + + def _trim(self, pad): + self.trims.append(pad) + if isinstance(self._trim_result, Exception): + raise self._trim_result + return self._trim_result + + def _mallopt(self, param, value): + self.mallopts.append((param, value)) + return 1 + + +@pytest.fixture(autouse=True) +def fresh_libc(monkeypatch): + """Each test loads the C library itself; nothing real is called.""" + monkeypatch.setattr(mt, '_libc', mt._UNLOADED) + monkeypatch.delenv('MALLOC_ARENA_MAX', raising=False) + yield + + +def _on_glibc(monkeypatch, libc): + monkeypatch.setattr(mt.sys, 'platform', 'linux') + cdll = mock.Mock(return_value=libc) + monkeypatch.setattr(mt.ctypes, 'CDLL', cdll) + return cdll + + +class Clock: + def __init__(self, t=1000.0): + self.t = t + + def __call__(self): + return self.t + + +# -- loading ---------------------------------------------------------------- + +@pytest.mark.parametrize('platform', ['win32', 'darwin', 'freebsd14']) +def test_not_linux_loads_nothing(monkeypatch, platform): + monkeypatch.setattr(mt.sys, 'platform', platform) + cdll = mock.Mock(side_effect=AssertionError('must not load')) + monkeypatch.setattr(mt.ctypes, 'CDLL', cdll) + assert mt._load_libc() is None + assert mt.cap_arenas() is False + trimmer = mt.MallocTrimmer(interval=0) + assert not trimmer.available + assert trimmer.maybe_trim() is False + cdll.assert_not_called() + + +def test_linux_without_glibc_is_a_noop(monkeypatch): + """musl: no gnu_get_libc_version (and no malloc_trim) -- nothing is called.""" + libc = FakeLibc(glibc=False) + del libc.malloc_trim + _on_glibc(monkeypatch, libc) + assert mt._load_libc() is None + assert mt.cap_arenas() is False + assert mt.MallocTrimmer(interval=0).maybe_trim() is False + assert libc.mallopts == [] + + +def test_cdll_failure_is_a_noop(monkeypatch): + monkeypatch.setattr(mt.sys, 'platform', 'linux') + monkeypatch.setattr(mt.ctypes, 'CDLL', mock.Mock(side_effect=OSError('no libc'))) + assert mt._load_libc() is None + assert mt.cap_arenas() is False + + +def test_loads_once(monkeypatch): + cdll = _on_glibc(monkeypatch, FakeLibc()) + mt._load_libc() + mt._load_libc() + mt.MallocTrimmer() + assert cdll.call_count == 1 + + +def test_declares_c_signatures(monkeypatch): + libc = FakeLibc() + _on_glibc(monkeypatch, libc) + mt._load_libc() + assert libc.malloc_trim.argtypes == [ctypes.c_size_t] + assert libc.mallopt.argtypes == [ctypes.c_int, ctypes.c_int] + + +# -- cap_arenas --------------------------------------------------------------- + +def test_cap_arenas_calls_mallopt(monkeypatch): + libc = FakeLibc() + _on_glibc(monkeypatch, libc) + assert mt.cap_arenas() is True + assert libc.mallopts == [(mt.M_ARENA_MAX, 2)] + assert mt.M_ARENA_MAX == -8 # glibc's malloc.h + + +def test_cap_arenas_defers_to_the_environment(monkeypatch): + libc = FakeLibc() + _on_glibc(monkeypatch, libc) + monkeypatch.setenv('MALLOC_ARENA_MAX', '4') + assert mt.cap_arenas() is False + assert libc.mallopts == [] + + +def test_cap_arenas_swallows_errors(monkeypatch): + libc = FakeLibc() + libc.mallopt = mock.Mock(side_effect=RuntimeError('boom')) + _on_glibc(monkeypatch, libc) + assert mt.cap_arenas() is False + + +def test_cap_arenas_matches_the_unit(): + """The in-process default is the value the unit's MALLOC_ARENA_MAX carries.""" + unit = (Path(__file__).resolve().parent.parent / 'systemd' / 'ledmatrix.service').read_text() + assert f'Environment=MALLOC_ARENA_MAX={mt.DEFAULT_ARENA_MAX}\n' in unit + + +# -- MallocTrimmer ------------------------------------------------------------ + +def test_trim_waits_one_interval_then_rate_limits(monkeypatch): + libc = FakeLibc() + _on_glibc(monkeypatch, libc) + clock = Clock() + trimmer = mt.MallocTrimmer(interval=300, clock=clock) + assert trimmer.available + assert trimmer.maybe_trim() is False # start-up: not yet + clock.t += 299.9 + assert trimmer.maybe_trim() is False + clock.t += 0.1 + assert trimmer.maybe_trim() is True + assert libc.trims == [0] + clock.t += 100 + assert trimmer.maybe_trim() is False # rate-limited + clock.t += 200 + assert trimmer.maybe_trim() is True + assert libc.trims == [0, 0] + + +def test_trim_reports_nothing_released(monkeypatch): + libc = FakeLibc(trim_result=0) + _on_glibc(monkeypatch, libc) + clock = Clock() + trimmer = mt.MallocTrimmer(interval=10, clock=clock) + clock.t += 10 + assert trimmer.maybe_trim() is False + assert libc.trims == [0] + + +def test_trim_failure_disables_trimming(monkeypatch): + libc = FakeLibc(trim_result=RuntimeError('boom')) + _on_glibc(monkeypatch, libc) + clock = Clock() + trimmer = mt.MallocTrimmer(interval=10, clock=clock) + clock.t += 10 + assert trimmer.maybe_trim() is False + clock.t += 10 + assert trimmer.maybe_trim() is False + assert libc.trims == [0] # not retried + assert not trimmer.available + + +# -- wiring ------------------------------------------------------------------- + +def test_run_py_caps_arenas_before_threads(): + """run.py applies the cap before the watchdog or the controller import.""" + src = (Path(__file__).resolve().parent.parent / 'run.py').read_text() + cap = src.index('malloc_tuning.cap_arenas()') + assert cap < src.index('display_watchdog.watchdog.begin_startup()') + assert cap < src.index('from src.display_controller import main') + + +def test_render_loop_trims_between_screens(): + src = (Path(__file__).resolve().parent.parent / 'src' / 'display_controller.py').read_text() + loop = src.index('display_watchdog.watchdog.loop_pass()') + trim = src.index('trimmer.maybe_trim()') + assert loop < trim < src.index('outcome = runner.run(plan, manager_to_display)') + + +@pytest.mark.skipif(not mt.sys.platform.startswith('linux'), reason='glibc only') +def test_real_libc_on_linux(): + """On a real Linux C library the calls go through without raising.""" + if mt._load_libc() is None: + pytest.skip('not glibc') + trimmer = mt.MallocTrimmer(interval=0) + assert trimmer.available + assert trimmer.maybe_trim() in (True, False) + assert trimmer.available # did not fail and disable itself