feat(display): cap malloc arenas in-process and malloc_trim between screens (#774)

* feat(display): cap malloc arenas in-process and malloc_trim between screens

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: malloc_tuning with ctypes mocked; add to the mypy ratchet

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs(changelog): malloc arena cap and malloc_trim between screens

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs(changelog): spacing

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -19,6 +19,29 @@ accepts both, but the store flags the old spelling as deprecated
 
 ## Unreleased
 
+### The display hands freed memory back to the OS
+
+The display process's resident memory climbed in steps for hours while the
+data it held stayed flat: glibc keeps what Python frees in per-thread malloc
+arenas and returns little of it. `src/malloc_tuning.py` (new, standard library
+only, a no-op off Linux/glibc) does two things in-process, so it reaches
+devices without re-running the installer:
+
+- **Arena cap at start-up.** `run.py` calls `mallopt(M_ARENA_MAX, 2)` before any
+  thread exists, the same cap as the unit's `Environment=MALLOC_ARENA_MAX=2`.
+  Units installed before that line never got it (systemd runs the copy in
+  `/etc/systemd/system`); a `MALLOC_ARENA_MAX` in the environment still wins.
+- **`malloc_trim(0)` between screens**, at most every 5 minutes, from the top of
+  the render loop where no frame is being drawn. Measured on a Pi 4: 2-11 ms
+  per call.
+
+On ledpi (Pi 4, 192x48, Vegas on, nine plugins, a unit without
+`MALLOC_ARENA_MAX`), alternated main / branch / branch / main arms of 2.5 h:
+two hours in, resident memory was 551 MB on main (the second main arm was
+already at 651 MB after 1 h 44 min) against 412 and 386 MB with this change,
+and the 20-minute frame soaks came out at 0.147-0.165% late against main's
+0.151-0.188%.
+
 ## 3.8.1
 
 Smooth scrolling at the slower speeds, and the fixes and performance work

mypy-clean.txt

@@ -59,6 +59,7 @@ src/ipc/contract.py
 src/ipc/server.py
 src/logging_config.py
 src/logo_downloader.py
+src/malloc_tuning.py
 src/matrix_support.py
 src/pi5_matrix_support.py
 src/plugin_system/__init__.py

run.py

@@ -14,6 +14,12 @@ project_dir = os.path.dirname(os.path.abspath(__file__))
 if project_dir not in sys.path:
     sys.path.insert(0, project_dir)
 
+# Cap glibc's malloc arenas before any thread exists (arenas already made
+# stay): the in-process twin of the unit's MALLOC_ARENA_MAX=2, for units
+# installed before that line. A no-op off glibc. See src/malloc_tuning.py.
+from src import malloc_tuning
+malloc_tuning.cap_arenas()
+
 # Under systemd the watchdog clock is already running, and start-up (plugin
 # loads, initial updates) takes far longer than the render loop's limit. Widen
 # it before anything slow is imported; the render loop narrows it again once

src/display_controller.py

@@ -37,6 +37,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed  # pylint: disab
 import pytz
 
 from src import display_watchdog
+from src.malloc_tuning import MallocTrimmer
 from src.display_arbiter import (
     Arbiter, ArbiterInputs, ArbiterState, FramePolicy,
     ScreenPlan, Source, WifiNotice, live_pick, live_takeover, on_demand_bound, rotation_plan,
@@ -4217,6 +4218,7 @@ class DisplayController:
             logger.info(f"Initial mode set to: {self.current_display_mode} (index: {self.current_mode_index}, total modes: {len(self.available_modes)})")
             self._publish_current_mode_state()
             runner = ScreenRunner(_MODULE_CLOCK, _ScreenHost(self), logger)
+            trimmer = MallocTrimmer()
 
             while True:
                 # Arms the watchdog after the first frame -- or after the
@@ -4224,6 +4226,11 @@ class DisplayController:
                 # it from then on.
                 display_watchdog.watchdog.loop_pass()
 
+                # Between screens, nothing being drawn: every few minutes hand
+                # the memory glibc is holding for freed images back to the OS
+                # (src/malloc_tuning.py). A clock read when none is due.
+                trimmer.maybe_trim()
+
                 # Apply plugin enable/disable edits saved via the web UI. The
                 # config-watcher thread only sets the flag; loading/unloading and
                 # rebuilding available_modes happens here on the render thread so

src/malloc_tuning.py

@@ -0,0 +1,123 @@
+"""Keep glibc's malloc from holding on to memory the display has freed.
+
+The display process allocates and frees PIL images and numpy buffers all day
+from a dozen threads. glibc gives each allocating thread its own malloc arena
+(up to 8 x CPU count) and returns little of what is freed inside them to the
+OS, so resident memory climbs for hours while the live data stays flat. Two
+in-process remedies, both standard library only (ctypes) and both no-ops off
+Linux/glibc:
+
+* :func:`cap_arenas` -- ``mallopt(M_ARENA_MAX, 2)``, the in-process twin of the
+  unit's ``Environment=MALLOC_ARENA_MAX=2``. Units installed before that line
+  existed never got it (systemd runs the copy in /etc/systemd/system), so the
+  process applies it itself. Call it before any other thread starts: arenas
+  already created stay. A ``MALLOC_ARENA_MAX`` set in the environment wins.
+* :class:`MallocTrimmer` -- ``malloc_trim(0)`` at most every few minutes,
+  called from the render loop between screens, where no frame is being drawn.
+  glibc 2.8+ releases free pages from the middle of every arena, not only the
+  top of the main heap.
+
+Without glibc (macOS, Windows, musl, the dev server on any of them) nothing is
+loaded and every call returns False.
+"""
+import ctypes
+import logging
+import os
+import sys
+import time
+from typing import Any, Callable, Optional
+
+logger = logging.getLogger(__name__)
+
+#: glibc's mallopt() parameter number for the arena cap (malloc.h).
+M_ARENA_MAX = -8
+
+#: The arena cap applied when the environment does not set one; the same value
+#: as the unit's ``MALLOC_ARENA_MAX``.
+DEFAULT_ARENA_MAX = 2
+
+#: Seconds between malloc_trim() calls. A trim takes about 1-20 ms on a Pi 4,
+#: so this keeps it far from frame timing while still returning memory long
+#: before it piles up.
+TRIM_INTERVAL_SECONDS = 300.0
+
+_UNLOADED = object()
+_libc: Any = _UNLOADED
+
+
+def _load_libc() -> Optional[Any]:
+    """The process's C library if it is glibc with malloc_trim, else None."""
+    global _libc
+    if _libc is _UNLOADED:
+        _libc = None
+        if sys.platform.startswith('linux'):
+            try:
+                libc = ctypes.CDLL(None)
+                # gnu_get_libc_version is glibc-only, so musl (which has
+                # mallopt but no malloc_trim) is left alone as a whole.
+                if all(hasattr(libc, name) for name in
+                       ('gnu_get_libc_version', 'malloc_trim', 'mallopt')):
+                    libc.malloc_trim.argtypes = [ctypes.c_size_t]
+                    libc.malloc_trim.restype = ctypes.c_int
+                    libc.mallopt.argtypes = [ctypes.c_int, ctypes.c_int]
+                    libc.mallopt.restype = ctypes.c_int
+                    _libc = libc
+            except (OSError, AttributeError, TypeError):
+                logger.debug("glibc malloc controls unavailable", exc_info=True)
+    return _libc
+
+
+def cap_arenas(max_arenas: int = DEFAULT_ARENA_MAX) -> bool:
+    """Cap glibc's malloc arenas at ``max_arenas``. True when the cap was set.
+
+    Skipped when ``MALLOC_ARENA_MAX`` is in the environment: glibc has read it
+    already, and an operator who set it chose that value.
+    """
+    if os.environ.get('MALLOC_ARENA_MAX'):
+        return False
+    libc = _load_libc()
+    if libc is None:
+        return False
+    try:
+        return bool(libc.mallopt(M_ARENA_MAX, int(max_arenas)))
+    except Exception:  # pylint: disable=broad-except
+        logger.debug("mallopt(M_ARENA_MAX) failed", exc_info=True)
+        return False
+
+
+class MallocTrimmer:
+    """Calls ``malloc_trim(0)`` at most once per ``interval`` seconds.
+
+    :meth:`maybe_trim` is meant for an idle point of the render loop; it costs
+    one clock read when no trim is due. The first trim comes one interval
+    after construction, so start-up's allocations have settled.
+    """
+
+    def __init__(self, interval: float = TRIM_INTERVAL_SECONDS,
+                 clock: Callable[[], float] = time.monotonic) -> None:
+        self._interval = interval
+        self._clock = clock
+        self._libc = _load_libc()
+        self._next = clock() + interval
+
+    @property
+    def available(self) -> bool:
+        return self._libc is not None
+
+    def maybe_trim(self) -> bool:
+        """Trim if one is due. True when malloc_trim ran and released memory."""
+        if self._libc is None:
+            return False
+        now = self._clock()
+        if now < self._next:
+            return False
+        self._next = now + self._interval
+        try:
+            released = bool(self._libc.malloc_trim(0))
+        except Exception:  # pylint: disable=broad-except
+            logger.debug("malloc_trim failed; not trying again", exc_info=True)
+            self._libc = None
+            return False
+        logger.debug("malloc_trim(0) took %.1f ms, released=%s",
+                     (self._clock() - now) * 1000.0, released)
+        return released

test/test_malloc_tuning.py

@@ -0,0 +1,206 @@
+"""src/malloc_tuning.py: glibc arena cap and periodic malloc_trim, ctypes mocked."""
+import ctypes
+from pathlib import Path
+from unittest import mock
+
+import pytest
+
+from src import malloc_tuning as mt
+
+
+class FakeLibc:
+    """Stands in for ctypes.CDLL(None) on glibc: records calls."""
+
+    def __init__(self, trim_result=1, glibc=True):
+        self.trims = []
+        self.mallopts = []
+        self._trim_result = trim_result
+        if glibc:
+            self.gnu_get_libc_version = lambda: b'2.41'
+        self.malloc_trim = mock.Mock(side_effect=self._trim)
+        self.mallopt = mock.Mock(side_effect=self._mallopt)
+
+    def _trim(self, pad):
+        self.trims.append(pad)
+        if isinstance(self._trim_result, Exception):
+            raise self._trim_result
+        return self._trim_result
+
+    def _mallopt(self, param, value):
+        self.mallopts.append((param, value))
+        return 1
+
+
+@pytest.fixture(autouse=True)
+def fresh_libc(monkeypatch):
+    """Each test loads the C library itself; nothing real is called."""
+    monkeypatch.setattr(mt, '_libc', mt._UNLOADED)
+    monkeypatch.delenv('MALLOC_ARENA_MAX', raising=False)
+    yield
+
+
+def _on_glibc(monkeypatch, libc):
+    monkeypatch.setattr(mt.sys, 'platform', 'linux')
+    cdll = mock.Mock(return_value=libc)
+    monkeypatch.setattr(mt.ctypes, 'CDLL', cdll)
+    return cdll
+
+
+class Clock:
+    def __init__(self, t=1000.0):
+        self.t = t
+
+    def __call__(self):
+        return self.t
+
+
+# -- loading ----------------------------------------------------------------
+
+@pytest.mark.parametrize('platform', ['win32', 'darwin', 'freebsd14'])
+def test_not_linux_loads_nothing(monkeypatch, platform):
+    monkeypatch.setattr(mt.sys, 'platform', platform)
+    cdll = mock.Mock(side_effect=AssertionError('must not load'))
+    monkeypatch.setattr(mt.ctypes, 'CDLL', cdll)
+    assert mt._load_libc() is None
+    assert mt.cap_arenas() is False
+    trimmer = mt.MallocTrimmer(interval=0)
+    assert not trimmer.available
+    assert trimmer.maybe_trim() is False
+    cdll.assert_not_called()
+
+
+def test_linux_without_glibc_is_a_noop(monkeypatch):
+    """musl: no gnu_get_libc_version (and no malloc_trim) -- nothing is called."""
+    libc = FakeLibc(glibc=False)
+    del libc.malloc_trim
+    _on_glibc(monkeypatch, libc)
+    assert mt._load_libc() is None
+    assert mt.cap_arenas() is False
+    assert mt.MallocTrimmer(interval=0).maybe_trim() is False
+    assert libc.mallopts == []
+
+
+def test_cdll_failure_is_a_noop(monkeypatch):
+    monkeypatch.setattr(mt.sys, 'platform', 'linux')
+    monkeypatch.setattr(mt.ctypes, 'CDLL', mock.Mock(side_effect=OSError('no libc')))
+    assert mt._load_libc() is None
+    assert mt.cap_arenas() is False
+
+
+def test_loads_once(monkeypatch):
+    cdll = _on_glibc(monkeypatch, FakeLibc())
+    mt._load_libc()
+    mt._load_libc()
+    mt.MallocTrimmer()
+    assert cdll.call_count == 1
+
+
+def test_declares_c_signatures(monkeypatch):
+    libc = FakeLibc()
+    _on_glibc(monkeypatch, libc)
+    mt._load_libc()
+    assert libc.malloc_trim.argtypes == [ctypes.c_size_t]
+    assert libc.mallopt.argtypes == [ctypes.c_int, ctypes.c_int]
+
+
+# -- cap_arenas ---------------------------------------------------------------
+
+def test_cap_arenas_calls_mallopt(monkeypatch):
+    libc = FakeLibc()
+    _on_glibc(monkeypatch, libc)
+    assert mt.cap_arenas() is True
+    assert libc.mallopts == [(mt.M_ARENA_MAX, 2)]
+    assert mt.M_ARENA_MAX == -8  # glibc's malloc.h
+
+
+def test_cap_arenas_defers_to_the_environment(monkeypatch):
+    libc = FakeLibc()
+    _on_glibc(monkeypatch, libc)
+    monkeypatch.setenv('MALLOC_ARENA_MAX', '4')
+    assert mt.cap_arenas() is False
+    assert libc.mallopts == []
+
+
+def test_cap_arenas_swallows_errors(monkeypatch):
+    libc = FakeLibc()
+    libc.mallopt = mock.Mock(side_effect=RuntimeError('boom'))
+    _on_glibc(monkeypatch, libc)
+    assert mt.cap_arenas() is False
+
+
+def test_cap_arenas_matches_the_unit():
+    """The in-process default is the value the unit's MALLOC_ARENA_MAX carries."""
+    unit = (Path(__file__).resolve().parent.parent / 'systemd' / 'ledmatrix.service').read_text()
+    assert f'Environment=MALLOC_ARENA_MAX={mt.DEFAULT_ARENA_MAX}\n' in unit
+
+
+# -- MallocTrimmer ------------------------------------------------------------
+
+def test_trim_waits_one_interval_then_rate_limits(monkeypatch):
+    libc = FakeLibc()
+    _on_glibc(monkeypatch, libc)
+    clock = Clock()
+    trimmer = mt.MallocTrimmer(interval=300, clock=clock)
+    assert trimmer.available
+    assert trimmer.maybe_trim() is False          # start-up: not yet
+    clock.t += 299.9
+    assert trimmer.maybe_trim() is False
+    clock.t += 0.1
+    assert trimmer.maybe_trim() is True
+    assert libc.trims == [0]
+    clock.t += 100
+    assert trimmer.maybe_trim() is False          # rate-limited
+    clock.t += 200
+    assert trimmer.maybe_trim() is True
+    assert libc.trims == [0, 0]
+
+
+def test_trim_reports_nothing_released(monkeypatch):
+    libc = FakeLibc(trim_result=0)
+    _on_glibc(monkeypatch, libc)
+    clock = Clock()
+    trimmer = mt.MallocTrimmer(interval=10, clock=clock)
+    clock.t += 10
+    assert trimmer.maybe_trim() is False
+    assert libc.trims == [0]
+
+
+def test_trim_failure_disables_trimming(monkeypatch):
+    libc = FakeLibc(trim_result=RuntimeError('boom'))
+    _on_glibc(monkeypatch, libc)
+    clock = Clock()
+    trimmer = mt.MallocTrimmer(interval=10, clock=clock)
+    clock.t += 10
+    assert trimmer.maybe_trim() is False
+    clock.t += 10
+    assert trimmer.maybe_trim() is False
+    assert libc.trims == [0]                      # not retried
+    assert not trimmer.available
+
+
+# -- wiring -------------------------------------------------------------------
+
+def test_run_py_caps_arenas_before_threads():
+    """run.py applies the cap before the watchdog or the controller import."""
+    src = (Path(__file__).resolve().parent.parent / 'run.py').read_text()
+    cap = src.index('malloc_tuning.cap_arenas()')
+    assert cap < src.index('display_watchdog.watchdog.begin_startup()')
+    assert cap < src.index('from src.display_controller import main')
+
+
+def test_render_loop_trims_between_screens():
+    src = (Path(__file__).resolve().parent.parent / 'src' / 'display_controller.py').read_text()
+    loop = src.index('display_watchdog.watchdog.loop_pass()')
+    trim = src.index('trimmer.maybe_trim()')
+    assert loop < trim < src.index('outcome = runner.run(plan, manager_to_display)')
+
+
+@pytest.mark.skipif(not mt.sys.platform.startswith('linux'), reason='glibc only')
+def test_real_libc_on_linux():
+    """On a real Linux C library the calls go through without raising."""
+    if mt._load_libc() is None:
+        pytest.skip('not glibc')
+    trimmer = mt.MallocTrimmer(interval=0)
+    assert trimmer.available
+    assert trimmer.maybe_trim() in (True, False)
+    assert trimmer.available  # did not fail and disable itself