From 446207ffbc8e6dce00424557c2227c2f1a74e5fb Mon Sep 17 00:00:00 2001 From: ChuckBuilds Date: Thu, 20 Aug 2026 00:17:27 -0400 Subject: [PATCH] perf(systemd): cap glibc malloc arenas on the display service Measured on a live rig 2.5 hours after start: RSS 1030 MB Private_Dirty 988 MB anonymous mappings > 10 MB 23 largest 104, 79, 66, 63, 63 MB, on 64 MB-aligned addresses threads 9 cores 3 -> glibc ceiling = 8 x 3 = 24 arenas 23 against a ceiling of 24, all 64 MB-aligned: these are glibc's per-thread malloc arenas, not live objects. The data the process was actually holding accounts for perhaps 15 MB -- the widest scroll strip observed was 35,746 x 64, about 7 MB as RGB and the same again for its numpy mirror. It is bloat rather than a leak: sampled four times over 135 seconds, RSS sat between 990 and 1030 MB rather than climbing. glibc gives each allocating thread its own arena, grows them to hold peak demand, and never gives them back. A process that builds and drops large images across several threads is exactly the shape that produces this. The device had 59 MB free at the time, on 1845 MB total. MALLOC_ARENA_MAX=2 trades a little allocator concurrency for that resident memory. It is a tuning knob rather than a fix for a defect, so the rationale and the measurements sit next to it in the unit file, and a test asserts they stay there -- a bare environment variable invites removal by whoever meets it next. Two things this is NOT, both checked rather than assumed: - Not an OOM problem today. A grep for "oom" in the service journal returned 24 matches, all of which were the radar logging zoom=9 and zoom=7. The kernel OOM killer has not fired: dmesg has zero matches. - Not currently capped by the unit's MemoryMax=85% either. That directive is in this file but absent from the unit actually installed on the rig, which reports MemoryMax=infinity, so nothing is enforcing a ceiling there. The saving is unmeasured on hardware: applying it needs a service restart, which blanks the panel, so that is the user's call rather than something to do mid-audit. If p99 frame time regresses -- it sits at 18.4 ms against a 16.7 ms budget for 60 FPS, so there is not much headroom -- raise the value rather than remove it. --- systemd/ledmatrix.service | 12 +++++ test/test_systemd_malloc_arenas.py | 83 ++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 test/test_systemd_malloc_arenas.py diff --git a/systemd/ledmatrix.service b/systemd/ledmatrix.service index d5f064b6..ab5ddf4d 100644 --- a/systemd/ledmatrix.service +++ b/systemd/ledmatrix.service @@ -8,6 +8,18 @@ Type=simple User=root WorkingDirectory=__PROJECT_ROOT_DIR__ Environment=PYTHONDONTWRITEBYTECODE=1 +# glibc gives each allocating thread its own malloc arena, up to 8 x CPU count, +# and an arena that has grown is never handed back to the OS. This process runs +# 9 threads on a 3-core Pi, so the ceiling is 24 arenas -- and a rig measured at +# 1030 MB resident held 23 large anonymous mappings on 64 MB-aligned addresses, +# 920 MB of them, while the live data it was actually holding (widest scroll +# strip seen: 35,746 x 64) accounts for roughly 15 MB. That gap is arena bloat, +# not leaked objects: RSS was flat across repeated sampling, not climbing. +# +# Capping the arenas trades a little allocator concurrency for a large amount of +# resident memory on a device that has neither to spare. 2 is the usual value; +# raise it if frame times regress. +Environment=MALLOC_ARENA_MAX=2 ExecStart=/usr/bin/python3 __PROJECT_ROOT_DIR__/run.py # Restart=always, not on-failure: run.py exiting 0 (a clean shutdown path taken # for a reason that no longer applies, e.g. a config reload) would otherwise leave diff --git a/test/test_systemd_malloc_arenas.py b/test/test_systemd_malloc_arenas.py new file mode 100644 index 00000000..512154d0 --- /dev/null +++ b/test/test_systemd_malloc_arenas.py @@ -0,0 +1,83 @@ +"""The display unit must cap glibc's malloc arenas. + +glibc hands each allocating thread its own malloc arena, up to 8 x CPU count, +and an arena that has grown is never returned to the OS. This process runs +threads for the render loop, the update workers and the background fetchers, so +on a 3-core Pi the ceiling is 24 arenas. + +Measured on a live rig, 2.5 hours in: + + RSS 1030 MB + Private_Dirty 988 MB + anonymous mappings > 10 MB 23 (ceiling is 8 x 3 = 24) + largest few 104, 79, 66, 63, 63 MB, on 64 MB-aligned addresses + +against live data that accounts for perhaps 15 MB -- the widest scroll strip +observed was 35,746 x 64, about 7 MB as RGB and the same again for its numpy +mirror. Repeated sampling showed RSS flat between 990 and 1030 MB rather than +climbing, so this is arena bloat rather than a leak: memory Python has freed +but glibc is holding per-arena. + +The device had 59 MB free at the time. + +Capping the arena count trades a little allocator concurrency for that resident +memory. The render loop is latency-sensitive, so if p99 frame time regresses the +right response is to raise this rather than remove it. +""" +import re +from pathlib import Path + +import pytest + +UNIT = (Path(__file__).resolve().parent.parent / "systemd" / "ledmatrix.service") + + +def _environment(unit_text): + return dict( + line.split("=", 2)[1:3] if line.count("=") >= 2 else (line.split("=", 1)[1], "") + for line in unit_text.splitlines() + if line.startswith("Environment=") + ) + + +def test_the_unit_exists(): + assert UNIT.is_file(), f"{UNIT} is missing" + + +def test_malloc_arena_max_is_capped(): + env = _environment(UNIT.read_text(encoding="utf-8")) + assert "MALLOC_ARENA_MAX" in env, ( + "the display unit does not cap glibc arenas; on a 3-core Pi the default " + "ceiling is 24 and a measured rig held 23 of them, 920 MB" + ) + value = int(env["MALLOC_ARENA_MAX"]) + assert 1 <= value <= 4, ( + f"MALLOC_ARENA_MAX={value} is outside the useful range: 1-4 keeps the " + "resident saving, and anything larger gives most of it back" + ) + + +def test_the_reason_is_recorded_next_to_it(): + """A bare tuning knob invites removal by whoever meets it next.""" + text = UNIT.read_text(encoding="utf-8") + index = text.index("Environment=MALLOC_ARENA_MAX") + preamble = text[:index].splitlines()[-12:] + comment = "\n".join(line for line in preamble if line.startswith("#")) + assert "arena" in comment.lower(), "no explanation precedes the setting" + assert re.search(r"\d", comment), ( + "the explanation cites no measurement, so a reader cannot tell whether " + "it still applies to their hardware" + ) + + +@pytest.mark.parametrize("unit", ["ledmatrix.service"]) +def test_the_unit_still_parses_as_ini(unit): + """systemd will refuse a malformed unit, and the panel stays dark.""" + import configparser + + path = UNIT.parent / unit + parser = configparser.ConfigParser(strict=False) + # systemd allows repeated keys; ConfigParser needs them merged, not rejected. + parser.read_string(path.read_text(encoding="utf-8")) + assert parser.has_section("Service") + assert parser.has_option("Service", "ExecStart")