Merge branch 'perf/runtime-observability' of https://github.com/ChuckBuilds/LEDMatrix into consolidate/perf

test/test_health_write_churn.py

@@ -0,0 +1,110 @@
+"""A healthy plugin must not rewrite its health record every cycle.
+
+Every successful plugin update called record_success(), which persisted the
+record unconditionally. In steady state the only fields that had changed were
+total_successes and last_success_time -- a counter and a timestamp that
+health_monitor reads for display and that nothing reads back after a restart.
+
+Measured on a rig running 24 plugins: about 17 health-file rewrites a minute,
+roughly 25,000 a day. Each is ~400 bytes, but they land on an SD card where
+the unit of cost is an erase-block cycle, not the byte count, and where wear is
+what eventually kills the card.
+
+The circuit breaker still needs its own state to survive a restart, so the
+write is kept for exactly the fields it is rebuilt from -- and a failure, a
+circuit opening, or a recovery must still be written the moment it happens.
+"""
+import time
+
+import pytest
+
+from src.plugin_system.plugin_health import PluginHealthTracker, CircuitState
+
+
+class _Cache:
+    """Counts writes; serves back whatever was last written."""
+
+    def __init__(self):
+        self.store = {}
+        self.writes = 0
+
+    def set(self, key, data, ttl=None, **kwargs):
+        self.writes += 1
+        self.store[key] = data
+
+    def get(self, key, max_age=None, memory_ttl=None, **kwargs):
+        return self.store.get(key)
+
+
+@pytest.fixture
+def tracker():
+    cache = _Cache()
+    t = PluginHealthTracker(cache_manager=cache)
+    return t, cache
+
+
+def test_steady_state_success_stops_writing(tracker):
+    """The regression: 100 healthy cycles used to be 100 SD writes."""
+    t, cache = tracker
+    t.record_success("weather")
+    first = cache.writes
+    for _ in range(100):
+        t.record_success("weather")
+    assert cache.writes == first, (
+        f"{cache.writes - first} redundant writes across 100 healthy cycles"
+    )
+
+
+def test_the_counters_are_still_accurate_in_memory(tracker):
+    """Skipping the write must not skip the bookkeeping."""
+    t, _ = tracker
+    for _ in range(10):
+        t.record_success("weather")
+    state = t.get_health_state("weather")
+    assert state["total_successes"] == 10
+    assert state["last_success_time"] is not None
+    assert state["last_success_time"] <= time.time()
+
+
+def test_a_failure_is_written_immediately(tracker):
+    t, cache = tracker
+    t.record_success("weather")
+    before = cache.writes
+    t.record_failure("weather", RuntimeError("boom"))
+    assert cache.writes > before, "a failure must reach disk"
+
+
+def test_recovery_after_failure_is_written(tracker):
+    """consecutive_failures returning to 0 is durable state changing."""
+    t, cache = tracker
+    t.record_failure("weather", RuntimeError("boom"))
+    before = cache.writes
+    t.record_success("weather")
+    assert cache.writes > before, "recovery must reach disk"
+    assert t.get_health_state("weather")["consecutive_failures"] == 0
+
+
+def test_a_closing_circuit_is_written(tracker):
+    """Success in half-open closes the circuit -- that must survive a restart."""
+    t, cache = tracker
+    state = t.get_health_state("weather")
+    state["circuit_state"] = CircuitState.HALF_OPEN.value
+    state["half_open_start_time"] = time.time()
+    before = cache.writes
+    t.record_success("weather")
+    assert cache.writes > before, "a circuit transition must reach disk"
+    assert t.get_health_state("weather")["circuit_state"] == CircuitState.CLOSED.value
+
+
+def test_durable_state_survives_a_restart(tracker):
+    """What is skipped must genuinely not matter to the breaker."""
+    t, cache = tracker
+    for _ in range(3):
+        t.record_failure("weather", RuntimeError("boom"))
+    for _ in range(50):
+        t.record_success("weather")
+
+    revived = PluginHealthTracker(cache_manager=cache)
+    state = revived.get_health_state("weather")
+    assert state["consecutive_failures"] == 0
+    assert state["circuit_state"] == CircuitState.CLOSED.value

test/test_journald_log_priority.py

@@ -0,0 +1,101 @@
+"""Log lines must reach the journal with their real severity.
+
+Everything this process writes to stdout lands in the journal as PRIORITY=6,
+whatever the Python level was, because journald has no other signal. Measured
+on a live rig over 24 hours: 55 lines containing " - ERROR - " and 13
+containing " - WARNING - ", every one of them recorded as informational. So
+
+    journalctl -p err -u ledmatrix
+
+returned nothing while errors were being logged, and anyone triaging has to
+grep the message text instead. That is slower and it is wrong: a search for
+"oom" also matches the radar logging "zoom=9", which is exactly the false
+positive it produced during this audit.
+
+systemd reads a leading "<N>" on each stdout line and uses it as the priority
+(sd-daemon(3)), so this needs no extra dependency -- and it must only be
+applied when systemd is actually reading, or the prefixes become literal noise
+in a terminal, the emulator, and test output.
+"""
+import logging
+import os
+from unittest.mock import patch
+
+import pytest
+
+from src.logging_config import JournalPriorityFormatter, _SYSLOG_PRIORITY, _under_systemd
+
+
+class _Plain(logging.Formatter):
+    def format(self, record):
+        return record.getMessage()
+
+
+def _record(level, msg="hello"):
+    return logging.LogRecord("t", level, "f.py", 1, msg, None, None)
+
+
+@pytest.mark.parametrize("level,expected", [
+    (logging.CRITICAL, 2),
+    (logging.ERROR, 3),
+    (logging.WARNING, 4),
+    (logging.INFO, 6),
+    (logging.DEBUG, 7),
+])
+def test_each_level_maps_to_its_syslog_priority(level, expected):
+    out = JournalPriorityFormatter(_Plain()).format(_record(level))
+    assert out.startswith(f"<{expected}>"), out
+    assert _SYSLOG_PRIORITY[level] == expected
+
+
+def test_error_and_info_are_distinguishable():
+    """The whole point: journalctl -p err must be able to tell them apart."""
+    fmt = JournalPriorityFormatter(_Plain())
+    assert fmt.format(_record(logging.ERROR))[:3] != fmt.format(_record(logging.INFO))[:3]
+
+
+def test_every_line_of_a_multiline_record_is_tagged():
+    """The journal splits them, and an untagged continuation loses its level.
+
+    A traceback is the case that matters -- it is the most important thing in
+    the log and the longest.
+    """
+    out = JournalPriorityFormatter(_Plain()).format(
+        _record(logging.ERROR, "Traceback:\nline one\nline two"))
+    lines = out.split("\n")
+    assert len(lines) == 3
+    assert all(line.startswith("<3>") for line in lines), lines
+
+
+def test_the_message_survives_intact():
+    out = JournalPriorityFormatter(_Plain()).format(_record(logging.WARNING, "disk full"))
+    assert out == "<4>disk full"
+
+
+def test_an_unknown_level_falls_back_to_info():
+    out = JournalPriorityFormatter(_Plain()).format(_record(25))
+    assert out.startswith("<6>")
+
+
+def test_prefixing_is_off_outside_systemd():
+    """Otherwise a terminal run, the emulator and pytest all show `<6>`."""
+    with patch.dict(os.environ, {}, clear=True):
+        assert not _under_systemd()
+    with patch.dict(os.environ, {"JOURNAL_STREAM": "8:12345"}):
+        assert _under_systemd()
+
+
+def test_setup_uses_the_wrapper_only_under_systemd():
+    from src.logging_config import setup_logging
+
+    for env, expect_wrapped in (({}, False), ({"JOURNAL_STREAM": "8:1"}, True)):
+        with patch.dict(os.environ, env, clear=True):
+            setup_logging()
+            handlers = [h for h in logging.getLogger().handlers
+                        if isinstance(h, logging.StreamHandler)]
+            assert handlers, "no stream handler installed"
+            wrapped = any(isinstance(h.formatter, JournalPriorityFormatter)
+                          for h in handlers)
+            assert wrapped is expect_wrapped, (
+                f"JOURNAL_STREAM={env}: wrapped={wrapped}, expected {expect_wrapped}")
+    logging.getLogger().handlers.clear()

test/test_logging_config.py

@@ -183,15 +183,27 @@ class TestSetupLogging:
         setup_logging()
         assert len(logging.getLogger().handlers) == 1
 
+    @staticmethod
+    def _selected_formatter():
+        """The formatter setup_logging() chose, past any journald wrapper.
+
+        Under systemd the console handler's formatter is wrapped so each line
+        carries its syslog priority. That wrapper is applied only when
+        JOURNAL_STREAM is set, which is true in CI and false in a terminal, so
+        asserting on the handler's formatter directly passes locally and fails
+        on the runner. These tests are about which formatter format_type
+        selects, so they look through the wrapper.
+        """
+        formatter = logging.getLogger().handlers[0].formatter
+        return getattr(formatter, "inner", formatter)
+
     def test_json_format_selects_structured_formatter(self):
         setup_logging(format_type="json")
-        assert isinstance(
-            logging.getLogger().handlers[0].formatter, StructuredFormatter)
+        assert isinstance(self._selected_formatter(), StructuredFormatter)
 
     def test_readable_format_selects_contextual_formatter(self):
         setup_logging(format_type="readable")
-        assert isinstance(
-            logging.getLogger().handlers[0].formatter, ContextualFormatter)
+        assert isinstance(self._selected_formatter(), ContextualFormatter)
 
     def test_log_file_adds_file_handler(self, tmp_path):
         log_file = tmp_path / "test.log"

test/test_vegas_log_volume.py

@@ -0,0 +1,63 @@
+"""The Vegas content path must trace at DEBUG, not INFO.
+
+plugin_adapter narrates every step of acquiring content from every plugin --
+"Has get_vegas_content", "Native: calling get_vegas_content()", "Native content
+returned None", "Has scroll_helper", the per-item sizes -- and it does that for
+each plugin on each cycle.
+
+Measured on a live rig: 13,408 log lines an hour, of which 13,366 were INFO and
+35 were WARNING. plugin_adapter alone produced 2,457 of them. That is ~223
+lines a minute of string formatting on a Pi that is also driving the panel, all
+of it written through journald to the SD card, and it buries the 35 lines that
+actually indicate a problem.
+
+Nothing is lost by moving it to DEBUG: the 19 warning/error/exception calls in
+the module are untouched, so real failures still surface at their own level.
+
+One INFO call is deliberate and stays -- the padding-strip message chooses its
+level at runtime (`logger.warning if (left and right) else logger.info`) and
+test_vegas_plugin_adapter.py pins it.
+"""
+import ast
+from pathlib import Path
+
+import pytest
+
+ADAPTER = (Path(__file__).resolve().parent.parent / "src" / "vegas_mode"
+           / "plugin_adapter.py")
+
+
+def _info_calls(path):
+    """Direct logger.info(...) call sites in a module."""
+    tree = ast.parse(path.read_text(encoding="utf-8"))
+    found = []
+    for node in ast.walk(tree):
+        if (isinstance(node, ast.Call)
+                and isinstance(node.func, ast.Attribute)
+                and node.func.attr == "info"
+                and getattr(node.func.value, "id", None) == "logger"):
+            found.append(node.lineno)
+    return found
+
+
+def test_the_content_path_does_not_trace_at_info():
+    calls = _info_calls(ADAPTER)
+    assert not calls, (
+        "plugin_adapter should trace at DEBUG; found logger.info at lines "
+        f"{calls}. This path runs per plugin per cycle and its output goes to "
+        "the SD card via journald."
+    )
+
+
+def test_real_failures_still_have_a_level_of_their_own():
+    """Demoting the trace must not have swept up the error reporting."""
+    source = ADAPTER.read_text(encoding="utf-8")
+    loud = sum(source.count(f"logger.{level}(")
+               for level in ("warning", "error", "exception"))
+    assert loud >= 15, f"only {loud} warning/error/exception calls remain"
+
+
+def test_the_deliberate_runtime_chosen_level_survives():
+    """The padding-strip message picks its level at runtime; leave it alone."""
+    source = ADAPTER.read_text(encoding="utf-8")
+    assert "logger.warning if (left and right) else logger.info" in source