From e6249dcc7e835dcb3377ed71340f2fc4c5d1bfdc Mon Sep 17 00:00:00 2001
From: Chuck <33324927+ChuckBuilds@users.noreply.github.com>
Date: Tue, 18 Aug 2026 19:04:57 -0400
Subject: [PATCH] fix(service): survive corrupt health cache and clean exits
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Three independent failure modes that each end with a dark panel and no
automatic recovery.

1. PluginHealthTracker._load_health_state returned the cached value
   verbatim. If that value is not a dict, every caller raises
   AttributeError: 'list' object has no attribute 'get' — during
   DisplayController.__init__, so the process dies before the display
   loop starts. systemd restarts it, the same bad entry is read back
   from disk, and it dies again: an unattended restart loop that
   survives reboots because the cause is persisted. Observed in the
   field with plugin_health:<id> holding an unrelated plugin's list
   payload. Now non-dict entries are discarded with a warning and the
   defaults are rebuilt.

2. ledmatrix.service used Restart=on-failure, so any exit with status 0
   left the unit stopped and the panel dark indefinitely — systemd
   treats it as success and never brings it back. Restart=always.

3. ledmatrix-wifi-monitor.service used StandardOutput=syslog, which
   systemd has marked obsolete; it warns and rewrites it to journal on
   every load.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/plugin_system/plugin_health.py     | 13 ++++++++++++-
 systemd/ledmatrix-wifi-monitor.service |  4 ++--
 systemd/ledmatrix.service              |  6 +++++-
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/src/plugin_system/plugin_health.py b/src/plugin_system/plugin_health.py
index ecbec076..71a05ea3 100644
--- a/src/plugin_system/plugin_health.py
+++ b/src/plugin_system/plugin_health.py
@@ -64,8 +64,19 @@ class PluginHealthTracker:
             cache_key, max_age=None, memory_ttl=0 if force_reload else None
         )
 
-        if cached:
+        if isinstance(cached, dict) and cached:
             return cached
+
+        # A cache entry that is not a dict means the persisted state was written
+        # by something other than _save_health_state (a key collision, a partial
+        # write, a restored backup). Returning it verbatim makes every caller
+        # blow up on .get(), which takes the display down in a restart loop that
+        # survives reboots because the bad entry is on disk. Discard and rebuild.
+        if cached is not None and not isinstance(cached, dict):
+            self.logger.warning(
+                f"Discarding malformed health state for {plugin_id}: expected "
+                f"dict, got {type(cached).__name__}. Falling back to defaults."
+            )
         
         # Default state
         return {
diff --git a/systemd/ledmatrix-wifi-monitor.service b/systemd/ledmatrix-wifi-monitor.service
index 2f08fde0..46b69bfe 100644
--- a/systemd/ledmatrix-wifi-monitor.service
+++ b/systemd/ledmatrix-wifi-monitor.service
@@ -10,8 +10,8 @@ WorkingDirectory=__PROJECT_ROOT_DIR__
 ExecStart=/usr/bin/python3 __PROJECT_ROOT_DIR__/scripts/utils/wifi_monitor_daemon.py --interval 30
 Restart=on-failure
 RestartSec=10
-StandardOutput=syslog
-StandardError=syslog
+StandardOutput=journal
+StandardError=journal
 SyslogIdentifier=ledmatrix-wifi-monitor
 
 [Install]
diff --git a/systemd/ledmatrix.service b/systemd/ledmatrix.service
index f3bc9e28..d2aba454 100644
--- a/systemd/ledmatrix.service
+++ b/systemd/ledmatrix.service
@@ -9,7 +9,11 @@ User=root
 WorkingDirectory=__PROJECT_ROOT_DIR__
 Environment=PYTHONDONTWRITEBYTECODE=1
 ExecStart=/usr/bin/python3 __PROJECT_ROOT_DIR__/run.py
-Restart=on-failure
+# Restart=always, not on-failure: run.py exiting 0 (a clean shutdown path taken
+# for a reason that no longer applies, e.g. a config reload) would otherwise leave
+# the service stopped and the panel dark indefinitely, with systemd considering
+# that a successful outcome and never bringing it back.
+Restart=always
 RestartSec=10
 StandardOutput=journal
 StandardError=journal