fix(vegas): let the watchdog see stalls that hold the GIL

The watchdog only noticed a late heartbeat, which a whole class of
freeze can never produce: if the loop is inside one long C call that
holds the GIL, this thread cannot run during the stall, and by the time
it does the loop has already checked in. On the dev rig that hid a
recurring 3.2s freeze completely -- twenty minutes of watching produced
one dump, for an unrelated 0.4s stall.

What it can still observe is that its own sleep ran long. A badly
overshot wait is now reported as a stall in its own right. The stacks
are stale by then and the message says so, but knowing the freeze is
GIL-holding is most of the diagnosis: it rules out lock contention and
scheduling, and points at a single long C call.

This also explains why lowering sys.setswitchinterval changed nothing --
the switch interval cannot preempt a C call that never releases the GIL.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Udr6MfaFLUPhX5Fgo67Jf5

src/cache/disk_cache.py

@@ -112,6 +112,22 @@ class DiskCache:
                     record_ts = None
             
             now = time.time()
+
+            # An explicit per-entry ttl wins over the caller's max_age. The
+            # caller that wrote the record knows what its data is; max_age is
+            # inferred from substrings in the key ("live", "odds", "stock") and
+            # is only a fallback for records that never said. Until now the ttl
+            # was stored and ignored, so `set(key, data, ttl=...)` did nothing
+            # at all -- 48 plugin call sites and 4 in the core were writing a
+            # number no read path consulted.
+            effective_max_age = max_age
+            if isinstance(record, dict):
+                stored_ttl = record.get('ttl')
+                if isinstance(stored_ttl, (int, float)) and not isinstance(stored_ttl, bool) \
+                        and stored_ttl >= 0:
+                    effective_max_age = stored_ttl
+            max_age = effective_max_age
+
             # max_age=None means "never expires" (mirrors MemoryCache and the
             # cache_manager docstring). Guard it explicitly — otherwise the
             # comparison below raises TypeError and the record is treated as a

src/cache/memory_cache.py

@@ -57,6 +57,16 @@ class MemoryCache:
             if timestamp is None:
                 return None
             
+            # An explicit per-entry ttl wins over the caller's max_age, matching
+            # DiskCache. max_age is inferred from substrings in the key and is
+            # only a fallback for records that did not say what they wanted.
+            record = self._cache[key]
+            if isinstance(record, dict):
+                stored_ttl = record.get('ttl')
+                if isinstance(stored_ttl, (int, float)) and not isinstance(stored_ttl, bool) \
+                        and stored_ttl >= 0:
+                    max_age = stored_ttl
+
             # Check expiration
             if max_age is not None and (now - timestamp) > max_age:
                 # Expired - remove it

src/cache_manager.py

@@ -594,8 +594,10 @@ class CacheManager:
         Args:
             key: Cache key
             data: Data to cache
-            ttl: Optional time-to-live in seconds (stored for compatibility but
+            ttl: Time-to-live in seconds for this entry. Takes precedence over
-                 expiration is still controlled via max_age when reading)
+                 the max_age a reader would otherwise apply, which is inferred
+                 from the key and is only a fallback for entries that did not
+                 say. Omit it to keep that inferred behaviour.
         """
         cache_data = {
             'data': data,

src/vegas_mode/coordinator.py

@@ -12,8 +12,11 @@ Supports three display modes per plugin:
 """
 
 import logging
-import time
+import os
+import sys
 import threading
+import time
+import traceback
 from typing import Optional, Dict, Any, List, Callable, TYPE_CHECKING
 
 from src.vegas_mode.config import VegasModeConfig
@@ -29,6 +32,81 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+# A frame is a "hitch" once it takes this many times the typical frame. Two
+# is deliberately forgiving: one dropped frame at 120fps is 8ms and invisible,
+# whereas a marquee moving a steady few pixels per frame shows a stall of
+# twice that as a visible jerk.
+_HITCH_FACTOR = 2.0
+
+# How many recent frames define "typical". Big enough to ride out noise, small
+# enough to track a genuine change in what the loop costs.
+_TYPICAL_SAMPLE = 60
+
+# A stall long enough that a viewer sees the marquee stop dead. Frame-time
+# statistics say one happened but not what did it, and by the time the numbers
+# are logged the stack is long gone -- so a watchdog samples every thread while
+# the loop is still wedged. Off unless LEDMATRIX_STALL_WATCHDOG is set, since
+# it dumps a lot of text.
+_STALL_DUMP_SECONDS = float(os.environ.get('LEDMATRIX_STALL_WATCHDOG', '0') or 0)
+
+
+class _StallWatchdog:
+    """Dumps every thread's stack when the render loop stops checking in."""
+
+    def __init__(self, threshold: float):
+        self.threshold = threshold
+        self._beat = time.time()
+        self._lock = threading.Lock()
+        self._stop = threading.Event()
+        self._dumped_for = 0.0
+        self._thread = threading.Thread(
+            target=self._watch, name="VegasStallWatchdog", daemon=True)
+        self._thread.start()
+
+    def beat(self) -> None:
+        with self._lock:
+            self._beat = time.time()
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    def _watch(self) -> None:
+        poll = self.threshold / 4.0
+        while True:
+            woke_at = time.time()
+            if self._stop.wait(poll):
+                break
+            with self._lock:
+                last = self._beat
+            now = time.time()
+            stalled = now - last
+            # A stall inside a C call that holds the GIL never shows up as a
+            # late beat: this thread cannot run during it, and by the time it
+            # does the loop has already checked in. What it can see is that
+            # its own sleep ran long. Treat a badly overshot wait as a stall
+            # in its own right -- the stacks are stale by then, but knowing
+            # the freeze is GIL-holding is itself the diagnosis.
+            overshoot = (now - woke_at) - poll
+            if overshoot > self.threshold:
+                logger.warning(
+                    "render loop stalled %.2fs holding the GIL -- no Python "
+                    "frames ran, so the stacks below are from after it ended; "
+                    "look for one long C call (a large PIL operation, a "
+                    "compress, a big allocation)", overshoot)
+                stalled = overshoot
+            elif stalled < self.threshold or last == self._dumped_for:
+                continue
+            self._dumped_for = last          # one dump per stall, not per poll
+            frames = sys._current_frames()
+            names = {t.ident: t.name for t in threading.enumerate()}
+            lines = ["render loop stalled %.2fs -- thread stacks:" % stalled]
+            for ident, frame in frames.items():
+                lines.append("  --- %s (%s) ---" % (names.get(ident, "?"), ident))
+                for fn, lineno, func, _text in traceback.extract_stack(frame)[-8:]:
+                    lines.append("    %s:%d in %s" % (fn, lineno, func))
+            logger.warning("\n".join(lines))
+
+
 
 class VegasModeCoordinator:
     """
@@ -119,6 +197,7 @@ class VegasModeCoordinator:
             'static_pauses': 0,
         }
         self._start_time: Optional[float] = None
+        self._stall_watchdog: Optional['_StallWatchdog'] = None
 
         logger.info(
             "VegasModeCoordinator initialized: enabled=%s, fps=%d, buffer_ahead=%d",
@@ -382,6 +461,20 @@ class VegasModeCoordinator:
         fps_log_interval = 5.0  # Log FPS every 5 seconds
         last_fps_log_time = start_time
         fps_frame_count = 0
+        # Stutter is invisible in a mean. At 120fps a 5s window covers ~600
+        # frames, so a 200ms freeze -- plainly visible on a marquee -- moves
+        # the average from 120.0 to 115.4 and reads as healthy. What a viewer
+        # notices is the worst frame, so track that separately.
+        frame_worst = 0.0
+        frame_hitches = 0
+        frame_times: List[float] = []
+        frame_typical = 0.0
+
+        # One per coordinator, not per iteration -- run_iteration is called
+        # repeatedly, so building one here would leak a thread each time.
+        if _STALL_DUMP_SECONDS > 0 and self._stall_watchdog is None:
+            self._stall_watchdog = _StallWatchdog(_STALL_DUMP_SECONDS)
+        watchdog = self._stall_watchdog
 
         logger.info("Starting Vegas iteration for %.1fs", duration)
 
@@ -417,6 +510,25 @@ class VegasModeCoordinator:
             frame_elapsed = time.time() - frame_started
             time.sleep(max(0.0, frame_interval - frame_elapsed))
 
+            # Measured before the sleep, so this is time spent working rather
+            # than time spent pacing. A frame that overruns the budget is one
+            # the viewer sees as a jerk in otherwise smooth motion.
+            if frame_elapsed > frame_worst:
+                frame_worst = frame_elapsed
+            # Measured against what frames actually cost here, not against
+            # the configured target. The target is routinely set above what
+            # the panel can hold so vsync does the pacing -- against that
+            # budget every ordinary frame looks like a hitch, which is how
+            # the first version of this counter reported 250 per window on a
+            # display that was running perfectly smoothly.
+            if frame_typical and frame_elapsed > _HITCH_FACTOR * frame_typical:
+                frame_hitches += 1
+            if len(frame_times) >= _TYPICAL_SAMPLE:
+                frame_typical = sorted(frame_times[-_TYPICAL_SAMPLE:])[_TYPICAL_SAMPLE // 2]
+            frame_times.append(frame_elapsed)
+            if watchdog:
+                watchdog.beat()
+
             # Increment frame count and check for interrupt periodically
             frame_count += 1
             fps_frame_count += 1
@@ -425,12 +537,22 @@ class VegasModeCoordinator:
             current_time = time.time()
             if current_time - last_fps_log_time >= fps_log_interval:
                 fps = fps_frame_count / (current_time - last_fps_log_time)
+                p99 = 0.0
+                if frame_times:
+                    ordered = sorted(frame_times)
+                    p99 = ordered[min(len(ordered) - 1,
+                                      int(len(ordered) * 0.99))]
                 logger.info(
-                    "Vegas FPS: %.1f (target: %d, frames: %d)",
+                    "Vegas FPS: %.1f (target: %d, frames: %d) "
-                    fps, self.vegas_config.target_fps, fps_frame_count
+                    "p99 %.1fms worst %.1fms hitches %d",
+                    fps, self.vegas_config.target_fps, fps_frame_count,
+                    p99 * 1000.0, frame_worst * 1000.0, frame_hitches
                 )
                 last_fps_log_time = current_time
                 fps_frame_count = 0
+                frame_worst = 0.0
+                frame_hitches = 0
+                frame_times.clear()
 
             if (self._interrupt_check and
                     frame_count % self._interrupt_check_interval == 0):

src/web_interface/error_handler.py

@@ -4,6 +4,7 @@ Centralized error handling for web interface.
 Provides helpers for consistent error responses across API endpoints.
 """
 
+import re
 from typing import Any, Optional
 from flask import jsonify
 
@@ -16,6 +17,78 @@ from src.logging_config import get_logger
 logger = get_logger(__name__)
 
 
+# Credentials that turn up inside exception text. A requests error quotes the
+# URL it failed on, and plugins that authenticate by query string put their key
+# there, so echoing an exception verbatim can hand out an API key. Redact the
+# value, keep the parameter name -- knowing *which* credential was involved is
+# part of the diagnosis.
+_REDACT_CREDENTIAL = re.compile(
+    r'((?:api[_-]?key|access[_-]?token|auth|apikey|key|passwd|password|pwd|'
+    r'secret|sig|signature|token)["\']?\s*[=:]\s*["\']?)([^\s&"\'<>,}]+)',
+    re.IGNORECASE,
+)
+
+# `Authorization: <scheme> <credential>`. The scheme name is kept because it
+# says which kind of credential failed; the credential goes. Any scheme
+# matches, not a fixed list: ApiKey, Negotiate, NTLM, AWS4-HMAC-SHA256 and
+# whatever a plugin's API invents next are all credentials, and a list would
+# silently leak the ones nobody thought of. Not covered by the generic pattern
+# above, whose value part stops at whitespace and so would keep the credential
+# once a space follows the scheme.
+_REDACT_AUTH_HEADER = re.compile(
+    r'((?:proxy-)?authorization["\']?\s*[=:]\s*["\']?\s*'
+    r'(?:[A-Za-z][\w.+-]*[ \t]+)?)'          # optional scheme name, kept
+    r'([^\s,"\'<>}]+)',                       # the credential, redacted
+    re.IGNORECASE,
+)
+
+# Credentials embedded in a URL: https://user:password@host. requests quotes
+# the full URL in its exceptions, so this is a realistic leak. The username is
+# kept -- it identifies which account failed without being the secret.
+_REDACT_URL_USERINFO = re.compile(r'([a-z][a-z0-9+.-]*://[^/\s:@]+:)([^/\s@]+)(@)',
+                                  re.IGNORECASE)
+
+# Long enough for an errno string with a path, short enough not to dump a
+# parser's worth of context into a JSON field.
+_MAX_DETAIL_LENGTH = 400
+
+
+def describe_exception(exc: BaseException,
+                       max_length: int = _MAX_DETAIL_LENGTH) -> str:
+    """
+    One-line, safe-to-return description of an exception.
+
+    The generic "an error occurred; see logs for details" tells a user nothing
+    and, when the failure is bad enough, the logs are unreachable too: a device
+    whose storage was failing returned that message from every endpoint
+    *including* the log viewer, because journalctl could not be executed. The
+    underlying `[Errno 5] Input/output error` named the fault immediately.
+
+    Returns "TypeName: message", credentials redacted and length capped. The
+    type alone is worth carrying -- a bare PermissionError says more than any
+    generic sentence.
+
+    Args:
+        exc: The exception to describe
+        max_length: Truncate beyond this many characters
+
+    Returns:
+        A single-line description, never empty
+    """
+    message = str(exc).strip()
+    text = f"{type(exc).__name__}: {message}" if message else type(exc).__name__
+    # Order matters: the URL and header forms are more specific than the
+    # generic key=value pattern, which would otherwise chew the scheme.
+    text = _REDACT_URL_USERINFO.sub(r'\1<redacted>\3', text)
+    text = _REDACT_AUTH_HEADER.sub(r'\1<redacted>', text)
+    text = _REDACT_CREDENTIAL.sub(r'\1<redacted>', text)
+    # Collapse newlines/tabs so the detail stays one line in a JSON field.
+    text = ' '.join(text.split())
+    if len(text) > max_length:
+        text = text[:max_length - 1].rstrip() + '…'
+    return text
+
+
 def create_error_response(
     error_code: ErrorCode,
     message: str,

test/test_cache_ttl_honoured.py

@@ -0,0 +1,130 @@
+"""Tests that a per-entry ttl actually controls expiry.
+
+Regression under test: `CacheManager.set(key, data, ttl=...)` stored the value
+and no read path ever consulted it. Expiry came from a `max_age` inferred from
+substrings in the key ("live", "odds", "stock"), so every caller passing `ttl=`
+-- 48 sites across the plugins and 4 in the core -- was writing a number that
+did nothing. The old docstring admitted as much: "stored for compatibility but
+expiration is still controlled via max_age when reading".
+
+Measured against a real device's cache (8,873 entries carrying a ttl), the
+inferred value and the intended one disagreed almost everywhere:
+
+    stocks    max_age  600  vs ttl   1800   4903 entries
+    news      max_age 3600  vs ttl    600   1770 entries
+    odds      max_age 1800  vs ttl   3600   1301 entries
+    images    max_age  300  vs ttl 2592000    20 entries
+
+No `sports_live` entry carries a ttl, so live scores keep their inferred
+30-second freshness either way.
+"""
+
+import time
+
+import pytest
+
+from src.cache.memory_cache import MemoryCache
+from src.cache.disk_cache import DiskCache
+
+
+@pytest.fixture
+def disk(tmp_path):
+    return DiskCache(cache_dir=str(tmp_path))
+
+
+def _record(ttl=None, age=0.0):
+    rec = {"data": {"v": 1}, "timestamp": time.time() - age}
+    if ttl is not None:
+        rec["ttl"] = ttl
+    return rec
+
+
+class TestDiskCacheHonoursTtl:
+    def test_ttl_longer_than_max_age_keeps_the_entry(self, disk):
+        # The odds case: written wanting an hour, expired at 30 minutes.
+        disk.set("odds_espn_football_nfl_401", _record(ttl=3600, age=1900))
+        assert disk.get("odds_espn_football_nfl_401", max_age=1800) is not None
+
+    def test_ttl_shorter_than_max_age_expires_the_entry(self, disk):
+        # The news case: written wanting 10 minutes, kept for an hour.
+        disk.set("news_NHL_1", _record(ttl=600, age=900))
+        assert disk.get("news_NHL_1", max_age=3600) is None
+
+    def test_without_a_ttl_max_age_still_applies(self, disk):
+        disk.set("plain_key", _record(age=400))
+        assert disk.get("plain_key", max_age=300) is None
+        disk.set("plain_key2", _record(age=100))
+        assert disk.get("plain_key2", max_age=300) is not None
+
+    def test_a_fresh_entry_within_its_ttl_survives(self, disk):
+        disk.set("k", _record(ttl=600, age=10))
+        assert disk.get("k", max_age=30) is not None
+
+    def test_ttl_zero_expires_immediately(self, disk):
+        # 0 means zero seconds, not "forever" -- max_age=None is how a caller
+        # asks for no expiry.
+        disk.set("k", _record(ttl=0, age=1))
+        assert disk.get("k", max_age=99999) is None
+
+    @pytest.mark.parametrize("bad", ["600", None, True, False, -5, {"a": 1}])
+    def test_a_nonsense_ttl_falls_back_to_max_age(self, disk, bad):
+        # Including bools: True is an int in Python and must not become a 1s ttl.
+        rec = _record(age=400)
+        rec["ttl"] = bad
+        disk.set("k_%s" % type(bad).__name__, rec)
+        assert disk.get("k_%s" % type(bad).__name__, max_age=300) is None
+
+
+class TestMemoryCacheHonoursTtl:
+    def test_ttl_longer_than_max_age_keeps_the_entry(self):
+        m = MemoryCache()
+        m.set("k", _record(ttl=3600))
+        m._timestamps["k"] = time.time() - 1900
+        assert m.get("k", max_age=1800) is not None
+
+    def test_ttl_shorter_than_max_age_expires_the_entry(self):
+        m = MemoryCache()
+        m.set("k", _record(ttl=600))
+        m._timestamps["k"] = time.time() - 900
+        assert m.get("k", max_age=3600) is None
+
+    def test_without_a_ttl_max_age_still_applies(self):
+        m = MemoryCache()
+        m.set("k", _record())
+        m._timestamps["k"] = time.time() - 400
+        assert m.get("k", max_age=300) is None
+
+    def test_both_layers_agree(self, tmp_path):
+        """A record must not be live in one layer and expired in the other."""
+        rec = _record(ttl=3600, age=1900)
+        d = DiskCache(cache_dir=str(tmp_path))
+        d.set("k", rec)
+        m = MemoryCache()
+        m.set("k", rec)
+        m._timestamps["k"] = rec["timestamp"]
+        assert (d.get("k", max_age=1800) is not None) == (m.get("k", max_age=1800) is not None)
+
+
+class TestEndToEnd:
+    def test_set_then_get_respects_the_ttl(self, tmp_path, monkeypatch):
+        """The behaviour a caller of CacheManager.set(ttl=...) expects."""
+        from src.cache_manager import CacheManager
+
+        cm = CacheManager()
+        cm._disk_cache_component = DiskCache(cache_dir=str(tmp_path))
+        cm._memory_cache_component = MemoryCache()
+
+        cm.set("odds_espn_football_nfl_401", {"spread": 6.5}, ttl=3600)
+
+        # Age the stored record past the inferred max_age for odds (1800s) but
+        # within the ttl the caller asked for.
+        path = cm._disk_cache_component.get_cache_path("odds_espn_football_nfl_401")
+        import json
+        rec = json.load(open(path))
+        rec["timestamp"] = time.time() - 1900
+        json.dump(rec, open(path, "w"))
+        cm._memory_cache_component.clear() if hasattr(
+            cm._memory_cache_component, "clear") else None
+
+        got = cm.get_with_auto_strategy("odds_espn_football_nfl_401")
+        assert got is not None, "the ttl the caller asked for was ignored"

test/test_vegas_stall_watchdog.py

@@ -0,0 +1,119 @@
+"""Tests the watchdog that catches a stalled render loop in the act.
+
+Frame-time statistics can say a stall happened but not what caused it, and by
+the time the numbers reach the log the stack is long gone. On a live rig the
+Vegas loop showed a 3.2s freeze roughly twice an hour with every other frame
+under 25ms -- invisible in the mean, and unattributable from the log alone.
+This watchdog samples every thread's stack while the loop is still wedged,
+which is how that freeze was traced to a plugin generating a 17,000px scroll
+image, logo PNG decode and all, on the render thread.
+"""
+
+import threading
+import time
+
+import pytest
+
+from src.vegas_mode.coordinator import _StallWatchdog
+
+
+@pytest.fixture
+def watchdog():
+    made = []
+
+    def build(threshold):
+        w = _StallWatchdog(threshold)
+        made.append(w)
+        return w
+
+    yield build
+    for w in made:
+        w.stop()
+    for w in made:
+        w._thread.join(timeout=2.0)
+        assert not w._thread.is_alive(), "watchdog thread outlived its owner"
+
+
+def _dumps(caplog):
+    return [r for r in caplog.records if 'render loop stalled' in r.getMessage()]
+
+
+class TestItFiresOnlyWhenStalled:
+    def test_a_beating_loop_is_never_reported(self, watchdog, caplog):
+        w = watchdog(0.2)
+        deadline = time.time() + 0.9
+        while time.time() < deadline:
+            w.beat()
+            time.sleep(0.02)
+        assert not _dumps(caplog)
+
+    def test_a_stalled_loop_is_reported(self, watchdog, caplog):
+        w = watchdog(0.2)
+        w.beat()
+        time.sleep(0.9)
+        assert _dumps(caplog), "no stall dump for a loop that stopped beating"
+
+    def test_one_dump_per_stall_not_per_poll(self, watchdog, caplog):
+        # The watchdog polls at threshold/4, so a stall lasting many poll
+        # intervals must not flood the log with a dump each time.
+        w = watchdog(0.2)
+        w.beat()
+        time.sleep(1.2)
+        assert len(_dumps(caplog)) == 1, (
+            "%d dumps for one stall" % len(_dumps(caplog)))
+
+    def test_a_later_stall_is_reported_again(self, watchdog, caplog):
+        w = watchdog(0.2)
+        w.beat()
+        time.sleep(0.6)
+        first = len(_dumps(caplog))
+        w.beat()                      # recovered
+        time.sleep(0.6)               # then stalled again
+        assert len(_dumps(caplog)) == first + 1
+
+
+class TestWhatItReports:
+    def test_the_dump_names_threads_and_shows_frames(self, watchdog, caplog):
+        started = threading.Event()
+        release = threading.Event()
+
+        def parked():
+            started.set()
+            release.wait(3.0)
+
+        t = threading.Thread(target=parked, name="CulpritThread", daemon=True)
+        t.start()
+        started.wait(2.0)
+        try:
+            w = watchdog(0.2)
+            w.beat()
+            time.sleep(0.7)
+            dumps = _dumps(caplog)
+            assert dumps
+            text = dumps[0].getMessage()
+            assert "CulpritThread" in text, text
+            assert " in " in text, "no frames in the dump"
+            assert ".py:" in text, "no file:line in the dump"
+        finally:
+            release.set()
+            t.join(timeout=2.0)
+
+    def test_it_reports_how_long_the_stall_ran(self, watchdog, caplog):
+        w = watchdog(0.2)
+        w.beat()
+        time.sleep(0.8)
+        text = _dumps(caplog)[0].getMessage()
+        assert "stalled" in text
+        # Long enough to have tripped, and not an absurd value.
+        stalled = float(text.split("stalled")[1].split("s")[0])
+        assert 0.2 <= stalled <= 3.0, stalled
+
+
+class TestItIsCheapWhenIdle:
+    def test_stop_is_prompt(self, caplog):
+        w = _StallWatchdog(4.0)          # long threshold, long poll interval
+        t0 = time.time()
+        w.stop()
+        w._thread.join(timeout=3.0)
+        assert not w._thread.is_alive(), "stop() did not end the thread"
+        assert time.time() - t0 < 2.0, "stop() waited out the poll interval"

test/test_web_error_detail.py

@@ -0,0 +1,248 @@
+"""Tests for surfacing the underlying error in web responses.
+
+Regression under test: every failing endpoint returned "An error occurred; see
+logs for details" and nothing else. On a device whose storage was failing that
+sentence came back from the restart action, from /system/status, and from
+/logs -- the log viewer itself -- because journalctl could not be executed. The
+exception underneath said `[Errno 5] Input/output error: 'systemctl'`, which
+names the fault outright, and nine handlers were discarding it entirely rather
+than even logging it.
+"""
+
+import pytest
+
+from src.web_interface.error_handler import describe_exception
+
+
+class TestDescribeException:
+    def test_names_the_type_and_message(self):
+        detail = describe_exception(OSError(5, "Input/output error", "systemctl"))
+        assert detail == "OSError: [Errno 5] Input/output error: 'systemctl'"
+
+    def test_the_reported_failure_is_legible(self):
+        # The whole point: this string is the diagnosis.
+        assert "Input/output error" in describe_exception(
+            OSError(5, "Input/output error", "systemctl"))
+
+    def test_a_bare_exception_still_names_its_type(self):
+        # A PermissionError with no message still says more than "unknown".
+        assert describe_exception(PermissionError()) == "PermissionError"
+        assert describe_exception(Exception()) == "Exception"
+
+    def test_message_is_kept_when_present(self):
+        assert describe_exception(ValueError("bad port")) == "ValueError: bad port"
+
+
+class TestCredentialRedaction:
+    """Exception text quotes URLs, and plugins authenticate by query string."""
+
+    @pytest.mark.parametrize("secret_text,leaked", [
+        ("failed: https://api.x.com/v1?api_key=SEC123&city=Tampa", "SEC123"),
+        ("token=abcdef123456 was rejected", "abcdef123456"),
+        ("connect failed password=hunter2", "hunter2"),
+        ("GET /?access_token=zzz999", "zzz999"),
+        ('{"secret": "topsecret"}', "topsecret"),
+        # requests quotes the URL it failed on, and both of these forms turn
+        # up in real client exceptions.
+        ("401 for https://user:hunter2@example.com/api", "hunter2"),
+        ("headers: {'Authorization': 'Bearer eyJ.SECRET.sig'}", "eyJ.SECRET.sig"),
+        ("Authorization: Basic dXNlcjpwYXNzd29yZA==", "dXNlcjpwYXNzd29yZA=="),
+        ("Proxy-Authorization: Bearer ptok999", "ptok999"),
+        # Any scheme, not a fixed list -- a list silently leaks whatever it
+        # does not name, and plugin APIs invent their own.
+        ("Authorization: ApiKey SECRET123", "SECRET123"),
+        ("Authorization: Negotiate YIIZnegotiateblob", "YIIZnegotiateblob"),
+        ("Authorization: NTLM TlRMTVNTUAAB", "TlRMTVNTUAAB"),
+        ("authorization: barecredential", "barecredential"),
+    ])
+    def test_credentials_never_reach_the_response(self, secret_text, leaked):
+        detail = describe_exception(RuntimeError(secret_text))
+        assert leaked not in detail
+        assert "<redacted>" in detail
+
+    def test_the_parameter_name_survives_redaction(self):
+        # Knowing *which* credential was involved is part of the diagnosis.
+        detail = describe_exception(RuntimeError("https://x/y?api_key=SEC123"))
+        assert "api_key" in detail
+
+    def test_unknown_schemes_keep_their_name(self):
+        for scheme in ("ApiKey", "Negotiate", "NTLM", "AWS4-HMAC-SHA256"):
+            detail = describe_exception(
+                RuntimeError("Authorization: %s SECRETVALUE" % scheme))
+            assert scheme in detail, detail
+            assert "SECRETVALUE" not in detail, detail
+
+    def test_auth_scheme_and_username_survive(self):
+        # Which kind of credential, and whose, without the credential itself.
+        assert "Bearer" in describe_exception(
+            RuntimeError("Authorization: Bearer eyJ.SECRET.sig"))
+        assert "user" in describe_exception(
+            RuntimeError("https://user:hunter2@example.com"))
+
+    def test_non_secret_context_is_preserved(self):
+        detail = describe_exception(RuntimeError("https://api.x.com/v1?city=Tampa"))
+        assert "city=Tampa" in detail
+        assert "<redacted>" not in detail
+
+
+class TestBounds:
+    def test_long_messages_are_truncated(self):
+        detail = describe_exception(ValueError("x" * 5000))
+        assert len(detail) <= 400
+
+    def test_newlines_are_collapsed_to_one_line(self):
+        detail = describe_exception(ValueError("line one\nline two\tthree"))
+        assert "\n" not in detail and "\t" not in detail
+        assert detail == "ValueError: line one line two three"
+
+    def test_custom_length_is_honoured(self):
+        assert len(describe_exception(ValueError("y" * 500), max_length=50)) <= 50
+
+
+class TestHandlersCarryDetail:
+    """The response shape callers actually see."""
+
+    def test_no_api_v3_handler_discards_its_exception(self):
+        """Every generic-message handler must log a traceback and return detail.
+
+        Nine of them bound `e` and never used it, so the promised log entry was
+        never written either. Checking merely that *something* was logged is
+        too weak -- a `logger.info("failed")` would satisfy it while throwing
+        the exception away just as completely, so this asserts the two things
+        that actually make the failure diagnosable: an error-level record with
+        the traceback, and the sanitized detail in the response.
+        """
+        import ast
+
+        src = open("web_interface/blueprints/api_v3.py").read()
+        tree = ast.parse(src)
+        generic = "An error occurred; see logs for details"
+
+        def logs_a_traceback(handler):
+            """An error/exception-level log call carrying exc_info."""
+            for call in [n for n in ast.walk(handler) if isinstance(n, ast.Call)]:
+                func = call.func
+                if not isinstance(func, ast.Attribute):
+                    continue
+                if func.attr == "exception":       # implies exc_info
+                    return True
+                if func.attr not in ("error", "critical"):
+                    continue
+                if any(kw.arg == "exc_info" and getattr(kw.value, "value", False) is True
+                       for kw in call.keywords):
+                    return True
+            return False
+
+        def describes_this_exception(node, bound):
+            """A describe_exception(<bound>) call anywhere under `node`."""
+            for call in [n for n in ast.walk(node) if isinstance(n, ast.Call)]:
+                if not (isinstance(call.func, ast.Name)
+                        and call.func.id == "describe_exception"):
+                    continue
+                if bound is None:
+                    return True     # bare `except:` cannot name it; accept
+                if any(isinstance(a, ast.Name) and a.id == bound
+                       for a in call.args):
+                    return True
+            return False
+
+        def returns_the_detail(handler):
+            """The detail must be inside what the handler actually returns.
+
+            Looking anywhere in the handler is too weak: a handler could
+            compute describe_exception(e), drop it on the floor, and return the
+            generic message with no details field, while still passing. So the
+            call has to appear within a `return` expression.
+            """
+            returns = [n for n in ast.walk(handler) if isinstance(n, ast.Return)]
+            if not returns:
+                return False
+            return all(describes_this_exception(r, handler.name) for r in returns)
+
+        offenders = []
+        for h in [n for n in ast.walk(tree) if isinstance(n, ast.ExceptHandler)]:
+            seg = ast.get_source_segment(src, h) or ""
+            if generic not in seg:
+                continue
+            missing = []
+            if not logs_a_traceback(h):
+                missing.append("error-level log with exc_info")
+            if not returns_the_detail(h):
+                missing.append("describe_exception(e) in the response")
+            if missing:
+                offenders.append((h.lineno, missing))
+
+        assert not offenders, (
+            "handlers returning the generic message without %s: %r"
+            % ("both a traceback log and the detail", offenders))
+
+    def test_client_errors_keep_their_own_status(self):
+        """A 405 must not be reported as a server-side UNKNOWN_ERROR.
+
+        Werkzeug's HTTPExceptions subclass Exception, so the catch-all saw them
+        too: a GET on a POST-only route came back 500 "an error occurred",
+        which tells the caller nothing and blames the wrong side. Found while
+        probing a device whose POST-only config endpoints answered every GET
+        with UNKNOWN_ERROR.
+        """
+        from flask import Flask, jsonify
+        from werkzeug.exceptions import HTTPException
+
+        app = Flask(__name__)
+
+        @app.errorhandler(Exception)
+        def handle(error):
+            if isinstance(error, HTTPException):
+                return jsonify({
+                    "status": "error",
+                    "error_code": (error.name or "HTTP_ERROR").upper().replace(" ", "_"),
+                    "message": error.description,
+                }), error.code or 500
+            return jsonify({
+                "status": "error",
+                "error_code": "UNKNOWN_ERROR",
+                "message": "An error occurred; see logs for details",
+                "details": describe_exception(error),
+            }), 500
+
+        @app.route("/only-post", methods=["POST"])
+        def only_post():
+            return jsonify({"ok": True})
+
+        @app.route("/boom")
+        def boom():
+            raise OSError(5, "Input/output error", "systemctl")
+
+        client = app.test_client()
+
+        resp = client.get("/only-post")
+        assert resp.status_code == 405, "a wrong method must stay a 405"
+        assert resp.get_json()["error_code"] == "METHOD_NOT_ALLOWED"
+
+        # A genuine server fault still reports as one, with its detail.
+        resp = client.get("/boom")
+        assert resp.status_code == 500
+        assert "Input/output error" in resp.get_json()["details"]
+
+    def test_global_handler_reports_the_underlying_error(self):
+        from flask import Flask, jsonify
+
+        app = Flask(__name__)
+
+        @app.errorhandler(Exception)
+        def handle(error):
+            return jsonify({
+                "status": "error",
+                "error_code": "UNKNOWN_ERROR",
+                "message": "An error occurred; see logs for details",
+                "details": describe_exception(error),
+            }), 500
+
+        @app.route("/boom")
+        def boom():
+            raise OSError(5, "Input/output error", "systemctl")
+
+        client = app.test_client()
+        body = client.get("/boom").get_json()
+        assert body["error_code"] == "UNKNOWN_ERROR"
+        assert "Input/output error" in body["details"]

web_interface/app.py

@@ -16,6 +16,8 @@ from datetime import datetime, timedelta
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from src.config_manager import ConfigManager
+from src.web_interface.error_handler import describe_exception
+from werkzeug.exceptions import HTTPException
 from src.exceptions import ConfigError
 from src.plugin_system.plugin_manager import PluginManager
 from src.plugin_system.store_manager import PluginStoreManager
@@ -391,15 +393,42 @@ def internal_error(error):
     import logging
     logger = logging.getLogger('web_interface')
     logger.error("Internal server error", exc_info=True)
-    return jsonify({
+    payload = {
         'status': 'error',
         'error_code': 'INTERNAL_ERROR',
         'message': 'An internal error occurred; see logs for details',
-    }), 500
+    }
+    # Flask hands the original exception over as `error.original_exception`
+    # when propagation is off; without it there is nothing to describe.
+    original = getattr(error, 'original_exception', None) or (
+        error if isinstance(error, BaseException) else None)
+    if original is not None:
+        payload['details'] = describe_exception(original)
+    return jsonify(payload), 500
 
 @app.errorhandler(Exception)
 def handle_exception(error):
-    """Handle all unhandled exceptions."""
+    """Handle all unhandled exceptions.
+
+    Returning only "see logs for details" is fine until the logs are exactly
+    what you cannot reach. A device with failing storage answered every
+    endpoint with that sentence -- including the log viewer, because journalctl
+    could not be executed -- while the exception underneath said
+    `[Errno 5] Input/output error`. Naming the error costs nothing here and is
+    frequently the whole diagnosis, so include it alongside the log pointer.
+    """
+    # Werkzeug's HTTPExceptions subclass Exception, so this catch-all sees
+    # them too and was reporting every 405, 400, 413 and 415 as a server-side
+    # UNKNOWN_ERROR 500. A GET on a POST-only route came back as "an error
+    # occurred" rather than "method not allowed", which tells the caller
+    # nothing and blames the wrong side. Hand those back as themselves.
+    if isinstance(error, HTTPException):
+        return jsonify({
+            'status': 'error',
+            'error_code': (error.name or 'HTTP_ERROR').upper().replace(' ', '_'),
+            'message': error.description,
+        }), error.code or 500
+
     import logging
     logger = logging.getLogger('web_interface')
     logger.error("Unhandled exception", exc_info=True)
@@ -407,6 +436,7 @@ def handle_exception(error):
         'status': 'error',
         'error_code': 'UNKNOWN_ERROR',
         'message': 'An error occurred; see logs for details',
+        'details': describe_exception(error),
     }), 500
 
 # Captive portal redirect middleware

web_interface/blueprints/api_v3.py

@@ -22,6 +22,7 @@ logger = logging.getLogger(__name__)
 from src.web_interface.api_helpers import success_response, error_response, validate_request_json
 from src.web_interface.errors import ErrorCode
 from src.web_interface.secret_helpers import find_secret_fields, separate_secrets
+from src.web_interface.error_handler import describe_exception
 from src.plugin_system.operation_types import OperationType
 from src.web_interface.validators import (
     validate_file_upload
@@ -272,7 +273,7 @@ def get_main_config():
         return jsonify({'status': 'success', 'data': config})
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/config/schedule', methods=['GET'])
 def get_schedule_config():
@@ -290,9 +291,11 @@ def get_schedule_config():
 
         return success_response(data=schedule_config)
     except Exception as e:
+        logger.error("%s failed", request.path, exc_info=True)
         return error_response(
             ErrorCode.CONFIG_LOAD_FAILED,
             "An error occurred; see logs for details",
+            details=describe_exception(e),
             status_code=500
         )
 
@@ -468,7 +471,7 @@ def save_schedule_config():
             ErrorCode.CONFIG_SAVE_FAILED,
             "An error occurred; see logs for details",
 
-            status_code=500
+            status_code=500, details=describe_exception(e)
         )
 
 @api_v3.route('/config/dim-schedule', methods=['GET'])
@@ -516,14 +519,14 @@ def get_dim_schedule_config():
         return error_response(
             ErrorCode.CONFIG_LOAD_FAILED,
             "An error occurred; see logs for details",
-            status_code=500
+            status_code=500, details=describe_exception(e)
         )
     except Exception as e:
         logging.error(f"[DIM SCHEDULE] Unexpected error loading config: {e}", exc_info=True)
         return error_response(
             ErrorCode.CONFIG_LOAD_FAILED,
             "An error occurred; see logs for details",
-            status_code=500
+            status_code=500, details=describe_exception(e)
         )
 
 @api_v3.route('/config/dim-schedule', methods=['POST'])
@@ -687,7 +690,7 @@ def save_dim_schedule_config():
             ErrorCode.CONFIG_SAVE_FAILED,
             "An error occurred; see logs for details",
 
-            status_code=500
+            status_code=500, details=describe_exception(e)
         )
 
 @api_v3.route('/config/main', methods=['POST'])
@@ -1314,7 +1317,7 @@ def save_main_config():
         return error_response(
             ErrorCode.CONFIG_SAVE_FAILED,
             "An error occurred; see logs for details",
-            status_code=500
+            status_code=500, details=describe_exception(e)
         )
 
 @api_v3.route('/config/secrets', methods=['GET'])
@@ -1328,7 +1331,7 @@ def get_secrets_config():
         return jsonify({'status': 'success', 'data': config})
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/config/raw/main', methods=['POST'])
 def save_raw_main_config():
@@ -1361,6 +1364,7 @@ def save_raw_main_config():
             return error_response(
                 ErrorCode.CONFIG_SAVE_FAILED,
                 error_message,
+                details=describe_exception(e),
 
                 context={'config_path': e.config_path} if hasattr(e, 'config_path') and e.config_path else None,
                 status_code=500
@@ -1370,6 +1374,7 @@ def save_raw_main_config():
             return error_response(
                 ErrorCode.UNKNOWN_ERROR,
                 error_message,
+                details=describe_exception(e),
 
                 status_code=500
             )
@@ -1409,7 +1414,8 @@ def save_raw_secrets_config():
         else:
             error_message = 'An error occurred; see logs for details'
 
-        return jsonify({'status': 'error', 'message': error_message}), 500
+        return jsonify({'status': 'error', 'message': error_message,
+                        'details': describe_exception(e)}), 500
 
 @api_v3.route('/system/status', methods=['GET'])
 def get_system_status():
@@ -1497,7 +1503,7 @@ def get_system_status():
         return jsonify({'status': 'success', 'data': status})
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/health', methods=['GET'])
 def get_health():
@@ -1596,9 +1602,11 @@ def get_health():
 
         return jsonify({'status': 'success', 'data': health_status})
     except Exception as e:
+        logger.error("%s failed", request.path, exc_info=True)
         return jsonify({
             'status': 'error',
             'message': 'An error occurred; see logs for details',
+            'details': describe_exception(e),
             'data': {'status': 'unhealthy'}
         }), 500
 
@@ -2368,7 +2376,7 @@ def get_display_current():
         return jsonify({'status': 'success', 'data': display_data})
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/display/on-demand/status', methods=['GET'])
 def get_on_demand_status():
@@ -2392,7 +2400,7 @@ def get_on_demand_status():
         })
     except Exception as exc:
         logger.error('Error in get_on_demand_status', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500
 
 @api_v3.route('/display/on-demand/start', methods=['POST'])
 def start_on_demand_display():
@@ -2495,7 +2503,7 @@ def start_on_demand_display():
         return jsonify({'status': 'success', 'data': response_data})
     except Exception as exc:
         logger.error('Error in start_on_demand_display', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500
 
 @api_v3.route('/display/on-demand/stop', methods=['POST'])
 def stop_on_demand_display():
@@ -2531,7 +2539,7 @@ def stop_on_demand_display():
         })
     except Exception as exc:
         logger.error('Error in stop_on_demand_display', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500
 
 @api_v3.route('/plugins/installed', methods=['GET'])
 def get_installed_plugins():
@@ -2679,7 +2687,7 @@ def get_installed_plugins():
         return jsonify({'status': 'success', 'data': {'plugins': plugins}})
     except Exception as e:
         logger.error('Error in get_installed_plugins', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 def _installed_plugin_ids():
     """Best-effort list of installed plugin IDs for the web process.
@@ -2745,7 +2753,7 @@ def get_plugin_health():
         })
     except Exception as e:
         logger.error('Error in get_plugin_health', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/health/<plugin_id>', methods=['GET'])
 def get_plugin_health_single(plugin_id):
@@ -2770,7 +2778,7 @@ def get_plugin_health_single(plugin_id):
         })
     except Exception as e:
         logger.error('Error in get_plugin_health_single', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/health/<plugin_id>/reset', methods=['POST'])
 def reset_plugin_health(plugin_id):
@@ -2795,7 +2803,7 @@ def reset_plugin_health(plugin_id):
         })
     except Exception as e:
         logger.error('Error in reset_plugin_health', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/metrics', methods=['GET'])
 def get_plugin_metrics():
@@ -2835,7 +2843,7 @@ def get_plugin_metrics():
         })
     except Exception as e:
         logger.error('Error in get_plugin_metrics', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/metrics/<plugin_id>', methods=['GET'])
 def get_plugin_metrics_single(plugin_id):
@@ -2860,7 +2868,7 @@ def get_plugin_metrics_single(plugin_id):
         })
     except Exception as e:
         logger.error('Error in get_plugin_metrics_single', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/metrics/<plugin_id>/reset', methods=['POST'])
 def reset_plugin_metrics(plugin_id):
@@ -2885,7 +2893,7 @@ def reset_plugin_metrics(plugin_id):
         })
     except Exception as e:
         logger.error('Error in reset_plugin_metrics', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/limits/<plugin_id>', methods=['GET', 'POST'])
 def manage_plugin_limits(plugin_id):
@@ -2940,7 +2948,7 @@ def manage_plugin_limits(plugin_id):
             })
     except Exception as e:
         logger.error('Error in manage_plugin_limits', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/toggle', methods=['POST'])
 def toggle_plugin():
@@ -3949,7 +3957,7 @@ def install_plugin():
 
     except Exception as e:
         logger.error('Error in install_plugin', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/install-from-url', methods=['POST'])
 def install_plugin_from_url():
@@ -4004,7 +4012,7 @@ def install_plugin_from_url():
 
     except Exception as e:
         logger.error('Error in install_plugin_from_url', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/registry-from-url', methods=['POST'])
 def get_registry_from_url():
@@ -4036,7 +4044,7 @@ def get_registry_from_url():
 
     except Exception as e:
         logger.error('Error in get_registry_from_url', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/saved-repositories', methods=['GET'])
 def get_saved_repositories():
@@ -4049,7 +4057,7 @@ def get_saved_repositories():
         return jsonify({'status': 'success', 'data': {'repositories': repositories}})
     except Exception as e:
         logger.error('Error in get_saved_repositories', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/saved-repositories', methods=['POST'])
 def add_saved_repository():
@@ -4080,7 +4088,7 @@ def add_saved_repository():
             }), 400
     except Exception as e:
         logger.error('Error in add_saved_repository', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/saved-repositories', methods=['DELETE'])
 def remove_saved_repository():
@@ -4110,7 +4118,7 @@ def remove_saved_repository():
             }), 404
     except Exception as e:
         logger.error('Error in remove_saved_repository', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/store/list', methods=['GET'])
 def list_plugin_store():
@@ -4163,7 +4171,7 @@ def list_plugin_store():
         return jsonify({'status': 'success', 'data': {'plugins': formatted_plugins}})
     except Exception as e:
         logger.error('Error in list_plugin_store', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/store/github-status', methods=['GET'])
 def get_github_auth_status():
@@ -4214,7 +4222,7 @@ def get_github_auth_status():
             })
     except Exception as e:
         logger.error('Error in get_github_auth_status', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/store/refresh', methods=['POST'])
 def refresh_plugin_store():
@@ -4241,7 +4249,7 @@ def refresh_plugin_store():
         })
     except Exception as e:
         logger.error('Error in refresh_plugin_store', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 def deep_merge(base_dict, update_dict):
     """
@@ -5763,7 +5771,7 @@ def get_plugin_schema():
         return jsonify({'status': 'success', 'data': {'schema': default_schema}})
     except Exception as e:
         logger.error('Error in get_plugin_schema', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/skins', methods=['GET'])
 def list_skins():
@@ -5798,9 +5806,9 @@ def list_skins():
                 'has_preview': bool(preview and (skin_dir / preview).is_file()),
             })
         return jsonify({'status': 'success', 'data': {'skins': payload}})
-    except Exception:
+    except Exception as e:
         logger.error('Error in list_skins', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/config/reset', methods=['POST'])
 def reset_plugin_config():
@@ -5880,7 +5888,7 @@ def reset_plugin_config():
         })
     except Exception as e:
         logger.error('Error in reset_plugin_config', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/action', methods=['POST'])
 def execute_plugin_action():
@@ -6140,7 +6148,7 @@ sys.exit(proc.returncode)
                             logger.error("Error executing action step 1", exc_info=True)
                             return jsonify({
                                 'status': 'error',
-                                'message': 'An error occurred; see logs for details'
+                                'message': 'An error occurred; see logs for details', 'details': describe_exception(e)
                             }), 500
                     else:
                         # Simple script execution
@@ -6190,7 +6198,7 @@ sys.exit(proc.returncode)
         return jsonify({'status': 'error', 'message': 'Action timed out'}), 408
     except Exception as e:
         logger.error('Error in execute_plugin_action', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/authenticate/spotify', methods=['POST'])
 def authenticate_spotify():
@@ -6323,12 +6331,12 @@ sys.exit(proc.returncode)
                 logger.error("Error getting Spotify auth URL", exc_info=True)
                 return jsonify({
                     'status': 'error',
-                    'message': 'An error occurred; see logs for details'
+                    'message': 'An error occurred; see logs for details', 'details': describe_exception(e)
                 }), 500
 
     except Exception as e:
         logger.error('Error in authenticate_spotify', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/authenticate/ytm', methods=['POST'])
 def authenticate_ytm():
@@ -6378,7 +6386,7 @@ def authenticate_ytm():
         return jsonify({'status': 'error', 'message': 'Authentication timed out'}), 408
     except Exception as e:
         logger.error('Error in authenticate_ytm', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/fonts/catalog', methods=['GET'])
 def get_fonts_catalog():
@@ -6473,7 +6481,10 @@ def get_fonts_catalog():
 
         return jsonify({'status': 'success', 'data': {'catalog': catalog}})
     except Exception as e:
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        logger.error("%s failed", request.path, exc_info=True)
+        return jsonify({'status': 'error',
+                        'message': 'An error occurred; see logs for details',
+                        'details': describe_exception(e)}), 500
 
 @api_v3.route('/fonts/tokens', methods=['GET'])
 def get_font_tokens():
@@ -6492,7 +6503,7 @@ def get_font_tokens():
         return jsonify({'status': 'success', 'data': {'tokens': tokens}})
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/fonts/overrides', methods=['GET'])
 def get_fonts_overrides():
@@ -6504,7 +6515,7 @@ def get_fonts_overrides():
         return jsonify({'status': 'success', 'data': {'overrides': overrides}})
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/fonts/overrides', methods=['POST'])
 def save_fonts_overrides():
@@ -6518,7 +6529,7 @@ def save_fonts_overrides():
         return jsonify({'status': 'success', 'message': 'Font overrides saved'})
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/fonts/overrides/<element_key>', methods=['DELETE'])
 def delete_font_override(element_key):
@@ -6528,7 +6539,7 @@ def delete_font_override(element_key):
         return jsonify({'status': 'success', 'message': f'Font override for {element_key} deleted'})
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/fonts/upload', methods=['POST'])
 def upload_font():
@@ -6593,7 +6604,7 @@ def upload_font():
         })
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 
 @api_v3.route('/fonts/preview', methods=['GET'])
@@ -6738,7 +6749,7 @@ def get_font_preview() -> tuple[Response, int] | Response:
         })
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 
 @api_v3.route('/fonts/<font_family>', methods=['DELETE'])
@@ -6826,7 +6837,7 @@ def delete_font(font_family: str) -> tuple[Response, int] | Response:
         })
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 
 @api_v3.route('/plugins/assets/upload', methods=['POST'])
@@ -6974,7 +6985,7 @@ def upload_plugin_asset():
 
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/of-the-day/json/upload', methods=['POST'])
 def upload_of_the_day_json():
@@ -7124,7 +7135,7 @@ def upload_of_the_day_json():
 
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/of-the-day/json/delete', methods=['POST'])
 def delete_of_the_day_json():
@@ -7171,7 +7182,7 @@ def delete_of_the_day_json():
 
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/<plugin_id>/static/<path:file_path>', methods=['GET'])
 def serve_plugin_static(plugin_id, file_path):
@@ -7217,7 +7228,7 @@ def serve_plugin_static(plugin_id, file_path):
 
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 
 @api_v3.route('/plugins/calendar/upload-credentials', methods=['POST'])
@@ -7299,7 +7310,7 @@ def upload_calendar_credentials():
 
     except Exception as e:
         logger.error('Error in upload_calendar_credentials', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/assets/delete', methods=['POST'])
 def delete_plugin_asset():
@@ -7342,7 +7353,7 @@ def delete_plugin_asset():
 
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/plugins/assets/list', methods=['GET'])
 def list_plugin_assets():
@@ -7370,7 +7381,7 @@ def list_plugin_assets():
 
     except Exception as e:
         logger.error('Unhandled exception', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/display/current-status', methods=['GET'])
 def get_current_display_status():
@@ -7391,9 +7402,9 @@ def get_current_display_status():
                 'last_updated': None,
             }
         return jsonify({'status': 'success', 'data': state})
-    except Exception:
+    except Exception as e:
         logger.error('Error in get_current_display_status', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/logs', methods=['GET'])
 def get_logs():
@@ -7432,9 +7443,11 @@ def get_logs():
             'message': 'Timeout while fetching logs'
         }), 500
     except Exception as e:
+        logger.error("%s failed", request.path, exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details',
+            'details': describe_exception(e)
         }), 500
 
 # Multi-Display Sync Endpoints
@@ -7499,9 +7512,11 @@ def get_wifi_status():
             }
         })
     except Exception as e:
+        logger.error("%s failed", request.path, exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details',
+            'details': describe_exception(e)
         }), 500
 
 @api_v3.route('/wifi/scan', methods=['GET'])
@@ -7623,7 +7638,7 @@ def connect_wifi():
         logger.error("Error connecting to WiFi", exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details', 'details': describe_exception(e)
         }), 500
 
 @api_v3.route('/wifi/disconnect', methods=['POST'])
@@ -7649,7 +7664,7 @@ def disconnect_wifi():
         logger.error("Error disconnecting from WiFi", exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details', 'details': describe_exception(e)
         }), 500
 
 @api_v3.route('/wifi/ap/enable', methods=['POST'])
@@ -7674,9 +7689,11 @@ def enable_ap_mode():
                 'message': message
             }), 400
     except Exception as e:
+        logger.error("%s failed", request.path, exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details',
+            'details': describe_exception(e)
         }), 500
 
 @api_v3.route('/wifi/ap/disable', methods=['POST'])
@@ -7699,9 +7716,11 @@ def disable_ap_mode():
                 'message': message
             }), 400
     except Exception as e:
+        logger.error("%s failed", request.path, exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details',
+            'details': describe_exception(e)
         }), 500
 
 @api_v3.route('/wifi/ap/auto-enable', methods=['GET'])
@@ -7720,9 +7739,11 @@ def get_auto_enable_ap_mode():
             }
         })
     except Exception as e:
+        logger.error("%s failed", request.path, exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details',
+            'details': describe_exception(e)
         }), 500
 
 @api_v3.route('/wifi/ap/auto-enable', methods=['POST'])
@@ -7752,9 +7773,11 @@ def set_auto_enable_ap_mode():
             }
         })
     except Exception as e:
+        logger.error("%s failed", request.path, exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details',
+            'details': describe_exception(e)
         }), 500
 
 @api_v3.route('/wifi/radio', methods=['GET'])
@@ -7774,7 +7797,7 @@ def get_wifi_radio():
         logger.error("Error getting WiFi radio state", exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details', 'details': describe_exception(e)
         }), 500
 
 @api_v3.route('/wifi/radio', methods=['POST'])
@@ -7822,7 +7845,7 @@ def set_wifi_radio():
         logger.error("Error setting WiFi radio state", exc_info=True)
         return jsonify({
             'status': 'error',
-            'message': 'An error occurred; see logs for details'
+            'message': 'An error occurred; see logs for details', 'details': describe_exception(e)
         }), 500
 
 @api_v3.route('/cache/list', methods=['GET'])
@@ -7847,7 +7870,7 @@ def list_cache_files():
         })
     except Exception as e:
         logger.error('Error in list_cache_files', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 @api_v3.route('/cache/delete', methods=['POST'])
 def delete_cache_file():
@@ -7873,7 +7896,7 @@ def delete_cache_file():
         })
     except Exception as e:
         logger.error('Error in delete_cache_file', exc_info=True)
-        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details'}), 500
+        return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500
 
 
 # =============================================================================