fix(logs): include ledmatrix-web logs in viewer and log subprocess stderr on failure

Two bugs conspired to produce "check the logs" toasts with an empty log viewer:

1. The log viewer (both SSE stream and REST endpoint) only queried
   ledmatrix.service via journalctl. Web API errors are logged by the
   Flask process running as ledmatrix-web.service, so they never
   appeared in the viewer. Add -u ledmatrix-web.service to both calls;
   also add --output=short-iso so timestamps from the two services
   sort cleanly when interleaved. Use shutil.which-resolved absolute
   paths for sudo/journalctl (S607 compliance) in api_v3.py; fall back
   to known Pi paths if which returns None.

2. app.py: resolve journalctl and systemctl to absolute paths via
   shutil.which at module init (_JOURNALCTL, _SYSTEMCTL). Replace bare
   names in logs_generator() and the cached systemctl is-active check.
   Guard both sites: logs_generator yields a clear SSE error message
   and sleeps 60 s if journalctl is not found; the systemctl block is
   skipped entirely if systemctl is not found, leaving the cache at its
   last-known value.

3. When execute_system_action() ran a systemctl command that returned
   non-zero, only the return code was logged — result.stderr was
   silently discarded. Log it at ERROR level and include returncode and
   stderr in the JSON response so callers get actionable failure details.
   Same fix applied to the early-return start_display branch.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

web_interface/app.py

@@ -2,8 +2,11 @@ from flask import Flask, request, redirect, url_for, jsonify, Response, send_fro
 import json
 import logging
 import os
+import queue
+import shutil
 import sys
 import subprocess
+import threading
 import time
 from pathlib import Path
 from datetime import datetime, timedelta
@@ -22,6 +25,9 @@ from src.plugin_system.state_manager import PluginStateManager
 from src.plugin_system.operation_history import OperationHistory
 from src.plugin_system.health_monitor import PluginHealthMonitor
 
+_JOURNALCTL = shutil.which('journalctl')
+_SYSTEMCTL = shutil.which('systemctl')
+
 # Create Flask app
 app = Flask(__name__)
 app.secret_key = os.urandom(24)
@@ -413,13 +419,53 @@ def add_security_headers(response):
     
     return response
 
-# SSE helper function
+class _StreamBroadcaster:
-def sse_response(generator_func):
+    """Fan-out broadcaster: one background generator thread pushes to all SSE clients.
-    """Helper to create SSE responses"""
+
-    def generate():
+    This means N browser tabs share one generator instead of each running their own,
-        for data in generator_func():
+    keeping PIL encodes / subprocess forks constant regardless of how many tabs are open.
-            yield f"data: {json.dumps(data)}\n\n"
+    """
-    return Response(generate(), mimetype='text/event-stream')
+
+    def __init__(self, generator_factory):
+        self._generator_factory = generator_factory
+        self._clients: set = set()
+        self._lock = threading.Lock()
+        self._thread: threading.Thread | None = None
+
+    def subscribe(self) -> queue.Queue:
+        q: queue.Queue = queue.Queue(maxsize=5)
+        with self._lock:
+            self._clients.add(q)
+            if not (self._thread and self._thread.is_alive()):
+                self._thread = threading.Thread(target=self._broadcast, daemon=True)
+                self._thread.start()
+        return q
+
+    def unsubscribe(self, q: queue.Queue) -> None:
+        with self._lock:
+            self._clients.discard(q)
+
+    def _broadcast(self):
+        for data in self._generator_factory():
+            with self._lock:
+                if not self._clients:
+                    # No subscribers — exit so the thread doesn't spin indefinitely.
+                    # subscribe() will restart it when a new client arrives.
+                    break
+                for q in self._clients:
+                    try:
+                        q.put_nowait(data)
+                    except queue.Full:
+                        # Client is reading too slowly; drop the oldest item and
+                        # deliver the latest so the queue never stalls the client.
+                        try:
+                            q.get_nowait()
+                        except queue.Empty:
+                            pass
+                        try:
+                            q.put_nowait(data)
+                        except queue.Full:
+                            pass
 
 # System status generator for SSE
 def system_status_generator():
@@ -450,12 +496,13 @@ def system_status_generator():
             # Check if display service is running (cached to avoid per-client subprocess forks)
             now = time.time()
             if (now - _ledmatrix_service_cache['timestamp']) >= _LEDMATRIX_SERVICE_CACHE_TTL:
-                try:
+                if _SYSTEMCTL:
-                    result = subprocess.run(['systemctl', 'is-active', 'ledmatrix'],
+                    try:
-                                            capture_output=True, text=True, timeout=2)
+                        result = subprocess.run([_SYSTEMCTL, 'is-active', 'ledmatrix'],
-                    _ledmatrix_service_cache['active'] = result.stdout.strip() == 'active'
+                                                capture_output=True, text=True, timeout=2)
-                except (subprocess.SubprocessError, OSError):
+                        _ledmatrix_service_cache['active'] = result.stdout.strip() == 'active'
-                    pass
+                    except (subprocess.SubprocessError, OSError) as e:
+                        app.logger.warning("systemctl status check failed: %s", e)
                 _ledmatrix_service_cache['timestamp'] = now
             service_active = _ledmatrix_service_cache['active']
             
@@ -547,8 +594,13 @@ def logs_generator():
             # Get recent logs from journalctl (simplified version)
             # Note: User should be in systemd-journal group to read logs without sudo
             try:
+                if not _JOURNALCTL:
+                    yield {'timestamp': time.time(), 'logs': 'journalctl not found; cannot read logs'}
+                    time.sleep(60)
+                    continue
                 result = subprocess.run(
-                    ['journalctl', '-u', 'ledmatrix.service', '-n', '50', '--no-pager'],
+                    [_JOURNALCTL, '-u', 'ledmatrix.service', '-u', 'ledmatrix-web.service',
+                     '-n', '50', '--no-pager', '--output=short-iso'],
                     capture_output=True, text=True, timeout=5
                 )
 
@@ -564,7 +616,7 @@ def logs_generator():
                         # No logs available
                         logs_data = {
                             'timestamp': time.time(),
-                            'logs': 'No logs available from ledmatrix service'
+                            'logs': 'No logs available from ledmatrix or ledmatrix-web service'
                         }
                         yield logs_data
                 else:
@@ -596,20 +648,50 @@ def logs_generator():
 
         time.sleep(5)  # Update every 5 seconds (reduced frequency for better performance)
 
+# One broadcaster per stream — shared across all SSE clients
+_stats_broadcaster = _StreamBroadcaster(system_status_generator)
+_display_broadcaster = _StreamBroadcaster(display_preview_generator)
+_logs_broadcaster = _StreamBroadcaster(logs_generator)
+
+
+def _sse_stream(broadcaster: _StreamBroadcaster) -> Response:
+    """Return a streaming SSE response backed by a shared broadcaster."""
+    q = broadcaster.subscribe()
+
+    def generate():
+        try:
+            while True:
+                try:
+                    data = q.get(timeout=30)
+                    yield f"data: {json.dumps(data)}\n\n"
+                except queue.Empty:
+                    # Send an SSE comment heartbeat to keep the connection alive
+                    # through proxies that close idle connections.
+                    yield ": heartbeat\n\n"
+        except GeneratorExit:
+            pass
+        finally:
+            broadcaster.unsubscribe(q)
+
+    return Response(generate(), mimetype='text/event-stream')
+
+
 # SSE endpoints
 @app.route('/api/v3/stream/stats')
 def stream_stats():
-    return sse_response(system_status_generator)
+    return _sse_stream(_stats_broadcaster)
 
 @app.route('/api/v3/stream/display')
 def stream_display():
-    return sse_response(display_preview_generator)
+    return _sse_stream(_display_broadcaster)
 
 @app.route('/api/v3/stream/logs')
 def stream_logs():
-    return sse_response(logs_generator)
+    return _sse_stream(_logs_broadcaster)
 
-# Exempt SSE streams from CSRF and add rate limiting
+# Exempt SSE streams from CSRF and apply a generous rate limit.
+# SSE connections are long-lived HTTP requests, not repeated API calls, so the
+# tight "20 per minute" default would be exhausted quickly on reconnects.
 if csrf:
     csrf.exempt(stream_stats)
     csrf.exempt(stream_display)
@@ -617,9 +699,9 @@ if csrf:
     # Note: api_v3 blueprint is exempted above after registration
 
 if limiter:
-    limiter.limit("20 per minute")(stream_stats)
+    limiter.limit("200 per minute")(stream_stats)
-    limiter.limit("20 per minute")(stream_display)
+    limiter.limit("200 per minute")(stream_display)
-    limiter.limit("20 per minute")(stream_logs)
+    limiter.limit("200 per minute")(stream_logs)
 
 # Main route - redirect to v3 interface as default
 @app.route('/')

web_interface/blueprints/api_v3.py

@@ -4,6 +4,7 @@ import os
 import re
 import stat
 import sys
+import shutil
 import subprocess
 import tempfile
 import time
@@ -25,6 +26,9 @@ from src.web_interface.validators import (
 )
 from src.error_aggregator import get_error_aggregator
 
+_SUDO = shutil.which('sudo')
+_JOURNALCTL = shutil.which('journalctl')
+
 # Will be initialized when blueprint is registered
 config_manager = None
 plugin_manager = None
@@ -1456,31 +1460,41 @@ def execute_system_action():
             if mode:
                 # For on-demand modes, we would need to integrate with the display controller
                 # For now, just start the display service
-                result = subprocess.run(['sudo', 'systemctl', 'start', 'ledmatrix'],
+                try:
-                                     capture_output=True, text=True)
+                    result = subprocess.run(['sudo', 'systemctl', 'start', 'ledmatrix'],
+                                         capture_output=True, text=True, timeout=10)
+                except subprocess.TimeoutExpired as e:
+                    logger.error("start_display (%s) timed out: %s", mode, e)
+                    return jsonify({'status': 'error', 'message': 'Command timed out', 'returncode': -1, 'stderr': 'timeout'})
                 logger.info("start_display (%s) returned code %d", mode, result.returncode)
-                return jsonify({
+                if result.returncode != 0 and result.stderr:
+                    logger.error("start_display (%s) stderr: %s", mode, result.stderr.strip())
+                resp = {
                     'status': 'success' if result.returncode == 0 else 'error',
                     'message': 'Display started' if result.returncode == 0 else 'Failed to start display',
-                })
+                }
+                if result.returncode != 0:
+                    resp['returncode'] = result.returncode
+                    resp['stderr'] = result.stderr.strip()
+                return jsonify(resp)
             else:
                 result = subprocess.run(['sudo', 'systemctl', 'start', 'ledmatrix'],
-                                     capture_output=True, text=True)
+                                     capture_output=True, text=True, timeout=10)
         elif action == 'stop_display':
             result = subprocess.run(['sudo', 'systemctl', 'stop', 'ledmatrix'],
-                                 capture_output=True, text=True)
+                                 capture_output=True, text=True, timeout=10)
         elif action == 'enable_autostart':
             result = subprocess.run(['sudo', 'systemctl', 'enable', 'ledmatrix'],
-                                 capture_output=True, text=True)
+                                 capture_output=True, text=True, timeout=10)
         elif action == 'disable_autostart':
             result = subprocess.run(['sudo', 'systemctl', 'disable', 'ledmatrix'],
-                                 capture_output=True, text=True)
+                                 capture_output=True, text=True, timeout=10)
         elif action == 'reboot_system':
             result = subprocess.run(['sudo', 'reboot'],
-                                 capture_output=True, text=True)
+                                 capture_output=True, text=True, timeout=10)
         elif action == 'shutdown_system':
             result = subprocess.run(['sudo', 'poweroff'],
-                                 capture_output=True, text=True)
+                                 capture_output=True, text=True, timeout=10)
         elif action == 'git_pull':
             # Use PROJECT_ROOT instead of hardcoded path
             project_dir = str(PROJECT_ROOT)
@@ -1555,20 +1569,29 @@ def execute_system_action():
             })
         elif action == 'restart_display_service':
             result = subprocess.run(['sudo', 'systemctl', 'restart', 'ledmatrix'],
-                                 capture_output=True, text=True)
+                                 capture_output=True, text=True, timeout=10)
         elif action == 'restart_web_service':
             # Try to restart the web service (assuming it's ledmatrix-web.service)
             result = subprocess.run(['sudo', 'systemctl', 'restart', 'ledmatrix-web'],
-                                 capture_output=True, text=True)
+                                 capture_output=True, text=True, timeout=10)
         else:
             return jsonify({'status': 'error', 'message': 'Unknown action'}), 400
 
         logger.info("system action '%s' returncode=%d", action, result.returncode)
-        return jsonify({
+        if result.returncode != 0 and result.stderr:
+            logger.error("system action '%s' stderr: %s", action, result.stderr.strip())
+        resp = {
             'status': 'success' if result.returncode == 0 else 'error',
             'message': 'Action completed' if result.returncode == 0 else 'Action failed; check logs for details',
-        })
+        }
+        if result.returncode != 0:
+            resp['returncode'] = result.returncode
+            resp['stderr'] = result.stderr.strip()
+        return jsonify(resp)
 
+    except subprocess.TimeoutExpired as e:
+        logger.error("system action '%s' timed out: %s", action, e)
+        return jsonify({'status': 'error', 'message': 'Command timed out', 'returncode': -1, 'stderr': 'timeout'})
     except Exception as e:
         logger.error("execute_system_action failed: %s", e, exc_info=True)
         return jsonify({'status': 'error', 'message': 'Action failed; see logs for details'}), 500
@@ -6425,9 +6448,14 @@ def list_plugin_assets():
 def get_logs():
     """Get system logs from journalctl"""
     try:
+        if not _JOURNALCTL:
+            return jsonify({'status': 'error', 'message': 'journalctl not found on this system'}), 503
         # Get recent logs from journalctl
+        _cmd = ([_SUDO, _JOURNALCTL] if _SUDO else [_JOURNALCTL]) + [
+            '-u', 'ledmatrix.service', '-u', 'ledmatrix-web.service',
+            '-n', '100', '--no-pager', '--output=short-iso']
         result = subprocess.run(
-            ['sudo', 'journalctl', '-u', 'ledmatrix.service', '-n', '100', '--no-pager'],
+            _cmd,
             capture_output=True,
             text=True,
             timeout=5
@@ -6438,7 +6466,7 @@ def get_logs():
             return jsonify({
                 'status': 'success',
                 'data': {
-                    'logs': logs_text if logs_text else 'No logs available from ledmatrix service'
+                    'logs': logs_text if logs_text else 'No logs available from ledmatrix or ledmatrix-web service'
                 }
             })
         else:

web_interface/static/v3/app.js

@@ -1,4 +1,4 @@
-/* global showNotification, updateSystemStats, htmx */
+/* global showNotification, updateSystemStats, updateDisplayPreview, htmx */
 // LED Matrix v3 JavaScript
 // Additional helpers for HTMX and Alpine.js integration
 
@@ -51,7 +51,8 @@ document.body.addEventListener('htmx:afterRequest', function(event) {
     }
 });
 
-// SSE reconnection helper
+// SSE reconnection helper — closes and reopens both SSE streams,
+// reattaching the open/error handlers defined in base.html.
 window.reconnectSSE = function() {
     if (window.statsSource) {
         window.statsSource.close();
@@ -60,14 +61,18 @@ window.reconnectSSE = function() {
             const data = JSON.parse(event.data);
             if (typeof updateSystemStats === 'function') updateSystemStats(data);
         };
+        if (window._statsOpenHandler) window.statsSource.addEventListener('open', window._statsOpenHandler);
+        if (window._statsErrorHandler) window.statsSource.addEventListener('error', window._statsErrorHandler);
     }
 
     if (window.displaySource) {
         window.displaySource.close();
         window.displaySource = new EventSource('/api/v3/stream/display');
-        window.displaySource.onmessage = function() {
+        window.displaySource.onmessage = function(event) {
-            // Handle display updates
+            const data = JSON.parse(event.data);
+            if (typeof updateDisplayPreview === 'function') updateDisplayPreview(data);
         };
+        if (window._displayErrorHandler) window.displaySource.addEventListener('error', window._displayErrorHandler);
     }
 };
 

web_interface/templates/v3/base.html

@@ -1370,34 +1370,64 @@
 
     <!-- SSE connection for real-time updates -->
     <script>
-        // Connect to SSE streams
+        // Assign to window so reconnectSSE() in app.js can reach them.
-        const statsSource = new EventSource('/api/v3/stream/stats');
+        window.statsSource = new EventSource('/api/v3/stream/stats');
-        const displaySource = new EventSource('/api/v3/stream/display');
+        window.displaySource = new EventSource('/api/v3/stream/display');
 
-        statsSource.onmessage = function(event) {
+        window.statsSource.onmessage = function(event) {
             const data = JSON.parse(event.data);
             updateSystemStats(data);
         };
 
-        displaySource.onmessage = function(event) {
+        window.displaySource.onmessage = function(event) {
             const data = JSON.parse(event.data);
             updateDisplayPreview(data);
         };
 
-        // Connection status
+        function _setConnectionStatus(connected, reconnecting) {
-        statsSource.addEventListener('open', function() {
+            const el = document.getElementById('connection-status');
-            document.getElementById('connection-status').innerHTML = `
+            if (!el) return;
-                <div class="w-2 h-2 bg-green-500 rounded-full"></div>
+            if (connected) {
-                <span class="text-gray-600">Connected</span>
+                el.innerHTML = `
-            `;
+                    <div class="w-2 h-2 bg-green-500 rounded-full"></div>
-        });
+                    <span class="text-gray-600">Connected</span>
+                `;
+            } else if (reconnecting) {
+                el.innerHTML = `
+                    <div class="w-2 h-2 bg-yellow-500 rounded-full animate-pulse"></div>
+                    <span class="text-gray-600">Reconnecting…</span>
+                `;
+            } else {
+                el.innerHTML = `
+                    <div class="w-2 h-2 bg-red-500 rounded-full"></div>
+                    <span class="text-gray-600" title="Connection lost — try refreshing the page">Disconnected</span>
+                `;
+            }
+        }
 
-        statsSource.addEventListener('error', function() {
+        var _statsErrorCount = 0;
-            document.getElementById('connection-status').innerHTML = `
+
-                <div class="w-2 h-2 bg-red-500 rounded-full"></div>
+        // Named on window so reconnectSSE() in app.js can reattach them after
-                <span class="text-gray-600">Disconnected</span>
+        // replacing the EventSource instances.
-            `;
+        window._statsOpenHandler = function() {
-        });
+            _statsErrorCount = 0;
+            _setConnectionStatus(true, false);
+        };
+        window._statsErrorHandler = function() {
+            _statsErrorCount++;
+            // EventSource readyState 0 = CONNECTING (auto-retrying), 2 = CLOSED
+            var reconnecting = window.statsSource.readyState === EventSource.CONNECTING;
+            _setConnectionStatus(false, reconnecting && _statsErrorCount <= 3);
+        };
+        window._displayErrorHandler = function() {
+            // Display stream errors don't change the status badge but log to console
+            // so failures aren't completely silent.
+            console.warn('LEDMatrix: display preview stream error (readyState=' + window.displaySource.readyState + ')');
+        };
+
+        window.statsSource.addEventListener('open', window._statsOpenHandler);
+        window.statsSource.addEventListener('error', window._statsErrorHandler);
+        window.displaySource.addEventListener('error', window._displayErrorHandler);
 
         function updateSystemStats(data) {
             // Update CPU in header