fix(reconciler): add sync, github, youtube to _SYSTEM_CONFIG_KEYS (#351)

config_manager.load_config() deep-merges config_secrets.json into the
main config before returning it. This means secrets top-level keys
(github, youtube) appear alongside structural config keys (sync) in the
dict that _get_config_state() iterates.

_SYSTEM_CONFIG_KEYS was missing all three, so the reconciler treated them
as plugin IDs and flagged them as PLUGIN_MISSING_ON_DISK on every startup,
showing the "Stale plugin config entries found" warning banner to users on
a fresh install where those plugins have never existed.

Add the three keys with brief comments explaining which file each comes
from so the distinction is clear when the list grows.

Co-authored-by: Chuck <chuck@example.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

src/plugin_system/state_reconciliation.py

@@ -185,13 +185,19 @@ class StateReconciliation:
                 message=f"Reconciliation failed: {str(e)}"
             )
     
-    # Top-level config keys that are NOT plugins
+    # Top-level config keys that are NOT plugins.
+    # Includes both config.json structural keys and config_secrets.json top-level
+    # keys (load_config() deep-merges secrets in, so secrets keys appear here too).
     _SYSTEM_CONFIG_KEYS = frozenset({
         'web_display_autostart', 'timezone', 'location', 'display',
         'plugin_system', 'vegas_scroll_speed', 'vegas_separator_width',
         'vegas_target_fps', 'vegas_buffer_ahead', 'vegas_plugin_order',
         'vegas_excluded_plugins', 'vegas_scroll_enabled', 'logging',
         'dim_schedule', 'network', 'system', 'schedule',
+        # Multi-display sync config (config.json structural key)
+        'sync',
+        # Secrets file top-level keys (merged in by load_config)
+        'github', 'youtube',
     })
 
     def _get_config_state(self) -> Dict[str, Dict[str, Any]]:

web_interface/app.py

@@ -2,8 +2,10 @@ from flask import Flask, request, redirect, url_for, jsonify, Response, send_fro
 import json
 import logging
 import os
+import queue
 import sys
 import subprocess
+import threading
 import time
 from pathlib import Path
 from datetime import datetime, timedelta
@@ -413,13 +415,53 @@ def add_security_headers(response):
     
     return response
 
-# SSE helper function
+class _StreamBroadcaster:
-def sse_response(generator_func):
+    """Fan-out broadcaster: one background generator thread pushes to all SSE clients.
-    """Helper to create SSE responses"""
+
-    def generate():
+    This means N browser tabs share one generator instead of each running their own,
-        for data in generator_func():
+    keeping PIL encodes / subprocess forks constant regardless of how many tabs are open.
-            yield f"data: {json.dumps(data)}\n\n"
+    """
-    return Response(generate(), mimetype='text/event-stream')
+
+    def __init__(self, generator_factory):
+        self._generator_factory = generator_factory
+        self._clients: set = set()
+        self._lock = threading.Lock()
+        self._thread: threading.Thread | None = None
+
+    def subscribe(self) -> queue.Queue:
+        q: queue.Queue = queue.Queue(maxsize=5)
+        with self._lock:
+            self._clients.add(q)
+            if not (self._thread and self._thread.is_alive()):
+                self._thread = threading.Thread(target=self._broadcast, daemon=True)
+                self._thread.start()
+        return q
+
+    def unsubscribe(self, q: queue.Queue) -> None:
+        with self._lock:
+            self._clients.discard(q)
+
+    def _broadcast(self):
+        for data in self._generator_factory():
+            with self._lock:
+                if not self._clients:
+                    # No subscribers — exit so the thread doesn't spin indefinitely.
+                    # subscribe() will restart it when a new client arrives.
+                    break
+                for q in self._clients:
+                    try:
+                        q.put_nowait(data)
+                    except queue.Full:
+                        # Client is reading too slowly; drop the oldest item and
+                        # deliver the latest so the queue never stalls the client.
+                        try:
+                            q.get_nowait()
+                        except queue.Empty:
+                            pass
+                        try:
+                            q.put_nowait(data)
+                        except queue.Full:
+                            pass
 
 # System status generator for SSE
 def system_status_generator():
@@ -596,20 +638,50 @@ def logs_generator():
 
         time.sleep(5)  # Update every 5 seconds (reduced frequency for better performance)
 
+# One broadcaster per stream — shared across all SSE clients
+_stats_broadcaster = _StreamBroadcaster(system_status_generator)
+_display_broadcaster = _StreamBroadcaster(display_preview_generator)
+_logs_broadcaster = _StreamBroadcaster(logs_generator)
+
+
+def _sse_stream(broadcaster: _StreamBroadcaster) -> Response:
+    """Return a streaming SSE response backed by a shared broadcaster."""
+    q = broadcaster.subscribe()
+
+    def generate():
+        try:
+            while True:
+                try:
+                    data = q.get(timeout=30)
+                    yield f"data: {json.dumps(data)}\n\n"
+                except queue.Empty:
+                    # Send an SSE comment heartbeat to keep the connection alive
+                    # through proxies that close idle connections.
+                    yield ": heartbeat\n\n"
+        except GeneratorExit:
+            pass
+        finally:
+            broadcaster.unsubscribe(q)
+
+    return Response(generate(), mimetype='text/event-stream')
+
+
 # SSE endpoints
 @app.route('/api/v3/stream/stats')
 def stream_stats():
-    return sse_response(system_status_generator)
+    return _sse_stream(_stats_broadcaster)
 
 @app.route('/api/v3/stream/display')
 def stream_display():
-    return sse_response(display_preview_generator)
+    return _sse_stream(_display_broadcaster)
 
 @app.route('/api/v3/stream/logs')
 def stream_logs():
-    return sse_response(logs_generator)
+    return _sse_stream(_logs_broadcaster)
 
-# Exempt SSE streams from CSRF and add rate limiting
+# Exempt SSE streams from CSRF and apply a generous rate limit.
+# SSE connections are long-lived HTTP requests, not repeated API calls, so the
+# tight "20 per minute" default would be exhausted quickly on reconnects.
 if csrf:
     csrf.exempt(stream_stats)
     csrf.exempt(stream_display)
@@ -617,9 +689,9 @@ if csrf:
     # Note: api_v3 blueprint is exempted above after registration
 
 if limiter:
-    limiter.limit("20 per minute")(stream_stats)
+    limiter.limit("200 per minute")(stream_stats)
-    limiter.limit("20 per minute")(stream_display)
+    limiter.limit("200 per minute")(stream_display)
-    limiter.limit("20 per minute")(stream_logs)
+    limiter.limit("200 per minute")(stream_logs)
 
 # Main route - redirect to v3 interface as default
 @app.route('/')

web_interface/static/v3/app.js

@@ -1,4 +1,4 @@
-/* global showNotification, updateSystemStats, htmx */
+/* global showNotification, updateSystemStats, updateDisplayPreview, htmx */
 // LED Matrix v3 JavaScript
 // Additional helpers for HTMX and Alpine.js integration
 
@@ -51,7 +51,8 @@ document.body.addEventListener('htmx:afterRequest', function(event) {
     }
 });
 
-// SSE reconnection helper
+// SSE reconnection helper — closes and reopens both SSE streams,
+// reattaching the open/error handlers defined in base.html.
 window.reconnectSSE = function() {
     if (window.statsSource) {
         window.statsSource.close();
@@ -60,14 +61,18 @@ window.reconnectSSE = function() {
             const data = JSON.parse(event.data);
             if (typeof updateSystemStats === 'function') updateSystemStats(data);
         };
+        if (window._statsOpenHandler) window.statsSource.addEventListener('open', window._statsOpenHandler);
+        if (window._statsErrorHandler) window.statsSource.addEventListener('error', window._statsErrorHandler);
     }
 
     if (window.displaySource) {
         window.displaySource.close();
         window.displaySource = new EventSource('/api/v3/stream/display');
-        window.displaySource.onmessage = function() {
+        window.displaySource.onmessage = function(event) {
-            // Handle display updates
+            const data = JSON.parse(event.data);
+            if (typeof updateDisplayPreview === 'function') updateDisplayPreview(data);
         };
+        if (window._displayErrorHandler) window.displaySource.addEventListener('error', window._displayErrorHandler);
     }
 };
 

web_interface/templates/v3/base.html

@@ -1370,34 +1370,64 @@
 
     <!-- SSE connection for real-time updates -->
     <script>
-        // Connect to SSE streams
+        // Assign to window so reconnectSSE() in app.js can reach them.
-        const statsSource = new EventSource('/api/v3/stream/stats');
+        window.statsSource = new EventSource('/api/v3/stream/stats');
-        const displaySource = new EventSource('/api/v3/stream/display');
+        window.displaySource = new EventSource('/api/v3/stream/display');
 
-        statsSource.onmessage = function(event) {
+        window.statsSource.onmessage = function(event) {
             const data = JSON.parse(event.data);
             updateSystemStats(data);
         };
 
-        displaySource.onmessage = function(event) {
+        window.displaySource.onmessage = function(event) {
             const data = JSON.parse(event.data);
             updateDisplayPreview(data);
         };
 
-        // Connection status
+        function _setConnectionStatus(connected, reconnecting) {
-        statsSource.addEventListener('open', function() {
+            const el = document.getElementById('connection-status');
-            document.getElementById('connection-status').innerHTML = `
+            if (!el) return;
-                <div class="w-2 h-2 bg-green-500 rounded-full"></div>
+            if (connected) {
-                <span class="text-gray-600">Connected</span>
+                el.innerHTML = `
-            `;
+                    <div class="w-2 h-2 bg-green-500 rounded-full"></div>
-        });
+                    <span class="text-gray-600">Connected</span>
+                `;
+            } else if (reconnecting) {
+                el.innerHTML = `
+                    <div class="w-2 h-2 bg-yellow-500 rounded-full animate-pulse"></div>
+                    <span class="text-gray-600">Reconnecting…</span>
+                `;
+            } else {
+                el.innerHTML = `
+                    <div class="w-2 h-2 bg-red-500 rounded-full"></div>
+                    <span class="text-gray-600" title="Connection lost — try refreshing the page">Disconnected</span>
+                `;
+            }
+        }
 
-        statsSource.addEventListener('error', function() {
+        var _statsErrorCount = 0;
-            document.getElementById('connection-status').innerHTML = `
+
-                <div class="w-2 h-2 bg-red-500 rounded-full"></div>
+        // Named on window so reconnectSSE() in app.js can reattach them after
-                <span class="text-gray-600">Disconnected</span>
+        // replacing the EventSource instances.
-            `;
+        window._statsOpenHandler = function() {
-        });
+            _statsErrorCount = 0;
+            _setConnectionStatus(true, false);
+        };
+        window._statsErrorHandler = function() {
+            _statsErrorCount++;
+            // EventSource readyState 0 = CONNECTING (auto-retrying), 2 = CLOSED
+            var reconnecting = window.statsSource.readyState === EventSource.CONNECTING;
+            _setConnectionStatus(false, reconnecting && _statsErrorCount <= 3);
+        };
+        window._displayErrorHandler = function() {
+            // Display stream errors don't change the status badge but log to console
+            // so failures aren't completely silent.
+            console.warn('LEDMatrix: display preview stream error (readyState=' + window.displaySource.readyState + ')');
+        };
+
+        window.statsSource.addEventListener('open', window._statsOpenHandler);
+        window.statsSource.addEventListener('error', window._statsErrorHandler);
+        window.displaySource.addEventListener('error', window._displayErrorHandler);
 
         function updateSystemStats(data) {
             // Update CPU in header