From 1ac6499b0c454f6bec8c5f3b1cb6ab7609da696e Mon Sep 17 00:00:00 2001 From: Chuck Date: Tue, 19 May 2026 20:51:15 -0400 Subject: [PATCH] fix(web-ui): dedup registry fetches, surface reconciliation warnings, add check-update endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Story 1 — src/plugin_system/store_manager.py: Add threading.Lock (_registry_fetch_lock) to fetch_registry(). The outer cache check remains the hot path (no lock). When the cache is cold, only one thread hits the network; concurrent callers block on the lock then get the result from the warm cache (double-checked locking). Eliminates duplicate GitHub requests on every page load when the 15-minute cache expires. Story 2 — web_interface/app.py + api_v3.py + overview.html: _run_startup_reconciliation() now writes /tmp/ledmatrix_reconciliation.json (atomic tempfile+replace, mirrors hw_status pattern) so the result survives the background thread. New GET /api/v3/plugins/reconciliation-status reads that file. Overview page gains a dismissible yellow banner that shows stale plugin_id values (e.g. sync, github, youtube) and tells the user to remove them or reinstall from the Plugin Store. Banner is suppressed for the session after dismiss using sessionStorage keyed on the plugin_id list. Story 3 — web_interface/blueprints/api_v3.py: Add GET /api/v3/system/check-update. Does git fetch origin main then compares local HEAD vs origin/main to compute update_available, remote_sha, and commits_behind. Result is cached for 5 minutes so it doesn't run git on every page load. Falls back to {update_available: false} on any error. Eliminates the 404 logged on every page load. Story 4 (Pi 5 rgbmatrix rebuild) was already fixed in PR #341. Co-Authored-By: Claude Sonnet 4.6 --- src/plugin_system/store_manager.py | 80 +++++++++++-------- web_interface/app.py | 27 +++++++ web_interface/blueprints/api_v3.py | 59 ++++++++++++++ .../templates/v3/partials/overview.html | 50 ++++++++++++ 4 files changed, 183 insertions(+), 33 deletions(-) diff --git a/src/plugin_system/store_manager.py b/src/plugin_system/store_manager.py index 5d4ba48a..d547fbf0 100644 --- a/src/plugin_system/store_manager.py +++ b/src/plugin_system/store_manager.py @@ -10,6 +10,7 @@ import json import stat import subprocess import shutil +import threading import zipfile import tempfile import requests @@ -100,6 +101,10 @@ class PluginStoreManager: # handlers. Bumping the cached-entry timestamp on failure serves # the stale payload cheaply until the backoff expires. self._failure_backoff_seconds = 60 + # Prevents concurrent callers from each firing a network request when + # the registry cache expires. Only one thread fetches; others wait and + # then get the result from the warm cache (double-checked locking). + self._registry_fetch_lock = threading.Lock() # Ensure plugins directory exists self.plugins_dir.mkdir(exist_ok=True) @@ -575,41 +580,50 @@ class PluginStoreManager: (current_time - self.registry_cache_time) < self.registry_cache_timeout): return self.registry_cache - try: - self.logger.info(f"Fetching plugin registry from {self.REGISTRY_URL}") - response = self._http_get_with_retries(self.REGISTRY_URL, timeout=10) - response.raise_for_status() - self.registry_cache = response.json() - self.registry_cache_time = current_time - self.logger.info(f"Fetched registry with {len(self.registry_cache.get('plugins', []))} plugins") - return self.registry_cache - except requests.RequestException as e: - self.logger.error(f"Error fetching registry: {e}") - if raise_on_failure: - raise - # Prefer stale cache over an empty list so the plugin list UI - # keeps working on a flaky connection (e.g. Pi on WiFi). Bump - # registry_cache_time into a short backoff window so the next - # request serves the stale payload cheaply instead of - # re-hitting the network on every request (matches the - # pattern used by github_cache / commit_info_cache). - if self.registry_cache: - self.logger.warning("Falling back to stale registry cache") - self.registry_cache_time = ( - time.time() + self._failure_backoff_seconds - self.registry_cache_timeout - ) + with self._registry_fetch_lock: + # Re-check inside the lock — a concurrent caller that was waiting + # may have already populated the cache while we blocked. + current_time = time.time() + if (self.registry_cache and self.registry_cache_time and + not force_refresh and + (current_time - self.registry_cache_time) < self.registry_cache_timeout): return self.registry_cache - return {"plugins": []} - except json.JSONDecodeError as e: - self.logger.error(f"Error parsing registry JSON: {e}") - if raise_on_failure: - raise - if self.registry_cache: - self.registry_cache_time = ( - time.time() + self._failure_backoff_seconds - self.registry_cache_timeout - ) + + try: + self.logger.info(f"Fetching plugin registry from {self.REGISTRY_URL}") + response = self._http_get_with_retries(self.REGISTRY_URL, timeout=10) + response.raise_for_status() + self.registry_cache = response.json() + self.registry_cache_time = current_time + self.logger.info(f"Fetched registry with {len(self.registry_cache.get('plugins', []))} plugins") return self.registry_cache - return {"plugins": []} + except requests.RequestException as e: + self.logger.error(f"Error fetching registry: {e}") + if raise_on_failure: + raise + # Prefer stale cache over an empty list so the plugin list UI + # keeps working on a flaky connection (e.g. Pi on WiFi). Bump + # registry_cache_time into a short backoff window so the next + # request serves the stale payload cheaply instead of + # re-hitting the network on every request (matches the + # pattern used by github_cache / commit_info_cache). + if self.registry_cache: + self.logger.warning("Falling back to stale registry cache") + self.registry_cache_time = ( + time.time() + self._failure_backoff_seconds - self.registry_cache_timeout + ) + return self.registry_cache + return {"plugins": []} + except json.JSONDecodeError as e: + self.logger.error(f"Error parsing registry JSON: {e}") + if raise_on_failure: + raise + if self.registry_cache: + self.registry_cache_time = ( + time.time() + self._failure_backoff_seconds - self.registry_cache_timeout + ) + return self.registry_cache + return {"plugins": []} def search_plugins(self, query: str = "", category: str = "", tags: List[str] = None, fetch_commit_info: bool = True, include_saved_repos: bool = True, saved_repositories_manager = None) -> List[Dict]: """ diff --git a/web_interface/app.py b/web_interface/app.py index 57286300..c70fb028 100644 --- a/web_interface/app.py +++ b/web_interface/app.py @@ -716,6 +716,33 @@ def _run_startup_reconciliation() -> None: "manual 'Reconcile' action to resolve.", len(result.inconsistencies_manual), ) + + # Write status file so the web UI can surface unresolved issues as a + # banner without the user having to read journalctl. Mirrors the + # hw_status pattern (/tmp/led_matrix_hw_status.json). + import json as _json, tempfile as _tempfile, os as _os + _recon_status = { + "done": True, + "successful": result.reconciliation_successful, + "fixed_count": len(result.inconsistencies_fixed), + "unresolved": [ + { + "plugin_id": inc.plugin_id, + "type": inc.inconsistency_type.value, + "description": inc.description, + } + for inc in result.inconsistencies_manual + ], + } + _recon_path = "/tmp/ledmatrix_reconciliation.json" + try: + if not _os.path.islink(_recon_path): + _fd, _tmp = _tempfile.mkstemp(dir="/tmp", prefix=".led_recon_") + with _os.fdopen(_fd, "w") as _f: + _json.dump(_recon_status, _f) + _os.replace(_tmp, _recon_path) + except Exception as _e: + _logger.warning("[Reconciliation] Could not write status file: %s", _e) except Exception as e: _logger.error("[Reconciliation] Error: %s", e, exc_info=True) finally: diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 58c4fe68..5df734e9 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -1384,6 +1384,52 @@ def get_system_version(): except Exception as e: return jsonify({'status': 'error', 'message': str(e)}), 500 +_update_check_cache: Dict[str, Any] = {'result': None, 'ts': 0.0} +_UPDATE_CHECK_TTL = 300 # 5 minutes — avoids a git fetch on every page load + +@api_v3.route('/system/check-update', methods=['GET']) +def check_for_update(): + """Check whether a newer LEDMatrix commit is available on origin/main.""" + now = time.time() + if _update_check_cache['result'] and now - _update_check_cache['ts'] < _UPDATE_CHECK_TTL: + return jsonify(_update_check_cache['result']) + + _safe: Dict[str, Any] = {'update_available': False, 'remote_sha': 'unknown', 'commits_behind': 0} + try: + cwd = str(PROJECT_ROOT) + subprocess.run( + ['git', 'fetch', 'origin', 'main', '--quiet'], + capture_output=True, timeout=10, cwd=cwd, + ) + local = subprocess.run( + ['git', 'rev-parse', 'HEAD'], + capture_output=True, text=True, timeout=5, cwd=cwd, + ).stdout.strip() + remote = subprocess.run( + ['git', 'rev-parse', 'origin/main'], + capture_output=True, text=True, timeout=5, cwd=cwd, + ).stdout.strip() + + if not local or not remote: + return jsonify(_safe) + + if local == remote: + result: Dict[str, Any] = {'update_available': False, 'remote_sha': remote, 'commits_behind': 0} + else: + count_str = subprocess.run( + ['git', 'rev-list', 'HEAD..origin/main', '--count'], + capture_output=True, text=True, timeout=5, cwd=cwd, + ).stdout.strip() + count = int(count_str) if count_str.isdigit() else 0 + result = {'update_available': count > 0, 'remote_sha': remote, 'commits_behind': count} + + _update_check_cache['result'] = result + _update_check_cache['ts'] = now + return jsonify(result) + except Exception as e: + logger.debug("check-update: %s", e) + return jsonify(_safe) + @api_v3.route('/system/action', methods=['POST']) def execute_system_action(): """Execute system actions (start/stop/reboot/etc)""" @@ -2433,6 +2479,19 @@ def reconcile_plugin_state(): status_code=500 ) +@api_v3.route('/plugins/reconciliation-status', methods=['GET']) +def get_reconciliation_status(): + """Return the result of the last startup reconciliation from /tmp status file.""" + _recon_path = "/tmp/ledmatrix_reconciliation.json" + try: + with open(_recon_path) as _f: + data = json.load(_f) + return jsonify({'status': 'success', 'data': data}) + except FileNotFoundError: + return jsonify({'status': 'success', 'data': {'done': False, 'unresolved': []}}) + except Exception as e: + return jsonify({'status': 'error', 'message': str(e)}), 500 + @api_v3.route('/plugins/config', methods=['GET']) def get_plugin_config(): """Get plugin configuration""" diff --git a/web_interface/templates/v3/partials/overview.html b/web_interface/templates/v3/partials/overview.html index 36af1300..92e3cc0c 100644 --- a/web_interface/templates/v3/partials/overview.html +++ b/web_interface/templates/v3/partials/overview.html @@ -1,3 +1,53 @@ + + + +

System Overview