rick/LEDMatrix
1
0
Fork 0
mirror of https://github.com/ChuckBuilds/LEDMatrix.git synced 2026-04-10 13:02:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(store): cache GitHub API calls and eliminate redundant requests (#251)

Browse Source 
The plugin store was making excessive GitHub API calls causing slow
page loads (10-30s):

- Installed plugins endpoint called get_plugin_info() per plugin (3
  GitHub API calls each) just to read the `verified` field from the
  registry. Use new get_registry_info() instead (zero API calls).
- _get_latest_commit_info() had no cache — all 31 monorepo plugins
  share the same repo URL, causing 31 identical API calls. Add 5-min
  cache keyed by repo:branch.
- _fetch_manifest_from_github() also uncached — add 5-min cache.
- load_config() called inside loop per-plugin — hoist outside loop.
- Install/update operations pass force_refresh=True to bypass caches
  and always get the latest commit SHA from GitHub.

Co-authored-by: Chuck <chuck@example.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Chuck
2026-02-15 14:46:31 -05:00
committed by GitHub
parent 5b0ad5ab71
commit 22c495ea7c
3 changed files with 75 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
src/plugin_system/store_manager.py
View File

@@ -54,6 +54,10 @@ class PluginStoreManager:
self.github_cache = {} # Cache for GitHub API responses
self.cache_timeout = 3600 # 1 hour cache timeout
self.registry_cache_timeout = 300 # 5 minutes for registry cache
self.commit_info_cache = {} # Cache for latest commit info: {key: (timestamp, data)}
self.commit_cache_timeout = 300 # 5 minutes (same as registry)
self.manifest_cache = {} # Cache for GitHub manifest fetches: {key: (timestamp, data)}
self.manifest_cache_timeout = 300 # 5 minutes
self.github_token = self._load_github_token()
self._token_validation_cache = {} # Cache for token validation results: {token: (is_valid, timestamp, error_message)}
self._token_validation_cache_timeout = 300 # 5 minutes cache for token validation
@@ -576,7 +580,7 @@ class PluginStoreManager:
return results
def _fetch_manifest_from_github(self, repo_url: str, branch: str = "master", manifest_path: str = "manifest.json") -> Optional[Dict]:
def _fetch_manifest_from_github(self, repo_url: str, branch: str = "master", manifest_path: str = "manifest.json", force_refresh: bool = False) -> Optional[Dict]:
"""
Fetch manifest.json directly from a GitHub repository.
@@ -585,6 +589,7 @@ class PluginStoreManager:
branch: Branch name (default: master)
manifest_path: Path to manifest within the repo (default: manifest.json).
For monorepo plugins this will be e.g. "plugins/football-scoreboard/manifest.json".
force_refresh: If True, bypass the cache.
Returns:
Manifest data or None if not found
@@ -603,24 +608,38 @@ class PluginStoreManager:
owner = parts[-2]
repo = parts[-1]
# Check cache first
cache_key = f"{owner}/{repo}:{branch}:{manifest_path}"
if not force_refresh and cache_key in self.manifest_cache:
cached_time, cached_data = self.manifest_cache[cache_key]
if time.time() - cached_time < self.manifest_cache_timeout:
return cached_data
raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{manifest_path}"
response = self._http_get_with_retries(raw_url, timeout=10)
if response.status_code == 200:
return response.json()
result = response.json()
self.manifest_cache[cache_key] = (time.time(), result)
return result
elif response.status_code == 404:
# Try main branch instead
if branch != "main":
raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/main/{manifest_path}"
response = self._http_get_with_retries(raw_url, timeout=10)
if response.status_code == 200:
return response.json()
result = response.json()
self.manifest_cache[cache_key] = (time.time(), result)
return result
# Cache negative result
self.manifest_cache[cache_key] = (time.time(), None)
except Exception as e:
self.logger.debug(f"Could not fetch manifest from GitHub for {repo_url}: {e}")
return None
def _get_latest_commit_info(self, repo_url: str, branch: str = "main") -> Optional[Dict[str, Any]]:
def _get_latest_commit_info(self, repo_url: str, branch: str = "main", force_refresh: bool = False) -> Optional[Dict[str, Any]]:
"""Return metadata about the latest commit on the given branch."""
try:
if 'github.com' not in repo_url:
@@ -637,6 +656,13 @@ class PluginStoreManager:
owner = parts[-2]
repo = parts[-1]
# Check cache first
cache_key = f"{owner}/{repo}:{branch}"
if not force_refresh and cache_key in self.commit_info_cache:
cached_time, cached_data = self.commit_info_cache[cache_key]
if time.time() - cached_time < self.commit_cache_timeout:
return cached_data
branches_to_try = self._distinct_sequence([branch, 'main', 'master'])
headers = {
@@ -659,7 +685,7 @@ class PluginStoreManager:
commit_author = commit_meta.get('author', {})
commit_date_iso = commit_author.get('date', '')
return {
result = {
'branch': branch_name,
'sha': commit_sha_full,
'short_sha': commit_sha_short,
@@ -668,6 +694,8 @@ class PluginStoreManager:
'author': commit_author.get('name', ''),
'message': commit_meta.get('message', ''),
}
self.commit_info_cache[cache_key] = (time.time(), result)
return result
if response.status_code == 403 and not self.github_token:
self.logger.debug("GitHub commit API rate limited (403). Consider adding a token.")
@@ -678,13 +706,16 @@ class PluginStoreManager:
if last_error:
self.logger.debug(f"Unable to fetch commit info for {repo_url}: {last_error}")
# Cache negative result to avoid repeated failing calls
self.commit_info_cache[cache_key] = (time.time(), None)
except Exception as e:
self.logger.debug(f"Error fetching latest commit metadata for {repo_url}: {e}")
return None
def get_plugin_info(self, plugin_id: str, fetch_latest_from_github: bool = True) -> Optional[Dict]:
def get_plugin_info(self, plugin_id: str, fetch_latest_from_github: bool = True, force_refresh: bool = False) -> Optional[Dict]:
"""
Get detailed information about a plugin from the registry.
@@ -694,6 +725,7 @@ class PluginStoreManager:
Args:
plugin_id: Plugin identifier
fetch_latest_from_github: If True (default), augment with GitHub commit metadata.
force_refresh: If True, bypass caches for commit/manifest data.
Returns:
Plugin metadata or None if not found
@@ -718,7 +750,7 @@ class PluginStoreManager:
plugin_info['last_updated'] = github_info.get('last_commit_date', plugin_info.get('last_updated'))
plugin_info['last_updated_iso'] = github_info.get('last_commit_iso', plugin_info.get('last_updated_iso'))
commit_info = self._get_latest_commit_info(repo_url, branch)
commit_info = self._get_latest_commit_info(repo_url, branch, force_refresh=force_refresh)
if commit_info:
plugin_info['last_commit'] = commit_info.get('short_sha')
plugin_info['last_commit_sha'] = commit_info.get('sha')
@@ -731,7 +763,7 @@ class PluginStoreManager:
plugin_subpath = plugin_info.get('plugin_path', '')
manifest_rel = f"{plugin_subpath}/manifest.json" if plugin_subpath else "manifest.json"
github_manifest = self._fetch_manifest_from_github(repo_url, branch, manifest_rel)
github_manifest = self._fetch_manifest_from_github(repo_url, branch, manifest_rel, force_refresh=force_refresh)
if github_manifest:
if 'last_updated' in github_manifest and not plugin_info.get('last_updated'):
plugin_info['last_updated'] = github_manifest['last_updated']
@@ -740,6 +772,23 @@ class PluginStoreManager:
return plugin_info
def get_registry_info(self, plugin_id: str) -> Optional[Dict]:
"""
Get plugin information from the registry cache only (no GitHub API calls).
Use this for lightweight lookups where only registry fields are needed
(e.g., verified status, latest_version).
Args:
plugin_id: Plugin identifier
Returns:
Plugin metadata from registry or None if not found
"""
registry = self.fetch_registry()
plugins = registry.get('plugins', []) or []
return next((p for p in plugins if p.get('id') == plugin_id), None)
def install_plugin(self, plugin_id: str, branch: Optional[str] = None) -> bool:
"""
Install a plugin from the official registry. Always installs the latest commit
@@ -754,7 +803,7 @@ class PluginStoreManager:
branch_info = f" (branch: {branch})" if branch else " (latest branch head)"
self.logger.info(f"Installing plugin: {plugin_id}{branch_info}")
plugin_info = self.get_plugin_info(plugin_id, fetch_latest_from_github=True)
plugin_info = self.get_plugin_info(plugin_id, fetch_latest_from_github=True, force_refresh=True)
if not plugin_info:
self.logger.error(f"Plugin not found in registry: {plugin_id}")
return False
@@ -1721,7 +1770,7 @@ class PluginStoreManager:
# Try to get remote info from registry (optional)
self.fetch_registry(force_refresh=True)
plugin_info_remote = self.get_plugin_info(plugin_id, fetch_latest_from_github=True)
plugin_info_remote = self.get_plugin_info(plugin_id, fetch_latest_from_github=True, force_refresh=True)
remote_branch = None
remote_sha = None
@@ -1993,7 +2042,7 @@ class PluginStoreManager:
# Try registry-based update
self.logger.info(f"Plugin {plugin_id} is not a git repository, checking registry...")
self.fetch_registry(force_refresh=True)
plugin_info_remote = self.get_plugin_info(plugin_id, fetch_latest_from_github=True)
plugin_info_remote = self.get_plugin_info(plugin_id, fetch_latest_from_github=True, force_refresh=True)
# If not in registry but we have a repo URL, try reinstalling from that URL
if not plugin_info_remote and repo_url:

8
web_interface/blueprints/api_v3.py
View File

@@ -1737,6 +1737,9 @@ def get_installed_plugins():
# Get all installed plugin info from the plugin manager
all_plugin_info = api_v3.plugin_manager.get_all_plugin_info()
# Load config once before the loop (not per-plugin)
full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {}
# Format for the web interface
plugins = []
for plugin_info in all_plugin_info:
@@ -1758,7 +1761,6 @@ def get_installed_plugins():
# Read from config file first, fall back to plugin instance if config doesn't have the key
enabled = None
if api_v3.config_manager:
full_config = api_v3.config_manager.load_config()
plugin_config = full_config.get(plugin_id, {})
# Check if 'enabled' key exists in config (even if False)
if 'enabled' in plugin_config:
@@ -1773,8 +1775,8 @@ def get_installed_plugins():
# Default to True if no config key and plugin not loaded (matches BasePlugin default)
enabled = True
# Get verified status from store registry (if available)
store_info = api_v3.plugin_store_manager.get_plugin_info(plugin_id)
# Get verified status from store registry (no GitHub API calls needed)
store_info = api_v3.plugin_store_manager.get_registry_info(plugin_id)
verified = store_info.get('verified', False) if store_info else False
# Get local git info for installed plugin (actual installed commit)

8
web_interface/blueprints/pages_v3.py
View File

@@ -186,6 +186,9 @@ def _load_plugins_partial():
# Get all installed plugin info
all_plugin_info = pages_v3.plugin_manager.get_all_plugin_info()
# Load config once before the loop (not per-plugin)
full_config = pages_v3.config_manager.load_config() if pages_v3.config_manager else {}
# Format for the web interface
for plugin_info in all_plugin_info:
plugin_id = plugin_info.get('id')
@@ -206,7 +209,6 @@ def _load_plugins_partial():
# Read from config file first, fall back to plugin instance if config doesn't have the key
enabled = None
if pages_v3.config_manager:
full_config = pages_v3.config_manager.load_config()
plugin_config = full_config.get(plugin_id, {})
# Check if 'enabled' key exists in config (even if False)
if 'enabled' in plugin_config:
@@ -221,8 +223,8 @@ def _load_plugins_partial():
# Default to True if no config key and plugin not loaded (matches BasePlugin default)
enabled = True
# Get verified status from store registry
store_info = pages_v3.plugin_store_manager.get_plugin_info(plugin_id)
# Get verified status from store registry (no GitHub API calls needed)
store_info = pages_v3.plugin_store_manager.get_registry_info(plugin_id)
verified = store_info.get('verified', False) if store_info else False
last_updated = plugin_info.get('last_updated')