diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 4d3c8dad..7c432109 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -3,21 +3,9 @@ name: Claude Code Review on: pull_request: types: [opened, synchronize, ready_for_review, reopened] - # Optional: Only run on specific file changes - # paths: - # - "src/**/*.ts" - # - "src/**/*.tsx" - # - "src/**/*.js" - # - "src/**/*.jsx" jobs: claude-review: - # Optional: Filter by PR author - # if: | - # github.event.pull_request.user.login == 'external-contributor' || - # github.event.pull_request.user.login == 'new-developer' || - # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' - runs-on: ubuntu-latest permissions: contents: read @@ -27,7 +15,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 1 @@ -45,6 +33,4 @@ jobs: plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' plugins: 'code-review@claude-code-plugins' prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 6b15fac7..cd9e45a9 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -26,7 +26,7 @@ jobs: actions: read # Required for Claude to read CI results on PRs steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 1 @@ -40,11 +40,3 @@ jobs: additional_permissions: | actions: read - # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. - # prompt: 'Update the pull request description to include a summary of changes.' - - # Optional: Add claude_args to customize behavior and configuration - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options - # claude_args: '--allowed-tools Bash(gh pr *)' - diff --git a/.gitignore b/.gitignore index fea80d19..86909a99 100644 --- a/.gitignore +++ b/.gitignore @@ -81,10 +81,10 @@ assets/stocks/crypto_icons/ # Plugin operation state written at runtime. # -# web_interface/app.py writes data/plugin_operations.json, data/plugin_state.json -# and data/operation_history.json as the web interface runs, into a directory that -# ships tracked (data/.gitkeep) and was otherwise unignored. So every rig that ever -# opened the web UI -- and every test run that constructs the app -- left three +# web_interface/app.py writes data/plugin_state.json and data/operation_history.json +# (older releases also data/plugin_operations.json) as the web interface runs, into +# a directory that ships tracked (data/.gitkeep). Unignored, every rig that ever +# opened the web UI -- and every test run that constructs the app -- would leave # untracked files behind and a permanently dirty `git status`. Same reasoning as # the logo rule above: a checkout that is always dirty is a checkout nobody reads. data/* diff --git a/CHANGELOG.md b/CHANGELOG.md index e9f2dab4..e18d56e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,14 +19,50 @@ accepts both, but the store flags the old spelling as deprecated ## Unreleased -- `src.common.frame_timing` -- times every frame the display presents, whoever - drew it, and writes cumulative counters to `/dev/shm`. Two tools read it: - `scripts/frame_soak.py` judges a running service (late frames, freezes, - where the time goes), and `scripts/render_bench.py` judges the hardware and - render path alone on a synthetic strip. Both fail a run above 0.1% late - frames, and both call a loop that never waited for the panel NOT LOCKED. A - stall watchdog logs the stack of whatever holds a scroll up for 250 ms or - more. See `docs/SCROLL_PERFORMANCE.md`, "Soaking a rig". +- Scripts and installer: + - `fix_web_permissions.sh` makes `safe_plugin_rm.sh` and `safe_pip_install.sh` root-owned again after resetting ownership. A web-user-owned copy of either is a root shell, since sudo lets the web user run them as root. It also restores `config_secrets.json` to mode 640. + - `configure_wifi_permissions.sh` checks its rules with `visudo -c` before installing them, and grants the NetworkManager captive-portal `cp` and `rm` commands `wifi_manager` runs. + - `configure_web_sudo.sh` uses a random temp file and installs its rules with mode 440. + - The installer prints its completion summary before the `-y` reboot, and describes the setup access point as an open network (it was shown with a password it doesn't have). + - `fix_cache_permissions.sh` applies `setup_cache.sh`'s `ledmatrix`-group model instead of setting 777. + - `check_system_compatibility.sh` reports anything but Debian 13 (Trixie) as unsupported, and reaches its summary. + - New `scripts/README.md` lists every script. +- Docs: + - New `docs/ARCHITECTURE.md` (processes, shared state, display loop, plugin system, web UI) and `docs/PERMISSIONS.md` (owners, modes, both sudoers files, repair scripts). + - Deprecated plugin APIs are marked in the plugin docs. + - `src/common/README.md` covers every module. + - Stale setup, service and troubleshooting claims are corrected. + +- Plugin store and plugin manager fixes: + - Updating a plugin that was installed from a ZIP no longer tries to reinstall it from the LEDMatrix repository's own URL. + - Repository URLs with `.git` in the middle are no longer mangled. The URL helpers now live in `src/plugin_system/repo_urls.py`. + - Installing from a URL works when the repository's only branch isn't `main` or `master`. + - A missing required config field is reported once, by name. + - A plugin that went over `max_memory_mb` once is no longer refused on every call after that. + - `reload_plugin` reads the manifest from the plugin's discovered directory. + - Removed: `last_display` from plugin state info and `get_last_display()` (nothing recorded them); `PluginOperationQueue`'s `history_file` and `lazy_load` arguments; and `data/plugin_operations.json`, which nothing read. + +- Core service fixes: + - `/api/v3/errors` shows each exception's real stack trace instead of `NoneType: None`. + - Wi-Fi disconnect takes the saved connection profile down. + - `wifi_config.json` is written atomically, and a save that fails now gets a 500. + - `plugin://` fonts load from the plugin's own install directory. `FontManager.register_plugin_fonts()` takes an optional `plugin_dir`. + - `APIHelper` keeps cached responses for the `cache_ttl` it was given, instead of always 300 s. + - Logo scales from 0.1 to 10 are honoured everywhere; values outside that range are clamped. + - `LogoHelper` and `logo_downloader`: an empty ESPN logo list counts as a failed download, and the placeholder is written at the requested path. + - Bundled font paths no longer depend on the directory the process was started from. + - Backups record `src.__version__`. + - Removed: `BackgroundDataService`'s `queue_size` stat and `clear_completed_requests()`. + +- Web API fixes: + - A plugin save drops repeated entries in lists whose schema says `uniqueItems`, instead of failing validation. + - `/api/v3/health` reports the real plugin count. + - A malformed `vegas_plugin_order` or `vegas_excluded_plugins` is refused with a 400 and nothing is saved. It used to wipe the saved list. + - The per-plugin health and metrics routes return the display service's latest state. + - Resetting a plugin's config takes a backup first and reports a failed save. + - System metrics that can't be read are `null` everywhere: `cpu_temp` off a Pi, and every metric without psutil, where `/system/status` now answers 200 instead of 503. + - `/plugins/store/refresh` no longer claims a commit-metadata refresh it doesn't do. + - The plugin-config list repair code is in one place, `src/web_interface/config_arrays.py`. - The web service (`ledmatrix-web`) logs through `src.logging_config` like the display service, so `journalctl -p err -u ledmatrix-web` works. Successful @@ -203,6 +239,15 @@ floor on the release that ships them): `api_extractors`). No known plugin imports it. A plugin that does must use `src.common` or its own copy of the code. +- `src.common.frame_timing` -- times every frame the display presents, whoever + drew it, and writes cumulative counters to `/dev/shm`. Two tools read it: + `scripts/frame_soak.py` judges a running service (late frames, freezes, + where the time goes), and `scripts/render_bench.py` judges the hardware and + render path alone on a synthetic strip. Both fail a run above 0.1% late + frames, and both call a loop that never waited for the panel NOT LOCKED. A + stall watchdog logs the stack of whatever holds a scroll up for 250 ms or + more. See `docs/SCROLL_PERFORMANCE.md`, "Soaking a rig". + - `display.scan_order_compensation` (`"auto"` by default): while something scrolls at one pixel per refresh, one half of each panel is shown a refresh behind the other, which removes the 1px step a 1:N-scan panel shows across diff --git a/CLAUDE.md b/CLAUDE.md index ddc8c516..c0805ca1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,14 +13,17 @@ loader does NOT fall back to it — `PluginManager.discover_plugins()` (`src/plugin_system/plugin_manager.py`) scans only the configured directory. Fallbacks exist in two narrower places: store operations - (`StoreManager._find_plugin_path()` in `store_manager.py`) and schema - lookup (`SchemaManager.get_schema_path()` in `schema_manager.py`, - which probes `plugins/` *before* `plugin-repos/`). + (`PluginStoreManager._find_plugin_path()` in `store_manager.py`, which + searches `store_search_dirs()` from `plugin_dirs.py`) and schema lookup + (`SchemaManager.get_schema_path()` in `schema_manager.py`, which probes + `plugins/` *before* `plugin-repos/`). +- `src/plugin_system/plugin_dirs.py` — the one resolver for "which directory + holds plugin X" (manifest `id` first, then `` / `ledmatrix-`) ## Plugin System - Plugins inherit from `BasePlugin` in `src/plugin_system/base_plugin.py` - Required abstract methods: `update()`, `display(force_clear=False)` -- Each plugin needs: `manifest.json`, `config_schema.json`, `manager.py`, `requirements.txt` +- Each plugin needs: `manifest.json`, `config_schema.json`, and the entry point (`manager.py` by default); `requirements.txt` if it has dependencies. Required manifest fields: `docs/PLUGIN_API_REFERENCE.md#manifest-required-fields` - Plugin instantiation args: `plugin_id, config, display_manager, cache_manager, plugin_manager` - Config schemas use JSON Schema Draft-7 - Display dimensions: always read dynamically from `self.display_manager.width/height` — not `display_manager.matrix.width/height`, because `matrix` is `None` when hardware init fails (the properties fall back to the canvas size) @@ -40,14 +43,14 @@ - Official plugins live in the `ledmatrix-plugins` monorepo (not individual repos) - Plugin repo naming convention: `ledmatrix-` (e.g., `ledmatrix-football-scoreboard`) - `plugins.json` registry at `https://raw.githubusercontent.com/ChuckBuilds/ledmatrix-plugins/main/plugins.json` -- Store manager (`src/plugin_system/store_manager.py`) handles install/update/uninstall -- Monorepo plugins are installed via ZIP extraction (no `.git` directory) +- Store manager (`PluginStoreManager` in `src/plugin_system/store_manager.py`) handles install/update/uninstall +- Monorepo plugins are installed without a `.git` directory: GitHub Trees API + raw downloads, falling back to ZIP extraction - Update detection for monorepo plugins uses version comparison (manifest version vs registry latest_version) - Plugin configs stored in `config/config.json`, NOT in plugin directories — safe across reinstalls - Third-party plugins can use their own repo URL with empty `plugin_path` ## Common Pitfalls -- paho-mqtt 2.x needs `callback_api_version=mqtt.CallbackAPIVersion.VERSION1` for v1 compat +- paho-mqtt 2.x requires a `CallbackAPIVersion` argument: `VERSION1` for code written against v1 callback signatures (the MQTT bridge uses `VERSION2`) - BasePlugin uses `get_logger()` from `src.logging_config`, not standard `logging.getLogger()` - `DisplayManager` has no `draw_image()` — paste onto the PIL image directly: `self.display_manager.image.paste(img, (x, y))` then `update_display()` diff --git a/README.md b/README.md index 7ae97dd0..16e66fe7 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,7 @@ This one-shot installer will automatically: - Install required system packages (git, python3, build tools, etc.) - Clone or update the LEDMatrix repository - Run the complete first-time installation script +- Print the web interface address, then **reboot the Pi automatically** (your SSH session will disconnect; give it a few minutes to come back) The installation process typically takes 10-30 minutes depending on your internet connection and Pi model. Pi 3B/3B+ and other 1GB boards land at the top of that range, because the C++ library is compiled serially to stay within available memory. All errors are reported explicitly with actionable fixes. @@ -689,9 +690,10 @@ Controls how long each installed plugin stays visible in seconds before switchin ### Display Format Settings - **`use_short_date_format`** (boolean, default: true) - - Use short date format (e.g., "Jan 15") instead of long format (e.g., "January 15th") - - Set to `false` for longer, more readable dates - - Set to `true` to save space and show more information + - Currently has no effect. The web UI still saves it, but no core code + reads it. Scoreboard plugins that offer a short date format read the + setting from their own plugin config instead. See + [CONFIG_REFERENCE.md](docs/CONFIG_REFERENCE.md#display--other-keys). ### Dynamic Duration Settings (`display.dynamic_duration`) @@ -779,15 +781,21 @@ Controls how long each installed plugin stays visible in seconds before switchin
Manual SSH Commands (for reference) -The quick actions essentially just execute the following commands on the Pi. +The web interface's quick actions (Start/Stop/Restart Display) call +`sudo systemctl start|stop|restart ledmatrix.service` — see +`execute_system_action()` in +[`web_interface/blueprints/api_v3/system.py`](web_interface/blueprints/api_v3/system.py). +The service runs [`run.py`](run.py) as root. -From the project root directory (ex: /home/ledpi/LEDMatrix): +To run the display in the foreground instead (for debugging), stop the service +first, then from the project root (e.g. `/home/ledpi/LEDMatrix`): ```bash -sudo python3 display_controller.py +sudo systemctl stop ledmatrix.service +sudo python3 run.py # add -d for debug logging ``` -This will start the display cycle but only stays active as long as your ssh session is active. +This only runs as long as your SSH session stays open. ### Convenience Scripts @@ -957,9 +965,10 @@ sudo systemctl enable ledmatrix-web.service 3. Check if another service is using port 5000 **Service Fails to Start:** -1. Check Python dependencies are installed -2. Verify the virtual environment is set up correctly -3. Check file permissions and ownership +1. Check Python dependencies are installed. The installer puts them in the + system Python with `pip install --break-system-packages` (there is no + virtual environment), so `python3 -c "import flask"` should succeed. +2. Check file permissions and ownership
diff --git a/docs/ADVANCED_FEATURES.md b/docs/ADVANCED_FEATURES.md index 5a9deb07..6c79cc40 100644 --- a/docs/ADVANCED_FEATURES.md +++ b/docs/ADVANCED_FEATURES.md @@ -185,63 +185,53 @@ their config section to control how oversized content is handled (see ### Plugin Integration (Developer Guide) +All of these have defaults in +[`BasePlugin`](../src/plugin_system/base_plugin.py); override only what you +need. + **1. Implement Content Method:** ```python def get_vegas_content(self): - """ - Return PIL Image or list of Images for Vegas mode. - - Returns: - PIL.Image or list[PIL.Image]: Content to display - - Single image: fixed-width content - - List of images: multiple segments - - None: skip this cycle - """ - # Example: Return single wide image - img = Image.new('RGB', (256, 32)) - # ... render your content ... - return img - - # Example: Return multiple segments - return [image1, image2, image3] + # Return a PIL Image, a list of Images, or None. + # A single image is one block; a list becomes one item per image. + return [self._render_game(game) for game in self.games] ``` +If it returns `None` (the default), Vegas falls back to the plugin's +`scroll_helper` image, then to capturing `display()` output +(`PluginAdapter.get_content()` in +[`src/vegas_mode/plugin_adapter.py`](../src/vegas_mode/plugin_adapter.py)). + **2. Specify Content Type:** ```python def get_vegas_content_type(self): - """ - Specify how content should be handled. - - Returns: - str: 'multi' | 'static' | 'none' - """ - return 'multi' # Default for most plugins + # 'multi' | 'static' | 'none' -- default is 'static' + return 'multi' ``` +`'none'` excludes the plugin from Vegas mode. + **3. Optionally Specify Display Mode:** -```python -def get_vegas_display_mode(self): - """ - Preferred display mode for this plugin. +These return `VegasDisplayMode` members, not strings: - Returns: - str: 'scroll' | 'fixed' | 'static' - """ - return 'scroll' +```python +from src.plugin_system.base_plugin import VegasDisplayMode + +def get_vegas_display_mode(self): + return VegasDisplayMode.SCROLL def get_supported_vegas_modes(self): - """ - List of supported modes. - - Returns: - list: ['scroll', 'fixed', 'static'] - """ - return ['scroll', 'static'] + return [VegasDisplayMode.SCROLL, VegasDisplayMode.STATIC] ``` +`VegasDisplayMode` has `SCROLL` (`"scroll"`), `FIXED_SEGMENT` (`"fixed"`) and +`STATIC` (`"static"`). The default `get_vegas_display_mode()` uses the +plugin's `vegas_mode` config value if set, otherwise maps the content type +(`multi` to `SCROLL`, anything else to `FIXED_SEGMENT`). + ### Content Rendering Guidelines **Image Dimensions:** @@ -966,11 +956,16 @@ from src.cache_manager import CacheManager service = get_background_service(CacheManager()) stats = service.get_statistics() -print(f"Active tasks: {stats['active_tasks']}") -print(f"Completed: {stats['completed']}") -print(f"Failed: {stats['failed']}") +print(f"Active: {stats['active_requests']}") +print(f"Completed: {stats['completed_requests']}") +print(f"Failed: {stats['failed_requests']}") ``` +Other keys: `total_requests`, `cached_hits`, `cache_misses`, +`average_fetch_time`, `completed_requests_count` (results currently held in +memory) — see `BackgroundDataService.get_statistics()` in +[`src/background_data_service.py`](../src/background_data_service.py). + **Enable Debug Logging:** ```python import logging @@ -981,6 +976,10 @@ logging.getLogger('src.background_data_service').setLevel(logging.DEBUG) ## 5. Permission Management +Ownership, modes, sudo rules and the repair scripts are listed in +[PERMISSIONS.md](PERMISSIONS.md). This section covers the helpers code uses +to keep files shareable. + ### Overview LEDMatrix uses a dual-user architecture: the display service runs as root (hardware access), while the web interface runs as a non-privileged user. Centralized permission management ensures both can access necessary files. @@ -1044,7 +1043,7 @@ ensure_file_permissions(config_path, get_config_file_mode(config_path)) | Config (secrets) | `rw-r-----` | `0o640` | Owner write, group read | | Assets | `rw-rw-r--` | `0o664` | Owner/group write, all read | | Plugins | `rw-rw-r--` | `0o664` | Owner/group write, all read | -| Cache files | `rw-rw-r--` | `0o664` | Owner/group write, all read | +| Cache files | `rw-rw----` | `0o660` | Owner/group write, no world access (`_CACHE_FILE_MODE` in `src/cache/disk_cache.py`) | **Directory Permissions:** @@ -1115,40 +1114,22 @@ These core utilities **already handle permissions** - you don't need to call per ### Manual Fixes -If you encounter permission issues: +[PERMISSIONS.md](PERMISSIONS.md) lists who owns what on an installed system, +the expected modes, and which `scripts/fix_perms/` script to run as which +user. In short: -```bash -# Targeted permission fixes (see scripts/fix_perms/README.md) -sudo ./scripts/fix_perms/fix_assets_permissions.sh # assets/ tree (logos, fonts) -sudo ./scripts/fix_perms/fix_cache_permissions.sh # all cache directories -sudo ./scripts/fix_perms/fix_plugin_permissions.sh # plugin directories -sudo ./scripts/fix_perms/fix_web_permissions.sh # web interface files +- `fix_assets_permissions.sh`, `fix_cache_permissions.sh` and + `fix_plugin_permissions.sh` are run with `sudo`. +- `fix_web_permissions.sh` is run as the web interface user, without + `sudo` (it refuses to run as root and calls `sudo` itself where needed). + It resets project file ownership for that user, then makes the two + helper scripts the web user may run as root (`safe_plugin_rm.sh`, + `safe_pip_install.sh`) root-owned again and restores `config_secrets.json` + to its owner, the `ledmatrix` group and mode `640`. It does not write + sudoers rules; `scripts/install/configure_web_sudo.sh` does that. -# Fix specific directory -sudo chown -R ledpi:ledpi /home/ledpi/LEDMatrix/config -sudo chmod -R 2775 /home/ledpi/LEDMatrix/config -sudo find /home/ledpi/LEDMatrix/config -type f -exec chmod 664 {} \; - -# Verify permissions -ls -la config/ -ls -la assets/ -``` - -### Verification - -```bash -# Check directory has setgid bit -ls -ld assets/ -# Should show: drwxrwsr-x (note the 's') - -# Check file has correct group -ls -l assets/logo.png -# Should show group 'ledpi' - -# Check file permissions -stat -c "%a %n" config/config.json -# Should show: 644 config/config.json -``` +Do not `chmod` the whole `config/` directory: `config_secrets.json` must stay +`640`. --- diff --git a/docs/ADVANCED_PLUGIN_DEVELOPMENT.md b/docs/ADVANCED_PLUGIN_DEVELOPMENT.md index b92e1649..20d5a792 100644 --- a/docs/ADVANCED_PLUGIN_DEVELOPMENT.md +++ b/docs/ADVANCED_PLUGIN_DEVELOPMENT.md @@ -13,7 +13,7 @@ Advanced patterns, examples, and best practices for developing LEDMatrix plugins - [Using Weather Icons](#using-weather-icons) - [Implementing Scrolling with Deferred Updates](#implementing-scrolling-with-deferred-updates) - [Cache Strategy Patterns](#cache-strategy-patterns) -- [Font Management and Overrides](#font-management-and-overrides) +- [Font Management](#font-management) - [Error Handling Best Practices](#error-handling-best-practices) - [Performance Optimization](#performance-optimization) - [Testing Plugins with Mocks](#testing-plugins-with-mocks) @@ -25,69 +25,12 @@ Advanced patterns, examples, and best practices for developing LEDMatrix plugins ## Using Weather Icons -The Display Manager provides built-in weather icon drawing methods for easy visual representation of weather conditions. - -### Basic Weather Icon Usage - -```python -def display(self, force_clear=False): - if force_clear: - self.display_manager.clear() - - # Draw weather icon based on condition - condition = self.data.get('condition', 'clear') - self.display_manager.draw_weather_icon(condition, x=5, y=5, size=16) - - # Draw temperature next to icon - temp = self.data.get('temp', 72) - self.display_manager.draw_text( - f"{temp}°F", - x=25, y=10, - color=(255, 255, 255) - ) - - self.display_manager.update_display() -``` - -### Supported Weather Conditions - -The `draw_weather_icon()` method automatically maps condition strings to appropriate icons: - -- `"clear"`, `"sunny"` → Sun icon -- `"clouds"`, `"cloudy"`, `"partly cloudy"` → Cloud icon -- `"rain"`, `"drizzle"`, `"shower"` → Rain icon -- `"snow"`, `"sleet"`, `"hail"` → Snow icon -- `"thunderstorm"`, `"storm"` → Storm icon - -### Custom Weather Icons - -For more control, use individual icon methods: - -```python -# Draw specific icons -self.display_manager.draw_sun(x=10, y=10, size=16) -self.display_manager.draw_cloud(x=10, y=10, size=16, color=(150, 150, 150)) -self.display_manager.draw_rain(x=10, y=10, size=16) -self.display_manager.draw_snow(x=10, y=10, size=16) -``` - -### Text with Weather Icons - -Use `draw_text_with_icons()` to combine text and icons: - -```python -icons = [ - ("sun", 5, 5), # Sun icon at (5, 5) - ("cloud", 100, 5) # Cloud icon at (100, 5) -] - -self.display_manager.draw_text_with_icons( - "Weather: Sunny, Cloudy", - icons=icons, - x=10, y=20, - color=(255, 255, 255) -) -``` +The Display Manager's icon methods — `draw_weather_icon()`, `draw_sun()`, +`draw_cloud()`, `draw_rain()`, `draw_snow()` and `draw_text_with_icons()` — +are deprecated, removed in 3.7.0. Draw your own icons instead: render them +onto a PIL image and paste it onto `self.display_manager.image`, or ship +icon images with the plugin. The weather plugin's `WeatherIcons` class is an +example. See [Deprecated APIs](PLUGIN_API_REFERENCE.md#deprecated-apis). --- @@ -251,11 +194,8 @@ def update(self): sport_key = "nhl" cache_key = f"{self.plugin_id}_{sport_key}_games" - # Uses sport-specific live_update_interval from config - cached = self.cache_manager.get_background_cached_data( - cache_key, - sport_key=sport_key - ) + # get_background_cached_data() is deprecated, removed in 3.7.0 — use get() + cached = self.cache_manager.get(cache_key, max_age=60) if cached: self.games = cached @@ -282,9 +222,9 @@ def on_config_change(self, new_config): --- -## Font Management and Overrides +## Font Management -Use the Font Manager for advanced font handling and user customization. +The display manager's built-in fonts and text measurement. For fonts shipped with a plugin, see [FONT_MANAGER.md](FONT_MANAGER.md). ### Using Different Fonts @@ -656,14 +596,12 @@ def update(self): ```python def update(self): - # Check if another plugin is enabled - enabled_plugins = self.plugin_manager.get_enabled_plugins() - if "weather" in enabled_plugins: - # Weather plugin is available - weather_plugin = self.plugin_manager.get_plugin("weather") - if weather_plugin: - # Use weather data - pass + # get_enabled_plugins() is deprecated, removed in 3.7.0 — check the + # instance's `enabled` flag instead + weather_plugin = self.plugin_manager.get_plugin("weather") + if weather_plugin is not None and weather_plugin.enabled: + # Use weather data + pass ``` ### Sharing Data Between Plugins diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 00000000..6a8918b1 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,215 @@ +# Architecture + +A map of the codebase for a new contributor: which process does what, how +they talk to each other, and where to start reading for common changes. + +## Processes + +| systemd unit | Runs as | Runs | Installed by | +|---|---|---|---| +| `ledmatrix.service` | root | [`run.py`](../run.py) → `DisplayController` | [`install_service.sh`](../scripts/install/install_service.sh) | +| `ledmatrix-web.service` | the installing user | [`start_web_conditionally.py`](../scripts/utils/start_web_conditionally.py) → [`web_interface/start.py`](../web_interface/start.py) (Flask, port 5000) | `install_service.sh`, [`install_web_service.sh`](../scripts/install/install_web_service.sh) | +| `ledmatrix-update-verify.path` / `.service` | the web user | Health check after an automatic update | the same installers, or [`src/auto_update_setup.py`](../src/auto_update_setup.py) at runtime | +| `ledmatrix-wifi-monitor.service` | root | [`wifi_monitor_daemon.py`](../scripts/utils/wifi_monitor_daemon.py) | [`install_wifi_monitor.sh`](../scripts/install/install_wifi_monitor.sh) | +| `ledmatrix-mqtt-bridge.service` | root | [MQTT bridge](../integrations/mqtt_bridge/README.md) (optional) | [`install_mqtt_bridge.sh`](../scripts/install/install_mqtt_bridge.sh) | +| `ledmatrix-dns-fix.service` | root | DNS workaround (optional) | [`install_dns_fix.sh`](../scripts/install/install_dns_fix.sh) | + +Unit templates are in [`systemd/`](../systemd/README.md). The display runs as +root because the LED matrix library needs direct GPIO access. The web +interface runs unprivileged and uses a fixed list of `sudo` rules for the +few privileged things it does; see [PERMISSIONS.md](PERMISSIONS.md). + +`start_web_conditionally.py` exits without starting Flask when +`web_display_autostart` is explicitly false in `config.json`. + +## How the two main processes share state + +The display and the web interface are separate processes that never call +each other. They share three things: + +1. **`config/config.json` and `config/config_secrets.json`.** The web + interface writes them through `ConfigManager` + ([`src/config_manager.py`](../src/config_manager.py)); the display + notices through `ConfigService` (below). +2. **The disk cache**, `/var/cache/ledmatrix` (owned `root:ledmatrix`, + setgid, files `0660`), read and written through `CacheManager` + ([`src/cache_manager.py`](../src/cache_manager.py), + [`src/cache/disk_cache.py`](../src/cache/disk_cache.py)). Readers in the + other process pass `memory_ttl=0` so they do not serve a stale in-memory + copy. +3. **A few files in `/tmp`.** + +| State | Where | Written by | Read by | +|---|---|---|---| +| On-demand request | cache `display_on_demand_request` | web: `start_on_demand_display()` / `stop_on_demand_display()` in [`api_v3/display.py`](../web_interface/blueprints/api_v3/display.py) | display: `_poll_on_demand_requests()` | +| On-demand state | cache `display_on_demand_state` | display: `_publish_on_demand_state()` | web: `/api/v3/display/on-demand/status` | +| Current screen | cache `display_current_state` | display | web: `/api/v3/display/current-status` | +| Plugin errors | cache `plugin_error_snapshot` | display: `ErrorSnapshotPublisher` ([`src/error_aggregator.py`](../src/error_aggregator.py)) | web: `read_error_report()` for `/api/v3/errors/*` | +| Error clear | cache `plugin_error_clear_request` | web | display | +| Font usage | cache `font_usage_snapshot` | display: `FontUsagePublisher` ([`src/font_usage.py`](../src/font_usage.py)) | web: Fonts tab | +| Plugin health | cache `plugin_health:` | display (web writes on reset) | web: `/api/v3/plugins/health` | +| Preview frame | `/tmp/led_matrix_preview.png` | display: `DisplayManager`, gated by [`snapshot_policy`](../src/common/snapshot_policy.py) | web: display SSE stream, `/api/v3/health` (file age) | +| Preview viewer marker | `/tmp/led_matrix_preview_viewer` | web, while a preview is open | display: writes full-rate snapshots only while it is fresh | +| Hardware init status | `/tmp/led_matrix_hw_status.json` | display | web: `/api/v3/hardware/status` | + +The on-demand start route also restarts `ledmatrix.service` by default so the +request takes effect straight away. + +## Display loop + +[`src/display_controller.py`](../src/display_controller.py), class +`DisplayController`. `__init__` loads config, starts the cache and the +error-snapshot publisher, runs the startup validator, creates the +`DisplayManager` ([`src/display_manager.py`](../src/display_manager.py)), +`FontManager` and `PluginManager`, loads the enabled plugins in parallel, +runs an initial `update()` pass within a 20-second budget +(`_INITIAL_UPDATE_BUDGET_SECONDS`; a plugin that misses it is deferred to +the scheduler), and sets up Vegas mode. + +`run()` is the main loop. Each pass, in order: apply a pending plugin +enable/disable, poll on-demand requests, run scheduled plugin updates, check +the on/off schedule and brightness, then show one screen. Priority is +on-demand, then WiFi status messages, then live priority, then Vegas mode, +then normal rotation. + +- **Rotation.** `available_modes` is the ordered list of display modes; + `current_mode_index` advances after each screen. + `_apply_plugin_rotation_order()` applies `display.plugin_rotation_order`. +- **Durations.** `_get_display_duration()`: `display.display_durations[mode]`, + else the plugin's `get_display_duration()`, else 30 s. Plugins that + support dynamic duration run until `is_cycle_complete()`, capped by + `display.dynamic_duration.max_duration_seconds` (default 180 s). +- **On-demand.** A request from the web interface pins one plugin (or mode) + for a duration. `_activate_on_demand()` / `_clear_on_demand()`; the + session is saved under `display_on_demand_config` so it survives a + restart. It also keeps the display on during scheduled off hours. +- **Live priority.** `_check_live_priority()` looks for a plugin whose + `has_live_priority()` and `has_live_content()` are both true and switches + to it, rotating between several live games. +- **Schedule and dim schedule.** `_check_schedule()` reads `schedule`; + `_check_dim_schedule()` reads `dim_schedule` and + `display.hardware.brightness`. Both are re-evaluated once a minute. +- **Long screens.** While a screen is showing (a dwell, a scroll, a Vegas + iteration), `_service_pending_changes()` repeats the on-demand, schedule + and brightness checks every 0.25 s, so a change does not wait for the + screen to end. +- **Config hot reload.** `ConfigService` + ([`src/config_service.py`](../src/config_service.py)) polls the config and + secrets files' mtimes every 2 s and notifies subscribers when the content + changes. The controller refreshes its cached settings; enabling or + disabling a plugin queues `_reconcile_enabled_plugins()`, which loads or + unloads it on the display thread; each plugin gets `on_config_change()` + for its own section. Set `LEDMATRIX_HOT_RELOAD=false` to turn this off. + Matrix hardware settings are only read at start-up. +- **Vegas mode.** [`src/vegas_mode/`](../src/vegas_mode/): the display loop + calls `VegasModeCoordinator.run_iteration()` + ([`coordinator.py`](../src/vegas_mode/coordinator.py)) when + `display.vegas_scroll.enabled` is set. `PluginAdapter` gets each plugin's + content (`get_vegas_content()`, else its `scroll_helper` image, else a + capture of `display()`), `StreamManager` orders it and `RenderPipeline` + scrolls it. See [ADVANCED_FEATURES.md](ADVANCED_FEATURES.md). +- **Multi-display sync.** `DisplaySyncManager` + ([`src/common/sync_manager.py`](../src/common/sync_manager.py)), enabled by + `sync.role`: a leader sends a follower its share of each frame over UDP + (port 5765). + +## Plugin system + +[`src/plugin_system/`](../src/plugin_system/): + +| Area | Where | +|---|---| +| Base class plugins implement | [`base_plugin.py`](../src/plugin_system/base_plugin.py) (`BasePlugin`, `VegasDisplayMode`) | +| Finding a plugin's directory | [`plugin_dirs.py`](../src/plugin_system/plugin_dirs.py): manifest `id` first, then directory `` or `ledmatrix-` | +| Discovery, load, unload, scheduled updates | [`plugin_manager.py`](../src/plugin_system/plugin_manager.py) (`PluginManager`) | +| Import and instantiate | [`plugin_loader.py`](../src/plugin_system/plugin_loader.py) (`PluginLoader.load_plugin()`: dependencies, module, class) | +| Timeouts | [`plugin_executor.py`](../src/plugin_system/plugin_executor.py) (`PluginExecutor`, 30 s default; a timed-out thread is abandoned, not killed) | +| Circuit breaker | [`plugin_health.py`](../src/plugin_system/plugin_health.py) (`PluginHealthTracker`: 3 consecutive failures open the circuit for 300 s) | +| Resource metrics | [`resource_monitor.py`](../src/plugin_system/resource_monitor.py) | +| Config schemas and defaults | [`schema_manager.py`](../src/plugin_system/schema_manager.py) | +| Install, update, uninstall | [`store_manager.py`](../src/plugin_system/store_manager.py) (`PluginStoreManager`) | +| Core-version gate | [`compatibility.py`](../src/plugin_system/compatibility.py) | + +Discovery scans only `plugin_system.plugins_directory` (default +`plugin-repos/`). Scheduled `update()` calls run on one background worker +thread; a per-plugin lock keeps `display()` from running during an update. + +**Store flow.** `install_plugin()` renames any existing copy aside +(`.standalone-backup-preinstall`), installs the new one, and puts the old +copy back if the install fails. Monorepo plugins come from the GitHub Trees +API, falling back to the repository ZIP; other plugins by `git clone` or +download. The manifest is checked (see +[required fields](PLUGIN_API_REFERENCE.md#manifest-required-fields)), the core +version gate runs, then dependencies are installed as root through +`scripts/fix_perms/safe_pip_install.sh`. `update_plugin()` pulls git +installs, undoing a pull whose new version is incompatible, and reinstalls +everything else through `_reinstall_with_rollback()`. + +## Web interface + +- **App.** [`web_interface/app.py`](../web_interface/app.py) builds the + Flask `app` at import time, creates the managers, and registers two + blueprints. `web_interface/start.py` runs it on port 5000. +- **Pages.** [`blueprints/pages_v3.py`](../web_interface/blueprints/pages_v3.py) + serves the shell `templates/v3/base.html` at `/` and each tab as a + partial at `/partials/` (templates in + `web_interface/templates/v3/partials/`). Plugin configuration tabs are + rendered from the plugin's schema by `plugin_config.html`. +- **API.** [`blueprints/api_v3/`](../web_interface/blueprints/api_v3/) is one + blueprint at `/api/v3`, split by area: `backup.py`, `config.py`, + `display.py`, `fonts.py`, `misc.py` (health, logs, errors, cache, sync), + `plugins.py`, `starlark.py`, `system.py` (service actions, updates, git), + `wifi.py`. `__init__.py` defines the blueprint and shared helpers and + imports the modules so their routes register. Endpoints are listed in + [REST_API_REFERENCE.md](REST_API_REFERENCE.md). +- **Front end.** HTMX loads each tab's partial on first open + (`hx-trigger="loadtab"`); Alpine.js holds page state. Scripts are in + `web_interface/static/v3/js/`; form widgets are bundled from + [`js/widgets/`](../web_interface/static/v3/js/widgets/README.md). +- **Server-sent events** (`app.py`): `/api/v3/stream/stats` (CPU, memory, + temperature, service state, every 10 s), `/api/v3/stream/display` (preview + frames when the PNG changes) and `/api/v3/stream/logs` (journal of both + services). One generator thread per stream is shared by all clients. + +## Updates + +- **Update Code** on the Overview tab and the automatic updater both call + `perform_core_update()` in + [`api_v3/system.py`](../web_interface/blueprints/api_v3/system.py): + `git pull --rebase`, reinstall changed requirement files, report whether a + restart is needed. +- **Automatic updates** (`auto_update.enabled`, off by default): + `AutoUpdater` in [`web_interface/auto_update.py`](../web_interface/auto_update.py) + runs in the web process, checks every 30 minutes, and updates at most + weekly between 02:00 and 05:00. Before pulling it copies + [`scripts/utils/auto_update_verify.py`](../scripts/utils/auto_update_verify.py) + to `data/auto_update_verifier.py`, then writes + `data/auto_update_verify.request`. That file triggers + `ledmatrix-update-verify.path`, which runs the verifier as a separate unit + (so restarting the web service does not kill it). The verifier restarts + both services, waits for the web API to answer and the display service to + stay up, and on failure resets to the previous commit and restarts again. + Plugin updates run only after a verified core update. State is in + `data/auto_update_state.json` and `data/auto_update_pending.json`. +- **Startup validator.** `StartupValidator` + ([`src/startup_validator.py`](../src/startup_validator.py)) runs twice in + `DisplayController.__init__`: config and cache directory first, then + enabled plugins once the plugin manager exists. It also warns when an + installed systemd unit differs from its template in `systemd/`. Results + are logged; startup continues either way. + +## Where to start reading + +| Task | Start with | +|---|---| +| Change rotation, durations or priorities | `DisplayController.run()` and `_get_display_duration()` in [`display_controller.py`](../src/display_controller.py) | +| Add a config key | [CONFIG_REFERENCE.md](CONFIG_REFERENCE.md), [`config/config.template.json`](../config/config.template.json), the tab's partial and `api_v3/config.py` | +| Change drawing or fonts | [`display_manager.py`](../src/display_manager.py), [`font_manager.py`](../src/font_manager.py), [`src/common/bdf_font.py`](../src/common/bdf_font.py) | +| Add a plugin-facing API | [`base_plugin.py`](../src/plugin_system/base_plugin.py) or [`src/common/`](../src/common/README.md); document it in [PLUGIN_API_REFERENCE.md](PLUGIN_API_REFERENCE.md) | +| Plugin install/update bugs | `PluginStoreManager` in [`store_manager.py`](../src/plugin_system/store_manager.py) | +| A plugin that won't load | `PluginManager.load_plugin()` and `PluginLoader.load_plugin()`; `python3 scripts/check_plugin.py --plugin ` | +| Add an API endpoint | the matching module in [`api_v3/`](../web_interface/blueprints/api_v3/) | +| Add a web UI tab or control | `templates/v3/base.html`, the tab's partial, `pages_v3.py` | +| Vegas scroll | [`src/vegas_mode/coordinator.py`](../src/vegas_mode/coordinator.py) | +| Installer or permissions | [`first_time_install.sh`](../first_time_install.sh), [`scripts/install/`](../scripts/install/), [PERMISSIONS.md](PERMISSIONS.md) | +| Work without a Pi | [DEV_PREVIEW.md](DEV_PREVIEW.md), [EMULATOR_SETUP_GUIDE.md](EMULATOR_SETUP_GUIDE.md), [HOW_TO_RUN_TESTS.md](HOW_TO_RUN_TESTS.md) | diff --git a/docs/CONFIG_DEBUGGING.md b/docs/CONFIG_DEBUGGING.md index 01b41224..4db3d051 100644 --- a/docs/CONFIG_DEBUGGING.md +++ b/docs/CONFIG_DEBUGGING.md @@ -172,10 +172,14 @@ ERROR - Plugin football-scoreboard configuration validation failed: 'api_key' is ### Enable Debug Logging -Set environment variable: +Run the display in the foreground with `-d`, or set `LEDMATRIX_DEBUG=true` +(the value must be `true`; `1` is ignored — see `setup_logging()` in +[`src/logging_config.py`](../src/logging_config.py)): ```bash -export LEDMATRIX_DEBUG=1 -python run.py +sudo systemctl stop ledmatrix.service +sudo python3 run.py -d +# or +sudo LEDMATRIX_DEBUG=true python3 run.py ``` ### Check Merged Configuration @@ -321,8 +325,10 @@ cp config/backups/config.json.backup.20240115_120000_000000 config/config.json ## Getting Help -1. Check logs: `tail -f logs/ledmatrix.log` -2. Enable debug: `LEDMATRIX_DEBUG=1` +1. Check logs. Both services log to journald, not to a file: + `sudo journalctl -u ledmatrix.service -f` (display) and + `sudo journalctl -u ledmatrix-web.service -f` (web interface) +2. Enable debug: `LEDMATRIX_DEBUG=true` or `python3 run.py -d` 3. Check error dashboard: `/api/v3/errors/summary` 4. Validate JSON: https://jsonlint.com/ 5. File an issue: https://github.com/ChuckBuilds/LEDMatrix/issues diff --git a/docs/DEVELOPER_QUICK_REFERENCE.md b/docs/DEVELOPER_QUICK_REFERENCE.md index 7eefb0c9..ed2c92e9 100644 --- a/docs/DEVELOPER_QUICK_REFERENCE.md +++ b/docs/DEVELOPER_QUICK_REFERENCE.md @@ -54,8 +54,8 @@ rows = self.layout.bounds.inset(1).split_v(3, 1, gap=1) self.draw_fit("12:34", rows[0]) # largest crisp font that fits self.draw_image(logo, rows[1], mode="fill_height", crop_to_ink=True) -# Weather icons -display_manager.draw_weather_icon("rain", x=10, y=10, size=16) +# Weather icons: draw_weather_icon() is deprecated, removed in 3.7.0 — +# draw your own icons (the weather plugin ships WeatherIcons) # Scrolling state display_manager.set_scrolling_state(True) @@ -72,20 +72,23 @@ cache_manager.delete("key") # alias for clear_cache(key) # Advanced caching data = cache_manager.get_cached_data_with_strategy("key", data_type="weather") -data = cache_manager.get_background_cached_data("key", sport_key="nhl") # Strategy strategy = cache_manager.get_cache_strategy("weather") -interval = cache_manager.get_sport_live_interval("nhl") ``` +`get_background_cached_data()` (use `get()`) and `get_sport_live_interval()` +are deprecated, removed in 3.7.0. See +[Deprecated APIs](PLUGIN_API_REFERENCE.md#deprecated-apis). + ## Plugin Manager Quick Methods ```python # Get plugins plugin = plugin_manager.get_plugin("plugin-id") all_plugins = plugin_manager.get_all_plugins() -enabled = plugin_manager.get_enabled_plugins() +# get_enabled_plugins() is deprecated, removed in 3.7.0 — check `enabled` +# on the entries in plugin_manager.plugins # Get info info = plugin_manager.get_plugin_info("plugin-id") @@ -168,7 +171,7 @@ def display(self, force_clear=False): - [ ] Plugin inherits from `BasePlugin` - [ ] Implements `update()` and `display()` methods -- [ ] `manifest.json` with required fields +- [ ] `manifest.json` with the [required fields](PLUGIN_API_REFERENCE.md#manifest-required-fields) - [ ] `config_schema.json` for web UI (recommended) - [ ] `README.md` with documentation - [ ] Error handling implemented diff --git a/docs/FONT_MANAGER.md b/docs/FONT_MANAGER.md index 0d552687..3390b1ac 100644 --- a/docs/FONT_MANAGER.md +++ b/docs/FONT_MANAGER.md @@ -9,12 +9,14 @@ ## Overview -The enhanced FontManager provides comprehensive font management for the LEDMatrix application with support for: -- Manager font registration and detection -- Plugin font management -- Programmatic per-element font overrides -- Performance monitoring and caching -- Dynamic font discovery +[`src/font_manager.py`](../src/font_manager.py) loads and caches the TTF and +BDF fonts in `assets/fonts/`, registers fonts that plugins ship, and records +which plugin uses which font so the web UI can show it. + +Several methods are deprecated and will be removed in LEDMatrix 3.7.0; they +log a warning on first call. They are listed in +[Deprecated methods](#deprecated-methods) below, and the full set is pinned in +[`test/test_deprecation.py`](../test/test_deprecation.py). ## Getting the FontManager @@ -34,157 +36,60 @@ standalone FontManager when none is available (test harnesses, mocks). `DisplayManager` has **no** `font_manager` attribute — `display_manager.font_manager` raises `AttributeError`. -## Architecture - -### Manager-Centric Design - -Managers define their own fonts, but the FontManager: -1. **Loads and caches fonts** for performance -2. **Detects font usage** for visibility -3. **Allows manual overrides** when needed -4. **Supports plugin fonts** with namespacing - -### Font Resolution Flow - -``` -Manager requests font → Check manual overrides → Apply manager choice → Cache & return -``` - -## For Manager Developers - -### Basic Font Usage +## Resolving a font ```python -from src.font_manager import FontManager +element_key = f"{self.plugin_id}.title" -class MyManager: - def __init__(self, config, display_manager, cache_manager, plugin_manager): - self.display_manager = display_manager - self.font_manager = plugin_manager.font_manager # Shared FontManager - self.manager_id = "my_manager" - - def display(self): - # Define your font choices - element_key = "my_manager.title" - font_family = "press_start" - font_size_px = 10 - color = (255, 255, 255) # RGB white - - # Register your font choice (for detection and future overrides) - self.font_manager.register_manager_font( - manager_id=self.manager_id, - element_key=element_key, - family=font_family, - size_px=font_size_px, - color=color - ) - - # Get the font (checks for manual overrides automatically) - font = self.font_manager.resolve_font( - element_key=element_key, - family=font_family, - size_px=font_size_px - ) - - # Use the font for rendering - self.display_manager.draw_text( - "Hello World", - x=10, y=10, - color=color, - font=font - ) -``` - -### Advanced Font Usage - -```python -class AdvancedManager: - def __init__(self, config, display_manager, cache_manager, plugin_manager): - self.display_manager = display_manager - self.font_manager = plugin_manager.font_manager - self.manager_id = "advanced_manager" - - # Define your font specifications - self.font_specs = { - "title": {"family": "press_start", "size_px": 12, "color": (255, 255, 0)}, - "body": {"family": "four_by_six", "size_px": 8, "color": (255, 255, 255)}, - "footer": {"family": "five_by_seven", "size_px": 7, "color": (128, 128, 128)} - } - - # Register all font specs - for element_type, spec in self.font_specs.items(): - element_key = f"{self.manager_id}.{element_type}" - self.font_manager.register_manager_font( - manager_id=self.manager_id, - element_key=element_key, - family=spec["family"], - size_px=spec["size_px"], - color=spec["color"] - ) - - def get_font(self, element_type: str): - """Helper method to get fonts with override support.""" - spec = self.font_specs[element_type] - element_key = f"{self.manager_id}.{element_type}" - - return self.font_manager.resolve_font( - element_key=element_key, - family=spec["family"], - size_px=spec["size_px"] - ) - - def display(self): - # Get fonts (automatically checks for overrides) - title_font = self.get_font("title") - body_font = self.get_font("body") - footer_font = self.get_font("footer") - - # Render with fonts - self.display_manager.draw_text("Title", font=title_font, color=self.font_specs["title"]["color"]) - self.display_manager.draw_text("Body Text", font=body_font, color=self.font_specs["body"]["color"]) - self.display_manager.draw_text("Footer", font=footer_font, color=self.font_specs["footer"]["color"]) -``` - -### Using Size Tokens - -```python -# Get available size tokens -tokens = self.font_manager.get_size_tokens() -# Returns: {'xs': 6, 'sm': 8, 'md': 10, 'lg': 12, 'xl': 14, 'xxl': 16} - -# Use token to get size -size_px = tokens.get('md', 10) # 10px - -# Then use in font resolution -font = self.font_manager.resolve_font( - element_key="my_manager.text", +# Register the choice so the web UI's Fonts tab can list it. +self.font_manager.register_manager_font( + manager_id=self.plugin_id, + element_key=element_key, family="press_start", - size_px=size_px + size_px=10, + color=(255, 255, 255), ) + +font = self.font_manager.resolve_font( + element_key=element_key, + family="press_start", + size_px=10, +) +self.display_manager.draw_text("Hello", x=10, y=10, font=font) ``` -## For Plugin Developers +`resolve_font()` applies any entry for `element_key` in +`config/font_overrides.json`, maps a plugin-local family to its namespaced +name when `plugin_id` is passed, and then calls `get_font(family, size_px)`. +On error it returns a fallback font rather than raising. -> **Note**: plugins that ship their own fonts via a `"fonts"` block -> in `manifest.json` are registered automatically during plugin load -> (`src/plugin_system/plugin_manager.py` calls -> `FontManager.register_plugin_fonts()`). The `plugin://…` source -> URIs documented below are resolved relative to the plugin's -> install directory. -> -> The web UI's **Fonts** tab lists, uploads, previews and deletes the -> font files in `assets/fonts/`. Its **Used by** column shows which -> loaded plugins registered each file through `register_manager_font()` -> (see [Font usage in the web UI](#font-usage-in-the-web-ui)), and it -> warns before deleting one of them. It has no override editor (the -> override panels and `/api/v3/fonts/overrides` endpoints were removed). -> The programmatic override workflow in -> [Manual Font Overrides](#manual-font-overrides) below still works. -> Let users pick fonts through your plugin's own config schema. +`get_font(family, size_px)` looks the family up in `font_catalog` and loads +it (cached per family and size). -### Plugin Font Registration +## Font families -In your plugin's `manifest.json`: +At start-up the FontManager scans `assets/fonts/` for `.ttf` and `.bdf` +files. Each becomes a family named after the file, lower-cased and without +the extension (`PressStart2P-Regular.ttf` → `pressstart2p-regular`). Four +aliases are added on top: + +| Alias | File | +|---|---| +| `press_start` | `assets/fonts/PressStart2P-Regular.ttf` | +| `four_by_six` | `assets/fonts/4x6-font.ttf` | +| `five_by_seven` | `assets/fonts/5x7.bdf` | +| `tom_thumb` | `assets/fonts/tom-thumb.bdf` | + +Read the catalog directly: `font_manager.font_catalog` is a dict of family +name to file path. Files added later are picked up on the next start of the +display service. + +## Plugin fonts + +Plugins that ship their own fonts declare them in a `"fonts"` block in +`manifest.json`. The plugin manager calls +`FontManager.register_plugin_fonts()` during plugin load. `plugin://…` +sources are resolved relative to the plugin's install directory. ```json { @@ -195,231 +100,123 @@ In your plugin's `manifest.json`: { "family": "custom_font", "source": "plugin://fonts/custom.ttf", - "metadata": { - "description": "Custom plugin font", - "license": "MIT" - } + "metadata": {"description": "Custom plugin font", "license": "MIT"} }, { "family": "web_font", "source": "https://example.com/fonts/font.ttf", - "metadata": { - "description": "Downloaded font", - "checksum": "sha256:abc123..." - } + "metadata": {"checksum": "sha256:abc123..."} } ] } } ``` -### Using Plugin Fonts +Registered families are namespaced as `::`. Pass +`plugin_id` to `resolve_font()` to use the short name: ```python -class MyPlugin(BasePlugin): - def __init__(self, plugin_id, config, display_manager, cache_manager, plugin_manager): - super().__init__(plugin_id, config, display_manager, cache_manager, plugin_manager) - self.font_manager = self._get_font_manager() - - def display(self): - # Use plugin font (automatically namespaced) - font = self.font_manager.resolve_font( - element_key=f"{self.plugin_id}.text", - family="custom_font", # Will be resolved as "my-plugin::custom_font" - size_px=10, - plugin_id=self.plugin_id - ) - - self.display_manager.draw_text("Plugin Text", font=font) -``` - -## Manual Font Overrides - -Overrides are set in code (there is no web UI or REST endpoint for them). -They are stored in `config/font_overrides.json` and persist across restarts. - -### Programmatic Overrides - -```python -# Set override -font_manager.set_override( - element_key="nfl.live.score", - family="four_by_six", - size_px=8 +font = self.font_manager.resolve_font( + element_key=f"{self.plugin_id}.text", + family="custom_font", # resolved as "my-plugin::custom_font" + size_px=10, + plugin_id=self.plugin_id, ) - -# Remove override -font_manager.remove_override("nfl.live.score") - -# Get all overrides -overrides = font_manager.get_overrides() ``` -## Font Discovery +## Overrides -### Available Fonts - -The FontManager automatically scans `assets/fonts/` for TTF and BDF fonts: - -```python -# Get all available fonts -fonts = font_manager.get_available_fonts() -# Returns: {'press_start': 'assets/fonts/PressStart2P-Regular.ttf', ...} - -# Check if font exists -if "my_font" in fonts: - font = font_manager.get_font("my_font", 10) -``` - -### Adding Custom Fonts - -Place font files in `assets/fonts/` directory: -- Supported formats: `.ttf`, `.bdf` -- Font family name is derived from filename (without extension) -- Will be automatically discovered on next initialization +`resolve_font()` still honours `config/font_overrides.json` (a map of +element key to `family` and/or `size_px`), which is read once at start-up. +The methods that edit it — `set_override()`, `remove_override()`, +`get_overrides()` — are deprecated, and there is no web UI or REST endpoint +for overrides (the override editor and `/api/v3/fonts/overrides` were +removed). To let users choose a font, add a field to your plugin's config +schema. ## Font usage in the web UI -The web interface runs in its own process and has no FontManager, so the -display service publishes which plugin uses which font -(`src/font_usage.py`), and the Fonts tab's **Used by** column reads it: +The web UI's **Fonts** tab lists, uploads, previews and deletes the font +files in `assets/fonts/`. The web interface runs in its own process and has +no FontManager, so the display service publishes which plugin uses which +font ([`src/font_usage.py`](../src/font_usage.py)), and the tab's **Used by** +column reads it: - **Source**: `register_manager_font()` registrations of the loaded plugins. `get_font()` and `resolve_font()` do not know the calling plugin and are not counted, and neither is a plugin that opens a font file directly with PIL — register the fonts your plugin draws with if you want them listed. -- **Names**: a family, alias (`press_start`, `four_by_six`, - `five_by_seven`, `tom_thumb`) or path is resolved through - `font_catalog` to the file it loads and reported under that file's name - without extension (`PressStart2P-Regular`, `4x6-font`, `5x7`, - `tom-thumb`), which is how the Fonts tab keys its rows. Fonts outside - `assets/fonts/` (a plugin's own `plugin_id::family` fonts) and families - that resolve to nothing are left out. +- **Names**: a family, alias or path is resolved through `font_catalog` to + the file it loads and reported under that file's name without extension + (`PressStart2P-Regular`, `4x6-font`, `5x7`, `tom-thumb`), which is how the + Fonts tab keys its rows. Fonts outside `assets/fonts/` (a plugin's own + `plugin_id::family` fonts) and families that resolve to nothing are left + out. - **When**: a daemon thread started once plugins have loaded checks every 10 seconds and writes the `font_usage_snapshot` cache key only when the usage changed (and once a day, so the cache's cleanup never expires it). Unloading a plugin drops its registrations (`forget_manager_fonts`). - **Unknown**: until the display service has published, the column reads "unknown" and `GET /api/v3/fonts/catalog` returns `used_by: null`. +- The tab warns before deleting a font that a loaded plugin registered. -## Performance Monitoring +## Text measurement ```python -# Get performance stats -stats = font_manager.get_performance_stats() - -print(f"Cache hit rate: {stats['cache_hit_rate']*100:.1f}%") -print(f"Total fonts cached: {stats['total_fonts_cached']}") -print(f"Failed loads: {stats['failed_loads']}") -print(f"Manager fonts: {stats['manager_fonts']}") -print(f"Plugin fonts: {stats['plugin_fonts']}") -``` - -## Text Measurement - -```python -# Measure text dimensions width, height, baseline = font_manager.measure_text("Hello", font) - -# Get font height font_height = font_manager.get_font_height(font) ``` -## Best Practices +## Tips -### For Managers - -1. **Register all fonts** you use for visibility -2. **Use consistent element keys** (e.g., `{manager_id}.{element_type}`) -3. **Cache font references** if using same font multiple times -4. **Use `resolve_font()`** not `get_font()` directly to support overrides -5. **Define sensible defaults** that work well on LED matrix - -### For Plugins - -1. **Use plugin-relative paths** (`plugin://fonts/...`) -2. **Include font metadata** (license, description) -3. **Provide fallback** fonts if custom fonts fail to load -4. **Test with different display sizes** - -### General - -1. **BDF fonts** are often better for small sizes on LED matrices -2. **TTF fonts** work well for larger sizes -3. **Monospace fonts** are easier to align -4. **Test on actual hardware** - what looks good on screen may not work on LED matrix - -## Migration from Old System - -### Old Way (Direct Font Loading) -```python -self.font = ImageFont.truetype("assets/fonts/PressStart2P-Regular.ttf", 8) -``` - -### New Way (FontManager) -```python -element_key = f"{self.manager_id}.text" -self.font_manager.register_manager_font( - manager_id=self.manager_id, - element_key=element_key, - family="pressstart2p-regular", - size_px=8 -) -self.font = self.font_manager.resolve_font( - element_key=element_key, - family="pressstart2p-regular", - size_px=8 -) -``` +- BDF fonts usually look better than TTF at small sizes on LED panels. +- Use `{plugin_id}.{element}` element keys. +- Register the fonts you draw with, so the Fonts tab can warn before one is + deleted. +- Replace direct `ImageFont.truetype("assets/fonts/...", 8)` calls with + `resolve_font()`: it caches, resolves paths against the install directory, + and handles BDF files. ## Troubleshooting -### Font Not Found -- Check font file exists in `assets/fonts/` -- Verify font family name matches filename (without extension, lowercase) -- Check logs for font discovery errors +**Font not found** +- Check the file exists in `assets/fonts/`. +- The family name is the filename without extension, lower-cased. +- Check the display service log for font discovery errors. -### Override Not Working -- Verify element key matches exactly what manager registered -- Check `config/font_overrides.json` for correct syntax -- Restart application to ensure overrides are loaded +**Plugin fonts not loading** +- Check the manifest's `"fonts"` block. +- Check the log for download or registration errors, and that font URLs are + reachable. -### Performance Issues -- Check cache hit rate in performance stats -- Reduce number of unique font/size combinations -- Clear cache if it grows too large: `font_manager.clear_cache()` +## API reference -### Plugin Fonts Not Loading -- Verify plugin manifest syntax -- Check plugin directory structure -- Review logs for download/registration errors -- Ensure font URLs are accessible +Current methods: -## API Reference +| Method | Purpose | +|---|---| +| `register_manager_font(manager_id, element_key, family, size_px, color=None)` | Record a font choice (feeds the Fonts tab) | +| `forget_manager_fonts(manager_id)` | Drop a manager's registrations (core calls it when a plugin unloads) | +| `resolve_font(element_key, family, size_px, plugin_id=None)` | Get a font, applying overrides and plugin namespacing | +| `get_font(family, size_px)` | Get a font directly | +| `get_native_bdf_size(family)` | Native pixel size of a BDF family, or `None` | +| `measure_text(text, font)` | `(width, height, baseline)` | +| `get_font_height(font)` | Line height | +| `register_plugin_fonts(plugin_id, font_manifest)` | Register a plugin's fonts (core calls it at load) | +| `clear_cache()` | Drop cached fonts and metrics | +| `font_catalog` (attribute) | Family name → file path | -### FontManager Methods +### Deprecated methods -- `register_manager_font(manager_id, element_key, family, size_px, color=None)` - Register font usage -- `forget_manager_fonts(manager_id)` - Drop a manager's registrations (core calls it when a plugin unloads) -- `resolve_font(element_key, family, size_px, plugin_id=None)` - Get font with override support -- `get_font(family, size_px)` - Get font directly (bypasses overrides) -- `measure_text(text, font)` - Measure text dimensions -- `get_font_height(font)` - Get font height -- `set_override(element_key, family=None, size_px=None)` - Set manual override -- `remove_override(element_key)` - Remove override -- `get_overrides()` - Get all overrides -- `get_detected_fonts()` - Get all detected font usage -- `get_manager_fonts(manager_id=None)` - Get fonts by manager -- `get_available_fonts()` - Get font catalog -- `get_size_tokens()` - Get size token definitions -- `get_performance_stats()` - Get performance metrics -- `clear_cache()` - Clear font cache -- `register_plugin_fonts(plugin_id, font_manifest)` - Register plugin fonts -- `unregister_plugin_fonts(plugin_id)` - Unregister plugin fonts - -## Example: Complete Manager Implementation - -For a working example of the font manager API in use, see -`src/font_manager.py` itself. +Removed in 3.7.0. Each logs a warning on first call. +| Method | Use instead | +|---|---| +| `get_available_fonts()`, `get_font_catalog()` | read `font_catalog` | +| `get_size_tokens()` | pass a pixel size | +| `get_performance_stats()` | — | +| `set_override()`, `remove_override()`, `get_overrides()` | a font field in your plugin's config schema | +| `get_manager_fonts()`, `get_detected_fonts()` | — | +| `get_plugin_fonts()`, `unregister_plugin_fonts()` | — | +| `add_font()`, `remove_font()`, `validate_font()` | the web UI's Fonts tab | diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index a7dceae0..d4c38915 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -116,8 +116,8 @@ weather and other location-aware plugins. 4. Wait for installation to finish — installed plugins appear in the **Installed Plugins** section above and get their own tab in the second nav row -5. Toggle the plugin to enabled -6. From **Overview**, click **Restart Display Service** +5. Toggle the plugin to enabled. The running display loads it within a + few seconds; no restart is needed You can also install community plugins straight from a GitHub URL using the **Install from GitHub** section further down the same tab — see @@ -128,9 +128,9 @@ You can also install community plugins straight from a GitHub URL using the 1. Each installed plugin gets its own tab in the second navigation row 2. Open that plugin's tab to edit its settings (favorite teams, API keys, update intervals, etc.) -3. Click **Save** -4. Restart the display service from **Overview** so the new settings take - effect +3. Click **Save**. The display service watches `config.json` and hands the + new settings to the running plugin, so no restart is needed. If a plugin + still shows old settings, restart the display service from **Overview** **Note:** how long each plugin stays on screen is not set in the plugin's own tab — use the **Rotation** tab's **Screen Durations** @@ -197,14 +197,15 @@ The fastest way to verify a plugin works without waiting for the rotation: **Check:** 1. Plugin is enabled (toggle on the **Plugin Manager** tab) -2. Display service was restarted after enabling -3. Plugin's display duration is non-zero -4. No errors in the **Logs** tab for that plugin +2. Plugin's display duration is non-zero +3. No errors in the **Logs** tab for that plugin. A plugin whose + `validate_config()` fails is not loaded until its settings are fixed **Fix:** 1. Enable the plugin from **Plugin Manager** -2. Click **Restart Display Service** on **Overview** -3. Check the **Logs** tab for plugin-specific errors +2. Check the **Logs** tab for plugin-specific errors +3. If it still does not appear, click **Restart Display Service** on + **Overview** ### Weather Plugin Shows "No Data" diff --git a/docs/HOW_TO_RUN_TESTS.md b/docs/HOW_TO_RUN_TESTS.md index 636a53b0..7ee9a097 100644 --- a/docs/HOW_TO_RUN_TESTS.md +++ b/docs/HOW_TO_RUN_TESTS.md @@ -52,10 +52,10 @@ pytest test/test_display_controller.py test/test_plugin_system.py ```bash # Run a specific test class -pytest test/test_display_controller.py::TestDisplayControllerModeRotation +pytest test/test_display_controller.py::TestDisplayControllerLivePriority # Run a specific test function -pytest test/test_display_controller.py::TestDisplayControllerModeRotation::test_basic_rotation +pytest test/test_display_controller.py::TestDisplayControllerSchedule::test_active_hours ``` ### Run Tests by Marker @@ -98,7 +98,7 @@ When you run `pytest`, you'll see: ``` test/test_display_controller.py::TestDisplayControllerInitialization::test_init_success PASSED -test/test_display_controller.py::TestDisplayControllerModeRotation::test_basic_rotation PASSED +test/test_display_controller.py::TestDisplayControllerOnDemand::test_activate_on_demand PASSED ... ``` @@ -174,10 +174,10 @@ pytest ```bash # Run with maximum verbosity and show print statements -pytest -vv -s test/test_display_controller.py::TestDisplayControllerModeRotation::test_basic_rotation +pytest -vv -s test/test_display_controller.py::TestDisplayControllerSchedule::test_active_hours # Run with Python debugger (pdb) -pytest --pdb test/test_display_controller.py::TestDisplayControllerModeRotation::test_basic_rotation +pytest --pdb test/test_display_controller.py::TestDisplayControllerSchedule::test_active_hours ``` ### Run Tests in Parallel (Faster) diff --git a/docs/PERMISSIONS.md b/docs/PERMISSIONS.md new file mode 100644 index 00000000..2d3d7125 --- /dev/null +++ b/docs/PERMISSIONS.md @@ -0,0 +1,154 @@ +# Permissions + +Who owns what on an installed system, which privileged commands the web +interface may run, and how to repair ownership when it goes wrong. The +installer, [`first_time_install.sh`](../first_time_install.sh), sets all of +this up; this page describes the result. + +## Users and groups + +| Account | Used by | Why | +|---|---|---| +| `root` | `ledmatrix.service` (the display) | The LED matrix library needs direct GPIO access | +| The installing user (e.g. `ledpi`) | `ledmatrix-web.service`, `ledmatrix-update-verify.service` | A web server should not run as root | +| `ledmatrix` group | shared files | Members: the installing user, `root`, and `daemon` if it exists. Created by [`setup_cache.sh`](../scripts/install/setup_cache.sh) and the installer | + +The installer also adds the web user to `systemd-journal` and `adm` so the +**Logs** tab can read the journal. Group changes apply after the user logs +in again (services pick them up on restart). + +## Files and directories + +| Path | Owner | Mode | Notes | +|---|---|---|---| +| Project directory | web user | dirs `755`, files `644`, `*.sh` `755` | Set in the installer's "Normalize project file permissions" step | +| `config/` | web user | `2775` | | +| `config/config.json` | web user | `644` | Written by the web interface | +| `config/config_secrets.json` | web user : `ledmatrix` | `640` | Owned by the web user because the web interface writes it; root reads it regardless of mode | +| `plugin-repos/`, `plugins/` | web user | dirs `2775`, files `664` | The web interface installs and removes plugins | +| `assets/` | web user | dirs `755`, files `644` | Root writes downloaded logos regardless | +| `/var/cache/ledmatrix/` | `root:ledmatrix` | `2775` (setgid) | Shared cache: see below | +| Cache files | creator : `ledmatrix` | `660` | | +| `scripts/fix_perms/safe_plugin_rm.sh`, `safe_pip_install.sh` | `root:root` | `755` | Run as root through sudo, so the web user must not be able to edit them | +| `/etc/sudoers.d/ledmatrix_web`, `ledmatrix_wifi` | `root` | `440` | | + +What keeps it that way at runtime: + +- **Config files.** Saves go through + [`src/config_manager_atomic.py`](../src/config_manager_atomic.py), which + applies `get_config_file_mode()` (`640` for secrets, `644` otherwise) and, + when running as root, moves the file's group to the project directory's + group (`ensure_shared_group_ownership()` in + [`src/common/permission_utils.py`](../src/common/permission_utils.py)). +- **Cache files.** [`src/cache/disk_cache.py`](../src/cache/disk_cache.py) + sets every file it writes to `0660` and gives it the cache directory's + group, without relying on the setgid bit. So a file root writes stays + readable by the web user. +- **Plugin directories.** [`run.py`](../run.py) sets + `sys.dont_write_bytecode`, because root-owned `__pycache__` directories + inside a plugin stop the web user updating or removing it. + +`ledmatrix-web.service` deliberately has no `CacheDirectory=`: systemd would +re-own `/var/cache/ledmatrix` to the web user and its primary group, and the +web interface could no longer read what the display writes (see the comment +in [`systemd/ledmatrix-web.service`](../systemd/ledmatrix-web.service)). + +If `/var/cache/ledmatrix` is not usable, `CacheManager` falls back to +`~/.ledmatrix_cache`, `/opt/ledmatrix/cache` or a temp directory +([`src/cache_manager.py`](../src/cache_manager.py)). The two services then +may not share a cache, and the web UI shows stale or empty display status, +on-demand state and plugin health. Fix the directory rather than living +with the fallback. + +## sudo rules + +### `/etc/sudoers.d/ledmatrix_web` + +Generated by `web_sudoers_rules()` in +[`scripts/install/lib_sudoers.sh`](../scripts/install/lib_sudoers.sh), the +only place these rules are defined. Installed by the installer and by +[`configure_web_sudo.sh`](../scripts/install/configure_web_sudo.sh), both of +which check them with `visudo -c` first. The web user may run, without a +password: + +- `reboot`, `poweroff` +- `systemctl start|stop|restart|enable|disable|status ledmatrix.service`, + `systemctl is-active ledmatrix[.service]` +- `systemctl start|stop|restart ledmatrix-web.service` +- `bash /scripts/fix_perms/safe_plugin_rm.sh *` — removes a + directory only if it resolves to a child of `plugin-repos/` or `plugins/` +- `bash /scripts/fix_perms/safe_pip_install.sh *` — installs a + `requirements.txt` only if it is the project's own or one under + `plugin-repos/` or `plugins/`, so the root display service can import the + packages +- `journalctl -u ledmatrix.service *`, `-u ledmatrix *`, `-t ledmatrix *`, + tagged `NOEXEC`: journalctl opens a pager on a terminal, and a shell + escape from that pager would be a root shell + +### `/etc/sudoers.d/ledmatrix_wifi` + +Written by +[`scripts/install/configure_wifi_permissions.sh`](../scripts/install/configure_wifi_permissions.sh) +(run as the web user; the installer calls it). It refuses to grant a binary +that is not root-owned or is group/world-writable. The rules cover: + +- `nmcli device wifi connect|disconnect *`, `nmcli device connect|disconnect *`, + `nmcli radio wifi on|off` +- `systemctl start|stop|restart hostapd`, `... dnsmasq`, + `systemctl restart NetworkManager` +- `sysctl -w net.ipv4.ip_forward=0|1` +- `nft add|delete table ip ledmatrix` +- `rfkill unblock wifi` +- `mkdir -p /etc/NetworkManager/dnsmasq-shared.d` +- `cp` of `/tmp/hostapd.conf` and `/tmp/dnsmasq.conf` to their fixed + destinations, and `rm -f /etc/dnsmasq.d/ledmatrix-captive.conf` +- `cp /tmp/ledmatrix-nm-dnsmasq.conf` to + `/etc/NetworkManager/dnsmasq-shared.d/ledmatrix-captive.conf`, and + `rm -f` of that file + +**`iptables` is deliberately not granted.** The captive portal's rules are +built from the interface name and port, so a rule covering them would need a +trailing wildcard, and `iptables --modprobe=` runs `` as root: a +wildcard grant is a root shell for the web user. Doing it safely needs a +wrapper script that builds the rules itself, like `safe_plugin_rm.sh`. On a +stock Raspberry Pi OS image the default user's blanket `NOPASSWD` rule +(`/etc/sudoers.d/010_pi-nopasswd`) hides this gap. + +### polkit + +The same script installs `/etc/polkit-1/rules.d/10-ledmatrix-wifi.rules`, +which lets the web user perform any `org.freedesktop.NetworkManager.*` +action without authentication. + +## Repair scripts + +In [`scripts/fix_perms/`](../scripts/fix_perms/). Run them from the project +directory. + +| Script | Run as | What it does | Notes | +|---|---|---|---| +| `fix_plugin_permissions.sh` | `sudo` | `plugins/` and `plugin-repos/` to `root:`, dirs `2775`, files `664`; makes a `700` home directory `755` so root can traverse it | Safe. Group-writable, so the web user keeps write access | +| `fix_assets_permissions.sh` | `sudo` | `assets/` to `:`, mode `777` recursively | Works, but looser than the installer's `755`/`644` | +| `fix_cache_permissions.sh` | `sudo` | Runs [`setup_cache.sh`](../scripts/install/setup_cache.sh) for `/var/cache/ledmatrix` (`root:ledmatrix`, `2775`, files `660`), then makes `~/.ledmatrix_cache` (the fallback cache) `:` mode `777` | Safe. The `~/.ledmatrix_cache` mode is still `777` | +| `fix_web_permissions.sh` | the web user, **without** `sudo` | Resets project file ownership for the web user (it calls `sudo` itself), then makes `safe_plugin_rm.sh` and `safe_pip_install.sh` `root:root` `755` again and restores `config_secrets.json` to its owner, group `ledmatrix`, mode `640` | Refuses to run as root. It does not write sudoers rules | +| `safe_plugin_rm.sh`, `safe_pip_install.sh` | — | Called by the web interface through sudo | Not for manual use | + +To reinstall the sudoers rules, run +`./scripts/install/configure_web_sudo.sh` (web rules) or +`./scripts/install/configure_wifi_permissions.sh` (WiFi rules and polkit) as +the web user, not with `sudo`. + +After any of these, restart both services: + +```bash +sudo systemctl restart ledmatrix.service ledmatrix-web.service +``` + +## Checking + +```bash +ls -ld /var/cache/ledmatrix # drwxrwsr-x root ledmatrix +stat -c '%U:%G %a %n' config/config.json config/config_secrets.json +id # web user should list ledmatrix +sudo -l # lists the NOPASSWD rules +``` diff --git a/docs/PLUGIN_API_REFERENCE.md b/docs/PLUGIN_API_REFERENCE.md index 5aadaadc..1620c61e 100644 --- a/docs/PLUGIN_API_REFERENCE.md +++ b/docs/PLUGIN_API_REFERENCE.md @@ -9,6 +9,7 @@ Complete API reference for plugin developers. This document describes all method ## Table of Contents +- [Manifest Required Fields](#manifest-required-fields) - [BasePlugin](#baseplugin) - [Display Manager](#display-manager) - [Cache Manager](#cache-manager) @@ -17,6 +18,50 @@ Complete API reference for plugin developers. This document describes all method --- +## Manifest Required Fields + +Three parts of core check `manifest.json`, each for a different set of +fields: + +| Check | Fields | What happens when one is missing | +|---|---|---| +| JSON schema, [`schema/manifest_schema.json`](../schema/manifest_schema.json) | `id`, `name`, `version`, `author`, `entry_point`, `class_name`, `compatible_versions` | Install from URL logs a warning (`PluginStoreManager._validate_manifest_schema()`); nothing is refused | +| Plugin Store install, [`src/plugin_system/store_manager.py`](../src/plugin_system/store_manager.py) | `id`, `name`, `class_name`, `display_modes` | Install is refused. A registry install first tries to detect a missing `class_name` from the entry-point file | +| Plugin loader, [`src/plugin_system/plugin_loader.py`](../src/plugin_system/plugin_loader.py) | `class_name` | The plugin fails to load | + +Defaults and other uses: + +- `entry_point` defaults to `manager.py`; the store writes the default back + into the manifest on install. +- `compatible_versions` (a list of semver ranges such as `">=2.0.0"`) is how + the store decides whether a plugin can run on this core. An install is + refused only when the field excludes the running version + (`compatibility.check()` in + [`src/plugin_system/compatibility.py`](../src/plugin_system/compatibility.py)). +- `version` is compared with the registry's `latest_version` to decide + whether an update is available. +- If `display_modes` is empty at load time, the display controller uses the + plugin id as the only mode. + +**Set all eight:** `id`, `name`, `version`, `author`, `entry_point`, +`class_name`, `display_modes`, `compatible_versions`. That satisfies every +check. The schema lists the optional fields. + +```json +{ + "id": "my-plugin", + "name": "My Plugin", + "version": "1.0.0", + "author": "YourName", + "entry_point": "manager.py", + "class_name": "MyPlugin", + "display_modes": ["my-plugin"], + "compatible_versions": [">=2.0.0"] +} +``` + +--- + ## BasePlugin All plugins must inherit from `BasePlugin` and implement the required methods. The base class provides access to managers and common functionality. @@ -432,82 +477,17 @@ self.display_manager.update_display() This is the canonical way to render arbitrary images. -### Weather Icons +### Weather Icons (deprecated) -#### `draw_weather_icon(condition: str, x: int, y: int, size: int = 16) -> None` +> Deprecated, removed in 3.7.0 — draw your own icons (the weather plugin +> ships `WeatherIcons`). See [Deprecated APIs](#deprecated-apis). -Draw a weather icon based on the condition string. - -**Parameters**: -- `condition` (str): Weather condition (e.g., "clear", "cloudy", "rain", "snow", "storm") -- `x` (int): X position -- `y` (int): Y position -- `size` (int): Icon size in pixels (default: 16) - -**Supported Conditions**: -- `"clear"`, `"sunny"` → Sun icon -- `"clouds"`, `"cloudy"`, `"partly cloudy"` → Cloud icon -- `"rain"`, `"drizzle"`, `"shower"` → Rain icon -- `"snow"`, `"sleet"`, `"hail"` → Snow icon -- `"thunderstorm"`, `"storm"` → Storm icon - -**Example**: -```python -self.display_manager.draw_weather_icon("rain", x=10, y=10, size=16) -``` - -#### `draw_sun(x: int, y: int, size: int = 16) -> None` - -Draw a sun icon with rays. - -**Parameters**: -- `x` (int): X position -- `y` (int): Y position -- `size` (int): Icon size (default: 16) - -#### `draw_cloud(x: int, y: int, size: int = 16, color: tuple = (200, 200, 200)) -> None` - -Draw a cloud icon. - -**Parameters**: -- `x` (int): X position -- `y` (int): Y position -- `size` (int): Icon size (default: 16) -- `color` (tuple): RGB color (default: light gray) - -#### `draw_rain(x: int, y: int, size: int = 16) -> None` - -Draw rain icon with cloud and droplets. - -#### `draw_snow(x: int, y: int, size: int = 16) -> None` - -Draw snow icon with cloud and snowflakes. - -#### `draw_text_with_icons(text: str, icons: List[tuple] = None, x: int = None, y: int = None, color: tuple = (255, 255, 255)) -> None` - -Draw text with weather icons at specified positions. - -**Parameters**: -- `text` (str): Text to display -- `icons` (List[tuple], optional): List of (icon_type, x, y) tuples -- `x` (int, optional): X position for text -- `y` (int, optional): Y position for text -- `color` (tuple): Text color - -**Note**: Automatically calls `update_display()` after drawing. - -**Example**: -```python -icons = [ - ("sun", 5, 5), - ("cloud", 100, 5) -] -self.display_manager.draw_text_with_icons( - "Weather: Sunny, Cloudy", - icons=icons, - x=10, y=20 -) -``` +- `draw_weather_icon(condition, x, y, size=16)` — icon for a condition + string such as `"clear"`, `"clouds"`, `"rain"`, `"snow"`, `"storm"` +- `draw_sun(x, y, size=16)`, `draw_cloud(x, y, size=16, color=(200, 200, 200))`, + `draw_rain(x, y, size=16)`, `draw_snow(x, y, size=16)` +- `draw_text_with_icons(text, icons=None, x=None, y=None, color=(255, 255, 255))` + — text plus a list of `(icon_type, x, y)` icons; calls `update_display()` ### Scrolling State Management @@ -601,6 +581,8 @@ Process any deferred updates if not currently scrolling. Called automatically by #### `get_scrolling_stats() -> dict` +> Deprecated, removed in 3.7.0. See [Deprecated APIs](#deprecated-apis). + Get current scrolling statistics for debugging. **Returns**: Dictionary with scrolling state information @@ -742,6 +724,8 @@ data = self.cache_manager.get_with_auto_strategy("nhl_live_scores") #### `get_background_cached_data(key: str, sport_key: Optional[str] = None) -> Optional[Dict[str, Any]]` +> Deprecated, removed in 3.7.0 — use `get()`. See [Deprecated APIs](#deprecated-apis). + Get background service cached data with sport-specific intervals. **Parameters**: @@ -779,6 +763,8 @@ max_age = strategy['max_age'] # Get configured max age #### `get_sport_live_interval(sport_key: str) -> int` +> Deprecated, removed in 3.7.0. See [Deprecated APIs](#deprecated-apis). + Get the live_update_interval for a specific sport from config. **Parameters**: @@ -803,6 +789,8 @@ Extract data type from cache key to determine appropriate cache strategy. #### `get_sport_key_from_cache_key(key: str) -> Optional[str]` +> Deprecated, removed in 3.7.0. See [Deprecated APIs](#deprecated-apis). + Extract sport key from cache key for sport-specific strategies. **Parameters**: @@ -847,10 +835,12 @@ for file_info in files: self.logger.info(f"Cache: {file_info['key']}, Age: {file_info['age_display']}") ``` -### Metrics Methods +### Metrics Methods (deprecated) #### `get_cache_metrics() -> Dict[str, Any]` +> Deprecated, removed in 3.7.0. See [Deprecated APIs](#deprecated-apis). + Get cache performance metrics. **Returns**: Dictionary with cache statistics (`total_requests`, `cache_hit_rate`, `background_hit_rate`, `api_calls_saved`, `average_fetch_time`, etc.) @@ -863,6 +853,8 @@ self.logger.info(f"Cache hit rate: {metrics['cache_hit_rate']:.2%}") #### `get_memory_cache_stats() -> Dict[str, Any]` +> Deprecated, removed in 3.7.0. See [Deprecated APIs](#deprecated-apis). + Get memory cache statistics. **Returns**: Dictionary with memory cache stats (size, max_size, etc.) @@ -907,6 +899,8 @@ for plugin_id, plugin in all_plugins.items(): #### `get_enabled_plugins() -> List[str]` +> Deprecated, removed in 3.7.0 — check `enabled` on the instances in `plugin_manager.plugins`. See [Deprecated APIs](#deprecated-apis). + Get list of enabled plugin IDs. **Returns**: List of plugin identifier strings @@ -985,9 +979,8 @@ def update(self): **Example - Checking if another plugin is enabled**: ```python -enabled_plugins = self.plugin_manager.get_enabled_plugins() -if "weather" in enabled_plugins: - # Weather plugin is enabled +weather = self.plugin_manager.plugins.get("weather") +if weather is not None and weather.enabled: pass ``` diff --git a/docs/PLUGIN_CONFIG_ARCHITECTURE.md b/docs/PLUGIN_CONFIG_ARCHITECTURE.md index b34ffc89..1ab47b9a 100644 --- a/docs/PLUGIN_CONFIG_ARCHITECTURE.md +++ b/docs/PLUGIN_CONFIG_ARCHITECTURE.md @@ -155,9 +155,9 @@ deep-merged back into the plugin's config at load time ### Custom input widgets Set `"x-widget": ""` on a property. Core widgets are in -`web_interface/static/v3/js/widgets/` (see its README); a plugin can ship its -own widget script, served from `/static/plugin-widgets//.js`. -See [widget-guide.md](widget-guide.md). +`web_interface/static/v3/js/widgets/`; a plugin can ship its own widget +script, served from `/static/plugin-widgets//.js`. See the +[widget guide](../web_interface/static/v3/js/widgets/README.md). ### Custom actions diff --git a/docs/PLUGIN_DEVELOPMENT_GUIDE.md b/docs/PLUGIN_DEVELOPMENT_GUIDE.md index 95857464..da844d57 100644 --- a/docs/PLUGIN_DEVELOPMENT_GUIDE.md +++ b/docs/PLUGIN_DEVELOPMENT_GUIDE.md @@ -520,19 +520,22 @@ When developing plugins, you'll need to use the APIs provided by the LEDMatrix s `display_manager.image` (a PIL Image) and call `update_display()`; there is no `draw_image()` helper method. - `draw_weather_icon()`, `draw_sun()`, `draw_cloud()` - Weather icons + (deprecated, removed in 3.7.0 — draw your own icons) - `get_text_width()`, `get_font_height()` - Text utilities - `set_scrolling_state()`, `defer_update()` - Scrolling state management **Cache Manager** (`self.cache_manager`): - `get()`, `set()`, `delete()` - Basic caching - `get_cached_data_with_strategy()` - Advanced caching with strategies -- `get_background_cached_data()` - Background service caching +- `get_background_cached_data()` - deprecated, removed in 3.7.0 — use `get()` **Plugin Manager** (`self.plugin_manager`): - `get_plugin()`, `get_all_plugins()` - Access other plugins - `get_plugin_info()` - Get plugin information -See [PLUGIN_API_REFERENCE.md](PLUGIN_API_REFERENCE.md) for complete documentation. +See [PLUGIN_API_REFERENCE.md](PLUGIN_API_REFERENCE.md) for complete +documentation, and its [Deprecated APIs](PLUGIN_API_REFERENCE.md#deprecated-apis) +table for everything removed in 3.7.0. ## 3rd Party Plugin Development @@ -577,12 +580,14 @@ Your plugin must: pass ``` -2. **Include manifest.json** with required fields: +2. **Include manifest.json** with the required fields listed in + [PLUGIN_API_REFERENCE.md](PLUGIN_API_REFERENCE.md#manifest-required-fields): ```json { "id": "my-plugin", "name": "My Plugin", "version": "1.0.0", + "author": "YourName", "class_name": "MyPlugin", "entry_point": "manager.py", "display_modes": ["my_plugin"], @@ -658,7 +663,7 @@ For your plugin to work well in the plugin store: with the registry's `latest_version`; releases and tags are not read - **README.md**: Clear installation and configuration instructions - **config_schema.json**: Recommended for web UI configuration -- **manifest.json**: Required with all required fields +- **manifest.json**: Required, with the [required fields](PLUGIN_API_REFERENCE.md#manifest-required-fields) - **requirements.txt**: If your plugin has Python dependencies ### Distribution Options diff --git a/docs/PLUGIN_QUICK_REFERENCE.md b/docs/PLUGIN_QUICK_REFERENCE.md index 8eb7f793..16bc0886 100644 --- a/docs/PLUGIN_QUICK_REFERENCE.md +++ b/docs/PLUGIN_QUICK_REFERENCE.md @@ -45,7 +45,8 @@ LEDMatrix/ ### 1. Minimal Plugin Structure -**manifest.json**: +**manifest.json** (the required fields are explained in +[PLUGIN_API_REFERENCE.md](PLUGIN_API_REFERENCE.md#manifest-required-fields)): ```json { "id": "my-plugin", @@ -54,6 +55,8 @@ LEDMatrix/ "author": "YourName", "entry_point": "manager.py", "class_name": "MyPlugin", + "display_modes": ["my-plugin"], + "compatible_versions": [">=2.0.0"], "category": "custom" } ``` diff --git a/docs/PLUGIN_REGISTRY_SETUP_GUIDE.md b/docs/PLUGIN_REGISTRY_SETUP_GUIDE.md index 0fcf7db8..f12a7a97 100644 --- a/docs/PLUGIN_REGISTRY_SETUP_GUIDE.md +++ b/docs/PLUGIN_REGISTRY_SETUP_GUIDE.md @@ -67,9 +67,9 @@ Don't edit `latest_version` or `last_updated` by hand for monorepo plugins: ## Adding or changing an official plugin -1. Add or edit `plugins//` in the monorepo. The store refuses - a manifest without `id`, `name`, `class_name` and `display_modes`; also - set `version`. +1. Add or edit `plugins//` in the monorepo, with the + manifest fields listed in + [PLUGIN_API_REFERENCE.md](PLUGIN_API_REFERENCE.md#manifest-required-fields). 2. Bump `version` in the plugin's `manifest.json` for every change, or users won't be offered the update. 3. Run `python update_registry.py` in ledmatrix-plugins and commit the diff --git a/docs/README.md b/docs/README.md index 475cb92e..bd88bdc6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,7 @@ the one-shot installer. The pages here go deeper. 2. [WEB_INTERFACE_GUIDE.md](WEB_INTERFACE_GUIDE.md) — using the web UI 3. [PLUGIN_STORE_GUIDE.md](PLUGIN_STORE_GUIDE.md) — installing and managing plugins 4. [WIFI_NETWORK_SETUP.md](WIFI_NETWORK_SETUP.md) — WiFi and AP-mode setup -5. [TROUBLESHOOTING.md](TROUBLESHOOTING.md) — common issues and fixes +5. [TROUBLESHOOTING.md](TROUBLESHOOTING.md) — common issues and fixes ([PERMISSIONS.md](PERMISSIONS.md) for "Permission denied") 6. [SSH_UNAVAILABLE_AFTER_INSTALL.md](SSH_UNAVAILABLE_AFTER_INSTALL.md) — recovering SSH after install 7. [CONFIG_DEBUGGING.md](CONFIG_DEBUGGING.md) — diagnosing config problems 8. [LOW_MEMORY_BOARDS.md](LOW_MEMORY_BOARDS.md) — Pi Zero 2 W / 3B+ / 1GB Pi 4 memory limits @@ -37,7 +37,7 @@ Going deeper: - [PLUGIN_CUSTOM_ICONS.md](PLUGIN_CUSTOM_ICONS.md) - [PLUGIN_REGISTRY_SETUP_GUIDE.md](PLUGIN_REGISTRY_SETUP_GUIDE.md) (+ [registry template](plugin_registry_template.json)) - [STARLARK_APPS_GUIDE.md](STARLARK_APPS_GUIDE.md) — Starlark-based mini-apps -- [widget-guide.md](widget-guide.md) — widget development +- [Widget guide](../web_interface/static/v3/js/widgets/README.md) — built-in `x-widget`s and custom widgets - [ADAPTIVE_LAYOUT.md](ADAPTIVE_LAYOUT.md) — render legibly on any panel size (opt-in font/layout scaling) - [plugin-safety-harness.md](plugin-safety-harness.md) — test a plugin across every screen and matrix size @@ -56,21 +56,25 @@ Going deeper: - [ADVANCED_FEATURES.md](ADVANCED_FEATURES.md) — Vegas scroll, on-demand display, cache management, background services, permissions - [FONT_MANAGER.md](FONT_MANAGER.md) — font system +- [PERMISSIONS.md](PERMISSIONS.md) — file ownership, sudo rules, repair scripts +- [MQTT bridge](../integrations/mqtt_bridge/README.md) — control the display from Home Assistant over MQTT ## Reference - [CONFIG_REFERENCE.md](CONFIG_REFERENCE.md) — every key in config.json and config_secrets.json - [REST_API_REFERENCE.md](REST_API_REFERENCE.md) — all web-interface HTTP endpoints - [PLUGIN_API_REFERENCE.md](PLUGIN_API_REFERENCE.md) — Python APIs available to plugins +- [src/common/README.md](../src/common/README.md) — shared helper modules plugins can import - [DEVELOPER_QUICK_REFERENCE.md](DEVELOPER_QUICK_REFERENCE.md) — common dev tasks ## Contributing to LEDMatrix itself +- [ARCHITECTURE.md](ARCHITECTURE.md) — processes, display loop, plugin system, web UI; where to start reading - [DEVELOPMENT.md](DEVELOPMENT.md) — environment setup - [HOW_TO_RUN_TESTS.md](HOW_TO_RUN_TESTS.md) — running the test suite - [MULTI_ROOT_WORKSPACE_SETUP.md](MULTI_ROOT_WORKSPACE_SETUP.md) — multi-repo workspace - [MIGRATION_GUIDE.md](MIGRATION_GUIDE.md) — breaking changes between releases -- [SPORTS_UNIFICATION.md](SPORTS_UNIFICATION.md) — how the sports scoreboard base classes are organized +- [SPORTS_UNIFICATION.md](SPORTS_UNIFICATION.md) — how shared sports scoreboard code moves into `src/common` ## Audits diff --git a/docs/SPORTS_UNIFICATION.md b/docs/SPORTS_UNIFICATION.md index 015c6e2d..b03e5767 100644 --- a/docs/SPORTS_UNIFICATION.md +++ b/docs/SPORTS_UNIFICATION.md @@ -73,14 +73,16 @@ B2 below promoted code into it (`SportsCore`, the mode classes, capabilities sections record that design, but none of it ships in core any more. Shared sports code lives in `src/common`: -``` -src/common/ - sports_scroll.py SportsScrollDisplay / …Manager — scroll orchestration - (content building stays in the plugins) - sports_helpers.py clamp/logo/rotation free functions + SportsHelpersMixin - (3.5.0) — the helpers byte-identical in the - plugins' sports.py, and the _favorite_key seam -``` +| Module | Since | Holds | +|---|---|---| +| `sports_scroll.py` | 3.2.0 | `SportsScrollDisplay` / `SportsScrollDisplayManager` — scroll orchestration (content building stays in the plugins) | +| `sports_card.py` | 3.3.0 | Free functions for card settings, colours, favourite-team rules, dates and font sizes | +| `sports_game_renderer.py` | 3.3.0 | `SportsGameRendererMixin` — scroll/Vegas card geometry | +| `sports_shared.py` | 3.3.0 | `SportsCoreSharedMixin`, `SportsLiveSharedMixin`, `SportsRecentSharedMixin` — the sport-independent `sports.py` methods | +| `sports_helpers.py` | 3.5.0 | clamp/logo/rotation free functions and `SportsHelpersMixin`, plus the `_favorite_key` seam | +| `espn_dates.py` | 3.5.0 | ESPN date-range and `limit` workarounds | + +Each is described in [src/common/README.md](../src/common/README.md). ### Converging on `src/common` @@ -90,7 +92,7 @@ modules taken from the plugin copies, each a **new module** rather than growth on an existing one: a plugin that deletes a method copy and relies on an older module having gained it fails at runtime with an `AttributeError`, while a missing module fails at load, where the version checks can see it. -`sports_helpers.py` is the first (it holds `_favorite_key`, the override point +`sports_helpers.py` is the newest (it holds `_favorite_key`, the override point listed below, for later phases); its parity test compares every body against the plugin copies when `LEDMATRIX_PLUGINS` points at a checkout, and `test/test_common_is_hardware_free.py` keeps `src/common` free of diff --git a/docs/STARLARK_APPS_GUIDE.md b/docs/STARLARK_APPS_GUIDE.md index efd5b77d..6fbd9454 100644 --- a/docs/STARLARK_APPS_GUIDE.md +++ b/docs/STARLARK_APPS_GUIDE.md @@ -102,9 +102,16 @@ cd /path/to/LEDMatrix bash scripts/download_pixlet.sh ``` +The script downloads only the Linux ARM64 build (Raspberry Pi OS 64-bit), +from the `tronbyt/pixlet` releases, to `bin/pixlet/pixlet-linux-arm64`. On +any other platform (32-bit Pi OS, x86_64, macOS), put a `pixlet` binary on +your `PATH`, or set the plugin's `pixlet_path`, or place it in `bin/pixlet/` +under the name `_find_pixlet_binary()` looks for +([`web_interface/blueprints/api_v3/__init__.py`](../web_interface/blueprints/api_v3/__init__.py)). + Verify installation: ```bash -./bin/pixlet/pixlet-linux-amd64 version +./bin/pixlet/pixlet-linux-arm64 version # Pixlet 0.50.2 (or later) ``` @@ -276,10 +283,8 @@ LEDMatrix/ │ ├── hour_hand.png │ └── minute_hand.png │ -├── bin/pixlet/ # Pixlet binaries -│ ├── pixlet-linux-amd64 -│ ├── pixlet-linux-arm64 -│ └── pixlet-darwin-arm64 +├── bin/pixlet/ # Pixlet binary +│ └── pixlet-linux-arm64 # the only one download_pixlet.sh fetches │ └── scripts/ └── download_pixlet.sh # Pixlet installer @@ -324,7 +329,7 @@ Many apps require API keys for external services: **Solutions**: 1. Check logs: `journalctl -u ledmatrix | grep -i pixlet` 2. Verify config: Ensure all required fields are filled -3. Test manually: `./bin/pixlet/pixlet-linux-amd64 render starlark-apps/{app-id}/{app-id}.star` +3. Test manually: `./bin/pixlet/pixlet-linux-arm64 render starlark-apps/{app-id}/{app-id}.star` 4. Missing assets: Some apps need images/fonts that may fail to download 5. API issues: Check API keys and rate limits diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index db5b3cac..fede8dbb 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -201,10 +201,11 @@ sudo systemctl restart ledmatrix-web **Solutions:** -1. **Install dependencies:** +1. **Install dependencies** as root, so the root display service can import + them: ```bash - pip3 install --break-system-packages -r requirements.txt - pip3 install --break-system-packages -r web_interface/requirements.txt + sudo python3 -m pip install --break-system-packages --no-cache-dir -r requirements.txt + sudo python3 -m pip install --break-system-packages --no-cache-dir -r web_interface/requirements.txt ``` 2. **Test imports step-by-step:** @@ -250,15 +251,18 @@ sudo systemctl restart ledmatrix-web **Solutions:** -```bash -# Fix ownership of LEDMatrix directory -sudo chown -R ledpi:ledpi /home/ledpi/LEDMatrix +[PERMISSIONS.md](PERMISSIONS.md) lists the expected owner and mode of every +file and directory, and which `scripts/fix_perms/` script to run as which +user. Don't `chown -R` the whole project: the two sudo helper scripts in +`scripts/fix_perms/` must stay owned by root. -# Fix config file permissions +```bash +# Config files: web user owns both; secrets must stay 640 +stat -c '%U:%G %a %n' config/config.json config/config_secrets.json sudo chmod 644 config/config.json sudo chmod 640 config/config_secrets.json -# Verify service runs as correct user +# Which user the web interface runs as sudo systemctl cat ledmatrix-web | grep User ``` @@ -462,15 +466,17 @@ sudo systemctl cat ledmatrix-web | grep User } ``` -2. **Restart display:** - ```bash - sudo systemctl restart ledmatrix - ``` + Or toggle the plugin on in the **Plugin Manager** tab, which writes the + same flag. -3. **Verify in web interface:** - - Open the **Plugin Manager** tab - - Toggle the plugin switch to enable - - From **Overview**, click **Restart Display Service** +2. **Wait a few seconds.** The display service watches `config.json` and + loads a newly enabled plugin without a restart + (`DisplayController._reconcile_enabled_plugins()` in + [`src/display_controller.py`](../src/display_controller.py)). This + needs hot reload, which is on unless `LEDMATRIX_HOT_RELOAD=false` is set. + +3. **If it still does not appear**, check the logs for a config validation + error, then restart: `sudo systemctl restart ledmatrix` #### Plugin Not Loading @@ -491,10 +497,12 @@ sudo systemctl cat ledmatrix-web | grep User # Verify all required fields present ``` -3. **Check dependencies installed:** +3. **Check dependencies installed.** Install them with `sudo`: the display + service runs as root and does not see packages pip put in your user's + `~/.local` (see [PLUGIN_DEPENDENCY_GUIDE.md](PLUGIN_DEPENDENCY_GUIDE.md)): ```bash if [ -f plugin-repos/plugin-id/requirements.txt ]; then - pip3 install --break-system-packages -r plugin-repos/plugin-id/requirements.txt + sudo python3 -m pip install --break-system-packages --no-cache-dir -r plugin-repos/plugin-id/requirements.txt fi ``` @@ -503,14 +511,9 @@ sudo systemctl cat ledmatrix-web | grep User sudo journalctl -u ledmatrix -f | grep plugin-id ``` -5. **Test plugin import:** +5. **Load and render the plugin headlessly:** ```bash - python3 -c " - import sys - sys.path.insert(0, 'plugin-repos/plugin-id') - from manager import PluginClass - print('Plugin imports successfully') - " + python3 scripts/check_plugin.py --plugin plugin-id ``` #### Stale Cache Data @@ -540,10 +543,12 @@ sudo systemctl cat ledmatrix-web | grep User sudo systemctl restart ledmatrix ``` -2. **Check cache permissions:** +2. **Check cache permissions.** Expected: `root:ledmatrix`, `drwxrwsr-x`. + `setup_cache.sh` restores that layout (see + [PERMISSIONS.md](PERMISSIONS.md#repair-scripts)): ```bash ls -ld /var/cache/ledmatrix - sudo ./scripts/fix_perms/fix_cache_permissions.sh + sudo bash scripts/install/setup_cache.sh ``` --- diff --git a/docs/WEB_INTERFACE_GUIDE.md b/docs/WEB_INTERFACE_GUIDE.md index 7b442c6d..1d354bf5 100644 --- a/docs/WEB_INTERFACE_GUIDE.md +++ b/docs/WEB_INTERFACE_GUIDE.md @@ -161,7 +161,11 @@ duration, and related settings — so you can configure Vegas mode entirely from the web UI without hand-editing JSON. See [ADVANCED_FEATURES.md](ADVANCED_FEATURES.md) for what the options do. -Changes require **Restart Display Service** from the Overview tab. +Brightness and the Vegas Scroll settings apply to the running display +within a few seconds. Matrix hardware settings (rows, columns, chain length, +mapping, GPIO slowdown, PWM and refresh settings) are only read when the +display starts, so those need **Restart Display Service** from the Overview +tab. ### Plugin Manager Tab @@ -248,16 +252,16 @@ View real-time system logs: 1. Open the **Display** tab 2. Adjust the **Brightness** slider (1–100) -3. Click **Save** -4. Click **Restart Display Service** on the **Overview** tab +3. Click **Save**. The panel picks up the new brightness within a few + seconds; no restart is needed ### Installing a New Plugin 1. Open the **Plugin Manager** tab 2. Scroll to the **Plugin Store** section and browse or search 3. Click **Install** next to the plugin -4. Toggle the plugin on in **Installed Plugins** -5. Click **Restart Display Service** on **Overview** +4. Toggle the plugin on in **Installed Plugins**. The running display + loads it within a few seconds; no restart is needed ### Configuring a Plugin diff --git a/docs/widget-guide.md b/docs/widget-guide.md index d4aa0f51..e433e85b 100644 --- a/docs/widget-guide.md +++ b/docs/widget-guide.md @@ -1,590 +1,7 @@ # Widget Development Guide -## Overview - -The LEDMatrix Widget Registry system allows plugins to use reusable UI components for configuration forms. This enables: - -- **Reusable Components**: Use existing widgets (file upload, checkboxes, etc.) without custom code -- **Custom Widgets**: Create plugin-specific widgets without modifying the LEDMatrix codebase -- **Backwards Compatibility**: Existing plugins continue to work without changes - -## Available Core Widgets - -### Plugin File Manager Widget (`plugin-file-manager`) - -Full inline file management UI for plugins that manage files via the `web_ui_actions` system. Renders a card grid, upload zone, create/delete modals, and an entry table editor — entirely inline, no iframe. - -`plugin_id` is **automatically injected** from template context. File operations call `/api/v3/plugins/action` immediately on user action; no Save Configuration needed. - -**Schema Configuration:** -```json -{ - "file_manager": { - "type": "null", - "title": "Data Files", - "x-widget": "plugin-file-manager", - "x-widget-config": { - "actions": { - "list": "list-files", - "get": "get-file", - "save": "save-file", - "upload": "upload-file", - "delete": "delete-file", - "create": "create-file", - "toggle": "toggle-category" - }, - "upload_hint": "JSON files with day numbers 1–365 as keys", - "directory_label": "my_data/", - "create_fields": [ - { "key": "category_name", "label": "Category Name", - "placeholder": "e.g., my_words", "pattern": "^[a-z0-9_]+$", - "hint": "Lowercase letters, numbers, underscores" }, - { "key": "display_name", "label": "Display Name", - "placeholder": "e.g., My Words", "hint": "Optional" } - ] - } - } -} -``` - -**`list` is required** — the widget calls it on render to populate the file grid; omitting it leaves the widget stuck in a loading state. All other actions are optional — omit any key to hide its UI element (e.g., no `create` = no New File button, no `toggle` = no enable/disable switch). - -The edit view auto-detects whether file content is tabular (object-of-objects with uniform keys) and shows a paginated table editor with inline cells. Otherwise falls back to a JSON textarea. - -**Used by:** of-the-day - ---- - -### Time Picker Widget (`time-picker`) - -Single time selection using the browser's native time input. Returns a string in `HH:MM` (24-hour) format. Generic — works in any plugin without configuration. - -**Schema Configuration:** -```json -{ - "target_time": { - "type": "string", - "x-widget": "time-picker", - "default": "00:00", - "x-options": { - "placeholder": "Select time", - "clearable": true - } - } -} -``` - -**Used by:** countdown - ---- - -### File Upload Single Widget (`file-upload-single`) - -Single-image upload for string fields. Uploads to the plugin's asset folder (`assets/plugins//uploads/`) and sets the string field value to the returned relative path. Shows a thumbnail preview and a clear button. The `plugin_id` is **automatically injected** from the template context — no need to specify it in the schema. - -**Schema Configuration:** -```json -{ - "image_path": { - "type": "string", - "x-widget": "file-upload-single", - "x-upload-config": { - "allowed_types": ["image/png", "image/jpeg", "image/bmp", "image/gif"], - "max_size_mb": 5 - } - } -} -``` - -Note: Unlike `file-upload` (array-level), this widget is for a single `string` field. It is ideal for per-item images inside `array-table` rows. - -**Used by:** countdown - ---- - -### File Upload Widget (`file-upload`) - -Upload and manage image files with drag-and-drop support, preview, delete, and scheduling. - -**Schema Configuration:** -```json -{ - "type": "array", - "x-widget": "file-upload", - "x-upload-config": { - "plugin_id": "my-plugin", - "max_files": 10, - "max_size_mb": 5, - "allowed_types": ["image/png", "image/jpeg", "image/bmp", "image/gif"] - } -} -``` - -**Used by:** static-image, news plugins - -### Checkbox Group Widget (`checkbox-group`) - -Multi-select checkboxes for array fields with enum items. - -**Schema Configuration:** -```json -{ - "type": "array", - "x-widget": "checkbox-group", - "items": { - "type": "string", - "enum": ["option1", "option2", "option3"] - }, - "x-options": { - "labels": { - "option1": "Option 1 Label", - "option2": "Option 2 Label" - } - } -} -``` - -**Used by:** odds-ticker, news plugins - -### Custom Feeds Widget (`custom-feeds`) - -Table-based RSS feed editor with logo uploads. - -**Schema Configuration:** -```json -{ - "type": "array", - "x-widget": "custom-feeds", - "items": { - "type": "object", - "properties": { - "name": { "type": "string" }, - "url": { "type": "string", "format": "uri" }, - "enabled": { "type": "boolean" }, - "logo": { "type": "object" } - } - }, - "maxItems": 50 -} -``` - -**Used by:** news plugin (for custom RSS feeds) - -## Using Existing Widgets - -To use an existing widget in your plugin's `config_schema.json`, simply add the `x-widget` property: - -```json -{ - "properties": { - "my_images": { - "type": "array", - "x-widget": "file-upload", - "x-upload-config": { - "plugin_id": "my-plugin", - "max_files": 5 - } - }, - "enabled_leagues": { - "type": "array", - "x-widget": "checkbox-group", - "items": { - "type": "string", - "enum": ["nfl", "nba", "mlb"] - }, - "x-options": { - "labels": { - "nfl": "NFL", - "nba": "NBA", - "mlb": "MLB" - } - } - } - } -} -``` - -The widget will be automatically rendered when the plugin configuration form is loaded. - -## Labelling Enum Options (`x-options.labels`) - -A plain `enum` renders as a dropdown whose option text is the value with -underscores replaced and title case applied — `day_first` becomes "Day First". -That is fine for values that read as their own label, and wrong for values that -do not: `vs` becomes "Vs", and `abbrev` says nothing about the `Sep 19` it -actually produces. - -Supply `x-options.labels` to set the visible text. This is the same convention -the `checkbox-group` widget uses: - -```json -{ - "date_format": { - "type": "string", - "enum": ["abbrev", "numeric", "day_first"], - "default": "abbrev", - "x-options": { - "labels": { - "abbrev": "Sep 19", - "numeric": "9/19", - "day_first": "19 Sep" - } - } - } -} -``` - -Labels are **display only** — the stored value is still the enum value, so -adding them never changes a saved config. The map may be partial: any value -without a label keeps the humanised fallback. Older cores that predate this -support ignore `x-options` and render the fallback for every option, so a -plugin can ship labels without requiring a core upgrade. - -Array-table columns (`x-widget: array-table`) accept the same -`x-options.labels` on a column definition, but their fallback is the **raw -value** rather than the humanised one, because those columns hold values such -as ticker symbols where `aapl` → "Aapl" would be wrong. Rows added in the -browser use the labels too (`array-table.js`), so a column reads the same -before and after a page reload. - -## Marking Fields as Advanced (`x-advanced`) - -Add `"x-advanced": true` to any top-level, non-object property to move it out -of the main form and into a single collapsed **Advanced Settings** section at -the bottom of the plugin's configuration page: - -```json -{ - "properties": { - "city": { - "type": "string", - "title": "City" - }, - "request_timeout": { - "type": "integer", - "default": 10, - "description": "HTTP timeout in seconds", - "x-advanced": true - } - } -} -``` - -Guidelines: - -- Use it for fine-tuning knobs most users never touch (timeouts, retry - behavior, cache TTLs, styling overrides). Anything a first-time user must - set to get the plugin working should stay basic. -- Nothing is hidden permanently — the section expands on click, and the - settings search finds and auto-expands advanced fields like any others. -- The flag is ignored on `object`-type properties (they already render as - their own collapsible sections) and is safely ignored by older cores, so - adding it never breaks compatibility. - -## Hiding Fields From the Form (`x-display: "hidden"`) - -Add `"x-display": "hidden"` to a property that must stay in the schema but -should not appear as a control: a deprecated key kept so existing configs keep -validating, or an internal value such as an auto-generated row id. - -```json -{ - "properties": { - "radar_zoom": { - "type": "integer", - "default": 6, - "title": "Radar Zoom Level (deprecated)", - "x-display": "hidden" - } - } -} -``` - -What the core does with it: - -- **Not rendered** at any depth: top-level fields, children of an object - section, and properties of array-of-object items (never a table column, even - if `x-columns` names it, and never in the row editor). A hidden field flagged - `x-advanced` is not listed or counted in Advanced Settings, and an object - whose children are all hidden draws no empty section. Hidden fields don't - show up in the settings search either, since it indexes the rendered form. -- **Stored value preserved on save.** Saving the form never changes a hidden - value. The unchecked-checkbox rule ignores a hidden boolean. Array rows carry - a hidden property's stored value through the form, so the value survives the - row being posted back; a new row gets no value (the plugin fills it in). -- **The API is unaffected.** A JSON save to `POST /api/v3/plugins/config` can - still set a hidden field. - -Older cores ignore the flag and render the field as a normal control. - -## Creating Custom Widgets - -### Step 1: Create Widget File - -Create a JavaScript file in your plugin's `widgets/` directory, named -`widgets/[widget-name].js`. The directory is not optional: it is the only -place the core will serve a widget from. - -```javascript -// Ensure LEDMatrixWidgets registry is available -if (typeof window.LEDMatrixWidgets === 'undefined') { - console.error('LEDMatrixWidgets registry not found'); - return; -} - -// Register your widget -window.LEDMatrixWidgets.register('my-custom-widget', { - name: 'My Custom Widget', - version: '1.0.0', - - /** - * Render the widget HTML - * @param {HTMLElement} container - Container element to render into - * @param {Object} config - Widget configuration from schema - * @param {*} value - Current value - * @param {Object} options - Additional options (fieldId, pluginId, etc.) - */ - render: function(container, config, value, options) { - const fieldId = options.fieldId || container.id; - - // Always escape HTML to prevent XSS - const escapeHtml = (text) => { - const div = document.createElement('div'); - div.textContent = text; - return div.innerHTML; - }; - - container.innerHTML = ` -
- -
- `; - - // Attach event listeners - const input = container.querySelector('input'); - input.addEventListener('change', (e) => { - this.handlers.onChange(fieldId, e.target.value); - }); - }, - - /** - * Get current value from widget - */ - getValue: function(fieldId) { - const input = document.querySelector(`#${fieldId}_input`); - return input ? input.value : null; - }, - - /** - * Set value programmatically - */ - setValue: function(fieldId, value) { - const input = document.querySelector(`#${fieldId}_input`); - if (input) { - input.value = value || ''; - } - }, - - /** - * Event handlers - */ - handlers: { - onChange: function(fieldId, value) { - // Trigger form change event - const event = new CustomEvent('widget-change', { - detail: { fieldId, value }, - bubbles: true - }); - document.dispatchEvent(event); - } - } -}); -``` - -### Step 2: Declare the Widget in `manifest.json` - -The manifest is the allowlist. A widget is served only if the plugin declares -it, so shipping a file under `widgets/` does not by itself publish it: - -```json -{ - "widgets": [ - { - "name": "my-custom-widget", - "script": "my-custom-widget.js", - "description": "What this widget is for" - } - ] -} -``` - -`name` is what you use in `x-widget` and in the URL. `script` is optional and -defaults to `[name].js`; it must be a plain filename directly inside -`widgets/` (no paths). Both are validated against -`schema/manifest_schema.json`. - -### Step 3: Reference Widget in Schema - -In your plugin's `config_schema.json`: - -```json -{ - "properties": { - "my_field": { - "type": "string", - "description": "My custom field", - "x-widget": "my-custom-widget", - "default": "" - } - } -} -``` - -### Step 4: Widget Loading - -The widget is loaded on demand when the plugin's configuration form renders a -field that references it. The system will: - -1. Check whether the widget is already registered in the core registry. -2. If not, fetch it from `/static/plugin-widgets/[plugin-id]/[widget-name].js`. - That route serves the declared `script` from your plugin's `widgets/` - directory, as `text/javascript`. -3. Render it by calling the `render` function your script registered. - -The fetch uses a dynamic `import()`, so the file must parse as an ES module. -A plain IIFE does — modules are strict mode, so avoid sloppy-mode constructs. - -**If the widget fails to load** (not declared, file missing, script throws, or -it never calls `register`), the field falls back to a plain text input holding -the current value. This is deliberate: a broken widget costs the user an -editor, not their configured value. - -**Limitation:** the on-demand path applies to `string`-typed fields (the -default branch of the config-form renderer). Fields typed `object`, `array`, -`boolean`, `integer` or `number`, and fields whose `enum` is set, are -dispatched by the server-side template to its own built-in renderers, so a -plugin-supplied `x-widget` on one of those is ignored today. - -## Widget API Reference - -### Widget Definition Object - -```javascript -{ - name: string, // Human-readable widget name - version: string, // Widget version - render: function, // Required: Render function - getValue: function, // Optional: Get current value - setValue: function, // Optional: Set value programmatically - handlers: object // Optional: Event handlers -} -``` - -### Render Function - -```javascript -render(container, config, value, options) -``` - -**Parameters:** -- `container` (HTMLElement): Container element to render into -- `config` (Object): Widget configuration from schema -- `value` (*): Current field value -- `options` (Object): Additional options - - `fieldId` (string): Field ID - - `pluginId` (string): Plugin ID - - `fullKey` (string): Full field key path - -### Get Value Function - -```javascript -getValue(fieldId) -``` - -**Returns:** Current widget value - -### Set Value Function - -```javascript -setValue(fieldId, value) -``` - -**Parameters:** -- `fieldId` (string): Field ID -- `value` (*): Value to set - -## Examples - -See [`web_interface/static/v3/js/widgets/example-color-picker.js`](../web_interface/static/v3/js/widgets/example-color-picker.js) for a complete example of a custom color picker widget. - -## Best Practices - -### Security - -1. **Always escape HTML**: Use `escapeHtml()` or `textContent` to prevent XSS -2. **Validate inputs**: Validate user input before processing -3. **Sanitize values**: Clean values before storing - -### Performance - -1. **Lazy loading**: Load widget scripts only when needed -2. **Event delegation**: Use event delegation for dynamic content -3. **Debounce**: Debounce frequent events (e.g., input changes) - -### Accessibility - -1. **Labels**: Always associate labels with inputs -2. **ARIA attributes**: Use appropriate ARIA attributes -3. **Keyboard navigation**: Ensure keyboard accessibility - -## Troubleshooting - -### Widget Not Loading - -1. Check browser console for errors -2. Verify widget file path is correct -3. Ensure `LEDMatrixWidgets.register()` is called -4. Check that widget name matches schema `x-widget` value - -### Widget Not Rendering - -1. Verify `render` function is defined -2. Check container element exists -3. Ensure widget is registered before form loads -4. Check for JavaScript errors in console - -### Value Not Saving - -1. Ensure widget triggers `widget-change` event -2. Verify form submission includes widget value -3. Check `getValue` function returns correct type -4. Verify field name matches schema property - -## Current Implementation Status - -**Phase 1 Complete:** -- ✅ Widget registry system created -- ✅ Core widgets extracted to separate files -- ✅ Widget handlers available globally (backwards compatible) -- ✅ Plugin widget loading system implemented - -**Current Behavior:** -- Core widgets are server-side rendered via Jinja2 templates (existing behavior preserved) -- Widget handlers are registered and available globally -- Custom widgets can be created, declared in `manifest.json`, and are served - and rendered on demand for `string`-typed fields -- Plugin widgets on non-string fields are not dispatched yet (see Step 4) - -**Backwards Compatibility:** -- All existing plugins using widgets continue to work without changes -- Server-side rendering remains the primary method -- Widget registry provides foundation for future enhancements - -## See Also - -- [Widget README](../web_interface/static/v3/js/widgets/README.md) - Complete widget development guide with examples -- [Plugin Development Guide](PLUGIN_DEVELOPMENT_GUIDE.md) - General plugin development -- [Plugin Configuration Guide](PLUGIN_CONFIGURATION_GUIDE.md) - Configuration setup +The widget guide lives next to the widgets, in +[web_interface/static/v3/js/widgets/README.md](../web_interface/static/v3/js/widgets/README.md). +It lists every built-in `x-widget`, the schema keywords the config form +understands (`x-options.labels`, `x-advanced`, `x-display: "hidden"`), and how +to ship a custom widget with a plugin. diff --git a/first_time_install.sh b/first_time_install.sh index 8b8ca6dc..8d67175a 100755 --- a/first_time_install.sh +++ b/first_time_install.sh @@ -18,7 +18,7 @@ on_error() { echo "-- Last 100 lines from log --" >&2 tail -n 100 "$LOG_FILE" >&2 || true fi - echo "\nCommon fixes:" >&2 + printf '\nCommon fixes:\n' >&2 echo "- Ensure the Pi is online (try: ping -c1 8.8.8.8)." >&2 echo "- If you saw an APT lock error: wait a minute, close other installers, then run: sudo dpkg --configure -a" >&2 echo "- Re-run this script. It is safe to run multiple times." >&2 @@ -115,7 +115,8 @@ fi echo "✓ OS requirements met" echo "" -# Get the actual user who invoked sudo (set after we ensure sudo below) +# The user who ran the installer: SUDO_USER once we are running under sudo +# (the re-exec below guarantees that), otherwise whoever we are now. if [ -n "${SUDO_USER:-}" ]; then ACTUAL_USER="$SUDO_USER" else @@ -202,7 +203,7 @@ echo "" # Check if running as root; if not, try to elevate automatically for novices if [ "$EUID" -ne 0 ]; then echo "This script needs administrator privileges. Attempting to re-run with sudo..." - exec sudo -E env LEDMATRIX_ELEVATED=1 bash "$0" "$@" + exec sudo -E bash "$0" "$@" fi echo "✓ Running as root (required for installation)" @@ -507,8 +508,11 @@ print_rgbmatrix_build_failure() { # it. The logic was pasted three times, identically, and is kept verbatim here. # Note: install_web_service.sh and install_service.sh no longer contain the # "User=root" / "User=${ACTUAL_USER}" strings grepped for below (the units come -# from systemd/*.service templates with User=__USER__), so until Step 8 has -# installed the unit this yields "root". +# from systemd/*.service templates with User=__USER__). So once the unit is +# installed (Step 7.5, by install_service.sh) the first branch reads its real +# User=; before that the second branch is taken whenever +# install_web_service.sh exists, matches neither string, and yields "root" -- +# the later branches are reached only if that script is missing. detect_web_service_user() { WEB_SERVICE_USER="root" if [ -f "/etc/systemd/system/ledmatrix-web.service" ]; then @@ -669,8 +673,9 @@ else echo "Setting ownership of assets directory..." chown -R "$ACTUAL_USER:$ACTUAL_USER" "$PROJECT_ROOT_DIR/assets" - # Set permissions to allow read/write for owner, group, and others (for root service user) - # Note: 777 allows root (service user) to write, which is necessary when service runs as root + # 777: read/write for owner, group and every other account. Root (the + # display service) does not need it -- root ignores mode bits -- so the + # "other" bits only matter to accounts that are neither the owner nor root. echo "Setting permissions for assets directory..." chmod -R 777 "$PROJECT_ROOT_DIR/assets" @@ -782,8 +787,8 @@ else chown -R root:"$ACTUAL_USER" "$PLUGIN_REPOS_DIR" fi -# Set directory permissions (775: rwxrwxr-x) -echo "Setting plugin-repos directory permissions to 2775 (sticky bit)..." +# Set directory permissions (2775: rwxrwsr-x, setgid so new entries inherit the group) +echo "Setting plugin-repos directory permissions to 2775 (setgid)..." find "$PLUGIN_REPOS_DIR" -type d -exec chmod 2775 {} \; # Set file permissions (664: rw-rw-r--) @@ -990,9 +995,9 @@ if [ -f "$PROJECT_ROOT_DIR/requirements.txt" ]; then PACKAGE_NUM=$((PACKAGE_NUM + 1)) echo "[$PACKAGE_NUM/$TOTAL_PACKAGES] Installing: $line" - # Check if package is already installed (basic check - may not catch all cases) - # Try installing with verbose output and timeout (if available) - # Use --no-cache-dir to avoid cache issues, --verbose for diagnostics + # Install with a timeout where available. --verbose output goes to + # $INSTALL_OUTPUT (filtered below, full copy in the log); --no-cache-dir + # avoids pip cache issues. INSTALL_OUTPUT=$(mktemp) INSTALL_SUCCESS=false @@ -1297,7 +1302,11 @@ else WEB_DEPS_OK=false fi else - echo "Web dependencies already installed from web_interface/requirements.txt in Step 5" + # No marker means Step 5 did not install web_interface/requirements.txt, + # and without the smart installer there is nothing else to try here. + echo "⚠ scripts/install_dependencies_apt.py not found, and Step 5 did not install" + echo " web_interface/requirements.txt, so web interface dependencies may be missing." + WEB_DEPS_OK=false fi # Create the marker only when installation actually succeeded, so a @@ -1562,11 +1571,12 @@ echo "-----------------------------------------------------" if [ -f "$PROJECT_ROOT_DIR/scripts/install/configure_wifi_permissions.sh" ]; then echo "Configuring WiFi management permissions..." # Run as the actual user (not root) since the script checks for that - sudo -u "$ACTUAL_USER" bash "$PROJECT_ROOT_DIR/scripts/install/configure_wifi_permissions.sh" || { + if sudo -u "$ACTUAL_USER" bash "$PROJECT_ROOT_DIR/scripts/install/configure_wifi_permissions.sh"; then + echo "✓ WiFi management permissions configured" + else echo "⚠ WiFi permissions configuration failed, but continuing installation" echo " You can run it manually later: ./scripts/install/configure_wifi_permissions.sh" - } - echo "✓ WiFi management permissions configured" + fi else echo "⚠ configure_wifi_permissions.sh not found; skipping WiFi permissions configuration" echo " You can configure WiFi permissions later by running:" @@ -1736,10 +1746,11 @@ echo "-------------------------------------" echo "Removing potential conflicting services (bluetooth and others)..." if [ "$SKIP_SOUND" = "1" ]; then echo "Skipping sound module configuration as requested (--skip-sound)." -elif apt_remove bluez bluez-firmware pi-bluetooth triggerhappy pigpio; then - echo "✓ Unnecessary services removed (or not present)" else - echo "⚠ Some packages could not be removed; continuing" + # apt_remove never fails (it ends in `|| true`); apt itself reports any + # package it could not remove. + apt_remove bluez bluez-firmware pi-bluetooth triggerhappy pigpio + echo "✓ Unnecessary services removed (or not present)" fi # Blacklist onboard sound module (idempotent) @@ -1954,20 +1965,6 @@ if systemctl list-unit-files | grep -q "ledmatrix-wifi-monitor.service"; then fi echo "" -if [ "$SKIP_REBOOT_PROMPT" = "1" ]; then - echo "Skipping reboot prompt as requested (--no-reboot-prompt)." -elif [ "$ASSUME_YES" = "1" ]; then - echo "Non-interactive mode: rebooting now to apply changes..." - reboot -else - read -p "A reboot is recommended to apply kernel and audio changes. Reboot now? (y/N): " -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]]; then - echo "Rebooting now..." - reboot - fi -fi - echo "==========================================" echo "Installation Complete!" echo "==========================================" @@ -2013,7 +2010,7 @@ if command -v nmcli >/dev/null 2>&1; then if [ -n "$WIFI_STATUS" ]; then echo "$WIFI_STATUS" | while IFS=':' read -r _ _ state; do if [ "$state" = "connected" ]; then - SSID=$(nmcli -t -f active,ssid device wifi 2>/dev/null | grep "^yes:" | cut -d: -f2 | head -1) + SSID=$(nmcli -t -f active,ssid device wifi 2>/dev/null | grep "^yes:" | cut -d: -f2 | head -1 || true) if [ -n "$SSID" ]; then echo " ✓ Connected to: $SSID" else @@ -2037,7 +2034,7 @@ echo "AP Mode Status:" if systemctl is-active --quiet hostapd 2>/dev/null; then echo " ✓ AP Mode is ACTIVE" echo " → Connect to WiFi network: LEDMatrix-Setup" - echo " → Password: ledmatrix123" + echo " → Open network, no password" echo " → Access web UI at: http://192.168.4.1:5000" AP_MODE_ACTIVE=true else @@ -2045,7 +2042,7 @@ else if ip addr show wlan0 2>/dev/null | grep -q "192.168.4.1"; then echo " ✓ AP Mode is ACTIVE (IP detected)" echo " → Connect to WiFi network: LEDMatrix-Setup" - echo " → Password: ledmatrix123" + echo " → Open network, no password" echo " → Access web UI at: http://192.168.4.1:5000" AP_MODE_ACTIVE=true else @@ -2147,3 +2144,22 @@ echo " - Main config: $PROJECT_ROOT_DIR/config/config.json" echo " - Secrets: $PROJECT_ROOT_DIR/config/config_secrets.json" echo "" echo "Enjoy your LED Matrix display!" + +# Reboot last. It used to come before the summary above, so with -y (and +# the one-shot installer, which always passes -y) the reboot was already +# under way while the summary printed, and the SSH session usually dropped +# before any of it -- the web UI address included -- could be read. +echo "" +if [ "$SKIP_REBOOT_PROMPT" = "1" ]; then + echo "Skipping reboot prompt as requested (--no-reboot-prompt)." +elif [ "$ASSUME_YES" = "1" ]; then + echo "Non-interactive mode: rebooting now to apply changes..." + reboot +else + read -p "A reboot is recommended to apply kernel and audio changes. Reboot now? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo "Rebooting now..." + reboot + fi +fi diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..d4c4a5be --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,59 @@ +# Scripts + +Helper scripts for installing, repairing, diagnosing and developing +LEDMatrix. Most users only ever run the one-shot installer (see the project +README); everything else here is for troubleshooting or development. + +Status key: **keep** — part of install/runtime or referenced by docs, CI, +tests or code; **dev-only** — for plugin/core development, not needed on a +display; **diagnostic** — run by hand on a Pi when something is wrong. + +## Directories + +| Directory | Status | What it holds | +|---|---|---| +| [`install/`](install/README.md) | keep | The installers: one-shot, services, sudoers/WiFi permissions, cache setup, and the shared `lib_*.sh` helpers `first_time_install.sh` sources | +| [`fix_perms/`](fix_perms/README.md) | keep | Permission repair scripts, plus the two root helpers the web interface runs through sudo (`safe_plugin_rm.sh`, `safe_pip_install.sh`) | +| [`utils/`](utils/README.md) | keep | Scripts run by systemd units or the web interface (conditional web start, WiFi monitor, update verify, DNS fix, Pixlet config editor, cache clearing) | +| [`dev/`](dev/README.md) | dev-only | Plugin linking, emulator runner, Vegas density audit, Pillow smoke test | +| `templates/` | dev-only | `dev_preview.html`, the page `dev_server.py` serves | + +## Top-level scripts + +| Script | Status | What it does | +|---|---|---| +| `build_rgbmatrix_nogil.sh` | keep | Rebuilds the rgbmatrix Python binding so `SwapOnVSync` releases the GIL (docs/SCROLL_PERFORMANCE.md) | +| `check_plugin.py` | dev-only | Renders a plugin across every mode and matrix size and fails on crashes, overflow or golden-image drift | +| `check_release_version.py` | keep | Checks a release tag, CHANGELOG and `src.__version__` agree (release-version-check workflow) | +| `check_system_compatibility.sh` | diagnostic | Pre-install check of hardware, OS (Trixie only), kernel, Python, packages, disk and network | +| `dev_server.py` | dev-only | Browser preview server for plugins without the display loop (http://localhost:5001) | +| `diagnose_dependencies.sh` | diagnostic | Investigates pip installs stuck on "Preparing metadata" | +| `diagnose_web_interface.sh` | diagnostic | Checks why the web interface is not reachable | +| `download_pixlet.sh` | keep | Downloads the bundled Pixlet binaries for Starlark apps (also run from the web UI) | +| `emergency_reconnect.sh` | diagnostic | Reconnects to your WiFi network if captive-portal testing leaves the Pi offline | +| `install_dependencies_apt.py` | keep | Dependency installer that tries apt packages first, then pip (installer Step 7, plugin loader) | +| `install_plugin_dependencies.sh` | diagnostic | Installs plugin requirements by hand when the automatic install fails | +| `prove_security.py` | keep | Security property checks run by pre-commit | +| `render_plugin.py` | dev-only | Runs a plugin's `update()` + `display()` and saves the frame as a PNG | +| `run_plugin_tests.py` | dev-only | Discovers and runs plugin test suites | +| `scroll_speeds.py` | keep | Shows and tries the scroll speeds your panel can display cleanly | +| `troubleshoot_captive_portal.sh` | diagnostic | Troubleshoots captive-portal WiFi setup after you can SSH back in | +| `update_plugin_repos.py` | dev-only | Pulls the latest `ledmatrix-plugins` monorepo | +| `verify_installation.sh` | diagnostic | Checks that an installation completed correctly | +| `verify_wifi_setup.sh` | diagnostic | Health check of the WiFi management setup | + +## Candidates for removal + +Nothing in the repo (docs, CI, tests, other scripts or code) refers to these. +They are kept for now; each one needs an owner decision before it goes. + +| Script | What it does | +|---|---| +| `add_defaults_to_schemas.py` | One-off: adds missing `default` values to plugin config schemas | +| `analyze_plugin_schemas.py` | One-off: reports duplicate/inconsistent fields across plugin schemas | +| `audit_plugins.py` | AST security audit of plugin code; says it is "designed to run in CI" but no workflow runs it | +| `audit_render_path.py` | Finds blocking calls reachable from a plugin's `display()` | +| `sports_scroll_check.py` | Drives a sports scoreboard scroll on the panel and reports its pacing | +| `test_captive_portal.sh` | Tests the captive portal from a device connected to the AP | +| `verify_wifi_before_testing.sh` | Pre-flight check before unplugging Ethernet to test WiFi | +| `dev/test_pillow_compat.py` | Pillow API smoke test to run after upgrading Pillow | diff --git a/scripts/check_system_compatibility.sh b/scripts/check_system_compatibility.sh index e897277a..22cc5050 100755 --- a/scripts/check_system_compatibility.sh +++ b/scripts/check_system_compatibility.sh @@ -28,12 +28,12 @@ print_success() { print_warning() { echo -e "${YELLOW}⚠${NC} $1" - ((WARNINGS++)) + WARNINGS=$((WARNINGS + 1)) } print_error() { echo -e "${RED}✗${NC} $1" - ((COMPATIBILITY_ISSUES++)) + COMPATIBILITY_ISSUES=$((COMPATIBILITY_ISSUES + 1)) } # Check if running on Raspberry Pi @@ -61,20 +61,18 @@ if [ -f /etc/os-release ]; then echo "OS: $PRETTY_NAME" echo "Version ID: ${VERSION_ID:-unknown}" + # first_time_install.sh refuses anything but Raspberry Pi OS / Debian 13 + # (Trixie), so anything else is an error here too, not a warning. if [[ "$ID" == "raspbian" ]] || [[ "$ID" == "debian" ]]; then - if [ "${VERSION_ID:-0}" -ge "12" ]; then - print_success "Running compatible Debian/Raspbian version (${VERSION_ID})" - - if [ "${VERSION_ID:-0}" -eq "13" ]; then - print_success "Detected Debian 13 Trixie - full compatibility expected" - elif [ "${VERSION_ID:-0}" -eq "12" ]; then - print_success "Detected Debian 12 Bookworm - full compatibility confirmed" - fi + if [ "${VERSION_ID:-0}" = "13" ]; then + print_success "Detected Debian 13 Trixie - supported" + elif [ "${VERSION_ID:-0}" = "12" ]; then + print_error "Debian 12 Bookworm is not supported - the installer requires Raspberry Pi OS Lite (Trixie), Debian 13" else - print_warning "Old Debian/Raspbian version (${VERSION_ID}) - upgrade recommended" + print_error "Debian/Raspbian ${VERSION_ID:-unknown} is not supported - the installer requires Raspberry Pi OS Lite (Trixie), Debian 13" fi else - print_warning "Not running Debian/Raspbian - compatibility not guaranteed" + print_error "${ID:-unknown} is not supported - the installer requires Raspberry Pi OS Lite (Trixie), Debian 13" fi else print_error "Could not detect OS version" diff --git a/scripts/dev/README.md b/scripts/dev/README.md index 01b1ebc9..6dfdc676 100644 --- a/scripts/dev/README.md +++ b/scripts/dev/README.md @@ -6,6 +6,8 @@ This directory contains scripts and utilities for development and testing. - **`dev_plugin_setup.sh`** - Sets up plugin development environment by linking plugin repositories - **`run_emulator.sh`** - Runs the LED Matrix display in emulator mode (for development without hardware) +- **`vegas_audit.py`** - Measures how much of the Vegas ticker strip actually shows content (dead-frame ratio) +- **`test_pillow_compat.py`** - Pillow API smoke test to run after upgrading Pillow (`python3 scripts/dev/test_pillow_compat.py`) ## Usage diff --git a/scripts/fix_perms/README.md b/scripts/fix_perms/README.md index 77575a71..d3843ee4 100644 --- a/scripts/fix_perms/README.md +++ b/scripts/fix_perms/README.md @@ -18,21 +18,26 @@ system user. permissions on the `assets/` tree so plugins can download and cache team logos, fonts, and other static content. -- **`fix_cache_permissions.sh`** — Creates (if missing) and fixes - permissions on `/var/cache/ledmatrix/` and `~/.ledmatrix_cache/` of the - user running `sudo`, and creates - `/var/cache/ledmatrix/placeholder_logos/` for the sports plugins. It does - not touch the cache manager's other fallbacks (`/opt/ledmatrix/cache`, - `$TMPDIR/ledmatrix_cache`). +- **`fix_cache_permissions.sh`** — Restores `/var/cache/ledmatrix/` to the + shared `ledmatrix`-group setup by running + `scripts/install/setup_cache.sh` (the same script the installer uses), + and creates/fixes `~/.ledmatrix_cache/` of the user running `sudo`. It + does not touch the cache manager's other fallbacks + (`/opt/ledmatrix/cache`, `$TMPDIR/ledmatrix_cache`). - **`fix_plugin_permissions.sh`** — Fixes ownership on the plugins directory so both the root display service and the web service user can read and write plugin files (manifests, configs, requirements installs). -- **`fix_web_permissions.sh`** — Fixes permissions on log files, - systemd journal access, and the sudoers entries the web interface - needs to control the display service. +- **`fix_web_permissions.sh`** — Adds you to the `systemd-journal` and + `adm` groups so the web UI can read logs, and makes the project + directory yours again, keeping the root-owned sudo helpers + (`safe_plugin_rm.sh`, `safe_pip_install.sh`) and `config_secrets.json` + the way the installer leaves them. Run it as the web interface's user, + **without** `sudo` (it refuses to run as root and calls `sudo` itself). + It does not write sudoers rules; that is + `scripts/install/configure_web_sudo.sh`. - **`safe_pip_install.sh`** — Installs a `requirements.txt` as root after checking it is the project's own or one under `plugin-repos/` or @@ -67,7 +72,9 @@ Run these scripts only when: sudo ./scripts/fix_perms/fix_cache_permissions.sh sudo ./scripts/fix_perms/fix_assets_permissions.sh sudo ./scripts/fix_perms/fix_plugin_permissions.sh -sudo ./scripts/fix_perms/fix_web_permissions.sh + +# Run as the web interface's user, without sudo (it asks for sudo itself) +./scripts/fix_perms/fix_web_permissions.sh ``` If you're not sure which one you need, run `fix_cache_permissions.sh` diff --git a/scripts/fix_perms/fix_assets_permissions.sh b/scripts/fix_perms/fix_assets_permissions.sh index 9000a338..bd28ae45 100755 --- a/scripts/fix_perms/fix_assets_permissions.sh +++ b/scripts/fix_perms/fix_assets_permissions.sh @@ -36,11 +36,12 @@ else exit 1 fi -# Set permissions to allow read/write for owner, group, and others (for root service user) -# Note: 777 allows root (service user) to write, which is necessary when service runs as root +# 777: read/write for owner, group and every other account. Root (the display +# service) does not need it -- root ignores mode bits -- so the "other" bits +# only matter to accounts that are neither $REAL_USER nor root. echo "Setting permissions for assets directory..." if sudo chmod -R 777 "$ASSETS_DIR"; then - echo "✓ Set assets directory permissions to 777 (writable by root service user)" + echo "✓ Set assets directory permissions to 777" else echo "✗ Failed to set assets directory permissions" exit 1 @@ -70,8 +71,7 @@ for SPORTS_DIR in "${SPORTS_DIRS[@]}"; do echo " - Current permissions:" ls -ld "$FULL_PATH" - # Ensure the directory is writable by both the real user and root (service user) - # Use 777 permissions to allow root (service) to write, or set group ownership + # Owned by the real user; 777 as above (root can write here regardless) sudo chmod 777 "$FULL_PATH" sudo chown "$REAL_USER:$REAL_GROUP" "$FULL_PATH" diff --git a/scripts/fix_perms/fix_cache_permissions.sh b/scripts/fix_perms/fix_cache_permissions.sh index a3fa3e15..4225b376 100755 --- a/scripts/fix_perms/fix_cache_permissions.sh +++ b/scripts/fix_perms/fix_cache_permissions.sh @@ -1,11 +1,23 @@ #!/bin/bash # LEDMatrix Cache Permissions Fix Script -# This script fixes permissions on all known cache directories so they're writable by the daemon or current user -# Also sets up placeholder logo directories for sports managers +# +# /var/cache/ledmatrix is shared by the display service (root) and the web +# interface (your user) through the ledmatrix group: root:ledmatrix, 2775, +# files 660. scripts/install/setup_cache.sh is what sets that up (the +# installer's Step 2 runs it, and install_web_service.sh keeps the group), so +# this script runs it rather than applying a model of its own. It used to set +# the directory 777 and re-group it to your own group, replacing the ledmatrix +# group everything else relies on. +# +# It also repairs ~/.ledmatrix_cache, the cache manager's fallback when +# /var/cache/ledmatrix is unusable. echo "Fixing LEDMatrix cache directory permissions..." +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SETUP_CACHE="$SCRIPT_DIR/../install/setup_cache.sh" + # Get the real user (not root when running with sudo) REAL_USER=${SUDO_USER:-$USER} # Resolve the home directory of the real user robustly @@ -16,72 +28,36 @@ else fi REAL_GROUP=$(id -gn "$REAL_USER") -# Known cache directories for LEDMatrix. Use the actual user's home instead of a hard-coded path. -CACHE_DIRS=( - "/var/cache/ledmatrix" - "$REAL_HOME/.ledmatrix_cache" -) - -for CACHE_DIR in "${CACHE_DIRS[@]}"; do - echo "" - echo "Checking cache directory: $CACHE_DIR" - if [ ! -d "$CACHE_DIR" ]; then - echo " - Directory does not exist. Creating it..." - sudo mkdir -p "$CACHE_DIR" - fi - echo " - Current permissions:" - ls -ld "$CACHE_DIR" - echo " - Fixing permissions..." - # Make directory writable by services regardless of user context - sudo chmod 777 "$CACHE_DIR" - sudo chown "$REAL_USER":"$REAL_GROUP" "$CACHE_DIR" - echo " - Updated permissions:" - ls -ld "$CACHE_DIR" - echo " - Testing write access as $REAL_USER..." - if sudo -u "$REAL_USER" test -w "$CACHE_DIR"; then - echo " ✓ $CACHE_DIR is now writable by $REAL_USER" - else - echo " ✗ $CACHE_DIR is still not writable by $REAL_USER" - fi - echo " - Permissions fix complete for $CACHE_DIR." -done - -# Set up placeholder logos directory for sports managers echo "" -echo "Setting up placeholder logos directory for sports managers..." - -PLACEHOLDER_DIR="/var/cache/ledmatrix/placeholder_logos" -if [ ! -d "$PLACEHOLDER_DIR" ]; then - echo "Creating placeholder logos directory: $PLACEHOLDER_DIR" - sudo mkdir -p "$PLACEHOLDER_DIR" - sudo chown "$REAL_USER":"$REAL_GROUP" "$PLACEHOLDER_DIR" - sudo chmod 777 "$PLACEHOLDER_DIR" +echo "Checking cache directory: /var/cache/ledmatrix" +if [ -f "$SETUP_CACHE" ]; then + bash "$SETUP_CACHE" else - echo "Placeholder logos directory already exists: $PLACEHOLDER_DIR" - sudo chmod 777 "$PLACEHOLDER_DIR" - sudo chown "$REAL_USER":"$REAL_GROUP" "$PLACEHOLDER_DIR" + echo " ✗ $SETUP_CACHE not found; /var/cache/ledmatrix left unchanged." fi +CACHE_DIR="$REAL_HOME/.ledmatrix_cache" +echo "" +echo "Checking cache directory: $CACHE_DIR" +if [ ! -d "$CACHE_DIR" ]; then + echo " - Directory does not exist. Creating it..." + sudo mkdir -p "$CACHE_DIR" +fi echo " - Current permissions:" -ls -ld "$PLACEHOLDER_DIR" +ls -ld "$CACHE_DIR" +echo " - Fixing permissions..." +sudo chmod 777 "$CACHE_DIR" +sudo chown "$REAL_USER":"$REAL_GROUP" "$CACHE_DIR" +echo " - Updated permissions:" +ls -ld "$CACHE_DIR" echo " - Testing write access as $REAL_USER..." -if sudo -u "$REAL_USER" test -w "$PLACEHOLDER_DIR"; then - echo " ✓ Placeholder logos directory is writable by $REAL_USER" +if sudo -u "$REAL_USER" test -w "$CACHE_DIR"; then + echo " ✓ $CACHE_DIR is now writable by $REAL_USER" else - echo " ✗ Placeholder logos directory is not writable by $REAL_USER" -fi - -# Test with daemon user (which the system might run as) -if sudo -u daemon test -w "$PLACEHOLDER_DIR" 2>/dev/null; then - echo " ✓ Placeholder logos directory is writable by daemon user" -else - echo " ✗ Placeholder logos directory is not writable by daemon user" + echo " ✗ $CACHE_DIR is still not writable by $REAL_USER" fi +echo " - Permissions fix complete for $CACHE_DIR." echo "" echo "All cache directory permission fixes attempted." echo "If you still see errors, check which user is running the LEDMatrix service and ensure it matches the owner above." -echo "" -echo "The system will now create placeholder logos in:" -echo " $PLACEHOLDER_DIR" -echo "This should eliminate the permission denied warnings for sports logos." \ No newline at end of file diff --git a/scripts/fix_perms/fix_plugin_permissions.sh b/scripts/fix_perms/fix_plugin_permissions.sh index 94d96255..e5bd515d 100755 --- a/scripts/fix_perms/fix_plugin_permissions.sh +++ b/scripts/fix_perms/fix_plugin_permissions.sh @@ -51,9 +51,10 @@ fi echo "Setting ownership to root:$ACTUAL_USER..." sudo chown -R root:"$ACTUAL_USER" "$PLUGINS_DIR" -# Set directory permissions (775: rwxrwxr-x) -# Root: read/write/execute, Group (ACTUAL_USER): read/write/execute, Others: read/execute -echo "Setting directory permissions to 2775 (rwxrwxr-x + sticky bit)..." +# Set directory permissions (2775: rwxrwsr-x) +# Owner (root) and group (ACTUAL_USER): read/write/execute, others: read/execute. +# The setgid bit makes new entries inherit the ACTUAL_USER group. +echo "Setting directory permissions to 2775 (rwxrwsr-x, setgid)..." find "$PLUGINS_DIR" -type d -exec sudo chmod 2775 {} \; # Set file permissions (664: rw-rw-r--) @@ -71,7 +72,7 @@ fi echo "Setting ownership of plugin-repos to root:$ACTUAL_USER..." sudo chown -R root:"$ACTUAL_USER" "$PLUGIN_REPOS_DIR" -echo "Setting plugin-repos directory permissions to 2775 (rwxrwxr-x + sticky bit)..." +echo "Setting plugin-repos directory permissions to 2775 (rwxrwsr-x, setgid)..." find "$PLUGIN_REPOS_DIR" -type d -exec sudo chmod 2775 {} \; echo "Setting plugin-repos file permissions to 664..." @@ -87,7 +88,7 @@ echo "plugin-repos/:" ls -la "$PLUGIN_REPOS_DIR" 2>/dev/null || echo " (empty or not accessible)" echo "" echo "Permissions summary:" -echo "- Root service: Can read/write plugins (for PWM hardware access)" +echo "- Root service: Can read/write plugins (as root it needs no permission bits)" echo "- Web service ($ACTUAL_USER): Can read/write plugins (for installation)" echo "- Others: Can read plugins" diff --git a/scripts/fix_perms/fix_web_permissions.sh b/scripts/fix_perms/fix_web_permissions.sh index 51b8fb7f..2a916f42 100755 --- a/scripts/fix_perms/fix_web_permissions.sh +++ b/scripts/fix_perms/fix_web_permissions.sh @@ -25,8 +25,9 @@ echo "" echo "This script will:" echo "1. Add the web user to the 'systemd-journal' group for log access" echo "2. Add the web user to the 'adm' group for additional system access" -echo "3. Configure sudoers for passwordless access to system commands" -echo "4. Set proper file permissions" +echo "3. Make the project directory yours again, keeping the root-owned sudo" +echo " helpers and config_secrets.json as the installer leaves them" +echo " (sudoers rules are configure_web_sudo.sh's job, not this script's)" echo "" # Ask for confirmation @@ -62,6 +63,51 @@ else echo "✗ Failed to set project ownership" fi +# The chown above also takes back two kinds of file that first_time_install.sh +# deliberately keeps from the web user. Put them back the way the installer +# leaves them (its Steps 11 and 11.1), whether or not the chown succeeded. +# +# 1. The helpers /etc/sudoers.d/ledmatrix_web lets the web user run as root +# (scripts/install/lib_sudoers.sh). A copy the web user owns is a root shell +# for whoever can edit it, so they stay root-owned and writable by root only. +# Keep this list in step with the installer's Step 11.1 loop; +# test/test_web_sudoers_installers_agree.py checks both against the grants. +for helper in safe_plugin_rm.sh safe_pip_install.sh; do + HELPER_PATH="$PROJECT_DIR/scripts/fix_perms/$helper" + if [ -f "$HELPER_PATH" ]; then + if sudo chown root:root "$HELPER_PATH" && sudo chmod 755 "$HELPER_PATH"; then + echo "✓ $helper is root-owned again (sudo runs it as root)" + else + echo "⚠ Could not make $HELPER_PATH root-owned, mode 755." + echo " Fix it by hand: sudo chown root:root $HELPER_PATH && sudo chmod 755 $HELPER_PATH" + fi + fi +done + +# 2. config_secrets.json: owned by the account ledmatrix-web.service runs as, +# group ledmatrix, mode 640 -- the same owner, group and mode as the +# installer's Step 11 gives it. +SECRETS_FILE="$PROJECT_DIR/config/config_secrets.json" +if [ -f "$SECRETS_FILE" ]; then + SECRETS_OWNER="" + if [ -f /etc/systemd/system/ledmatrix-web.service ]; then + SECRETS_OWNER=$(grep -m1 "^User=" /etc/systemd/system/ledmatrix-web.service | cut -d'=' -f2 || true) + fi + SECRETS_OWNER="${SECRETS_OWNER:-$WEB_USER}" + if getent group ledmatrix >/dev/null 2>&1; then + SECRETS_OWNERSHIP="$SECRETS_OWNER:ledmatrix" + else + # No ledmatrix group means the installer never ran; keep the chown's group. + SECRETS_OWNERSHIP="$SECRETS_OWNER" + fi + if sudo chown "$SECRETS_OWNERSHIP" "$SECRETS_FILE" && sudo chmod 640 "$SECRETS_FILE"; then + echo "✓ config_secrets.json restored to $SECRETS_OWNERSHIP, mode 640" + else + echo "⚠ Could not restore $SECRETS_FILE to $SECRETS_OWNERSHIP, mode 640." + echo " Fix it by hand: sudo chown $SECRETS_OWNERSHIP $SECRETS_FILE && sudo chmod 640 $SECRETS_FILE" + fi +fi + # Set proper permissions for config files if sudo chmod 644 "$PROJECT_DIR/config/config.json" 2>/dev/null; then echo "✓ Set config file permissions" @@ -86,7 +132,7 @@ echo "Step 5: Testing sudo access..." if sudo -n systemctl status ledmatrix.service > /dev/null 2>&1; then echo "✓ Sudo access test passed" else - echo "⚠ Sudo access test failed - you may need to run configure_web_sudo.sh" + echo "⚠ Sudo access test failed - you may need to run scripts/install/configure_web_sudo.sh" fi echo "" @@ -101,5 +147,5 @@ echo "" echo "After logging back in, test journal access with:" echo " journalctl --no-pager --lines=5" echo "" -echo "If you still have sudo issues, run:" -echo " ./configure_web_sudo.sh" +echo "If you still have sudo issues, run (as this user, without sudo):" +echo " $PROJECT_DIR/scripts/install/configure_web_sudo.sh" diff --git a/scripts/install/README.md b/scripts/install/README.md index 544ea336..530960f7 100644 --- a/scripts/install/README.md +++ b/scripts/install/README.md @@ -19,6 +19,21 @@ This directory contains scripts for installing and configuring the LEDMatrix sys (the user who runs the script, i.e. the one you installed LEDMatrix as; there is no `ledmatrix` system user) the passwordless `nmcli` and related WiFi permissions the web interface needs +- **`install_dns_fix.sh`** - Optional. Installs `ledmatrix-dns-fix.service`, + which adds `options single-request` to the resolver when API calls time + out (see `systemd/README.md`) +- **`install_mqtt_bridge.sh`** - Optional. Installs the Home Assistant MQTT + bridge service (see `integrations/mqtt_bridge/README.md`) + +Libraries (sourced, not run): + +- **`lib_sudoers.sh`** - The web interface's sudo allow-list + (`/etc/sudoers.d/ledmatrix_web`), shared by `first_time_install.sh` and + `configure_web_sudo.sh` +- **`lib_systemd_render.sh`** - `sed_escape_replacement`, used by every + script that renders a unit from `systemd/*.service` +- **`lib_lowmem.sh`** - Build-job sizing and temporary swap for the C++ + build on low-memory Pis (`first_time_install.sh` Step 6) ## Usage diff --git a/scripts/install/configure_web_sudo.sh b/scripts/install/configure_web_sudo.sh index 91ba1d3b..c0fc0a06 100755 --- a/scripts/install/configure_web_sudo.sh +++ b/scripts/install/configure_web_sudo.sh @@ -26,7 +26,6 @@ fi # Get the full paths to commands and validate each one MISSING_CMDS=() -PYTHON_PATH=$(command -v python3) || true SYSTEMCTL_PATH=$(command -v systemctl) || true REBOOT_PATH=$(command -v reboot) || true POWEROFF_PATH=$(command -v poweroff) || true @@ -35,8 +34,8 @@ JOURNALCTL_PATH=$(command -v journalctl) || true SAFE_RM_PATH="$PROJECT_ROOT/scripts/fix_perms/safe_plugin_rm.sh" SAFE_PIP_INSTALL_PATH="$PROJECT_ROOT/scripts/fix_perms/safe_pip_install.sh" -# Validate required commands (systemctl, bash, python3 are essential) -for CMD_NAME in SYSTEMCTL_PATH BASH_PATH PYTHON_PATH; do +# Validate required commands (systemctl and bash are essential) +for CMD_NAME in SYSTEMCTL_PATH BASH_PATH; do CMD_VAL="${!CMD_NAME}" if [ -z "$CMD_VAL" ]; then MISSING_CMDS+=("$CMD_NAME") @@ -70,7 +69,6 @@ fi . "$SUDOERS_LIB" echo "Command paths:" -echo " Python: $PYTHON_PATH" echo " Systemctl: $SYSTEMCTL_PATH" echo " Reboot: ${REBOOT_PATH:-(not found, skipping)}" echo " Poweroff: ${POWEROFF_PATH:-(not found, skipping)}" @@ -79,14 +77,24 @@ echo " Journalctl: ${JOURNALCTL_PATH:-(not found, skipping)}" echo " Safe plugin rm: $SAFE_RM_PATH" echo " Safe pip install: $SAFE_PIP_INSTALL_PATH" -# Create a temporary sudoers file -TEMP_SUDOERS="/tmp/ledmatrix_web_sudoers_$$" +# Create a temporary sudoers file. A predictable name in a world-writable +# directory is a symlink target, and these rules end up in /etc/sudoers.d, so +# let mktemp pick the name; the trap removes it however the script ends. +TEMP_SUDOERS=$(mktemp "${TMPDIR:-/tmp}/ledmatrix_web_sudoers.XXXXXX") || { + echo "Error: could not create a temporary file" >&2 + exit 1 +} +trap 'rm -f "$TEMP_SUDOERS"' EXIT web_sudoers_rules "$WEB_USER" "$PROJECT_ROOT" "$SYSTEMCTL_PATH" "$BASH_PATH" \ "$REBOOT_PATH" "$POWEROFF_PATH" "$JOURNALCTL_PATH" > "$TEMP_SUDOERS" # Never offer to install rules we have not parsed. A malformed drop-in in # /etc/sudoers.d makes sudo refuse every command for every user. +# visudo lives in /usr/sbin, which is not on every user's PATH. +if ! command -v visudo >/dev/null 2>&1 && [ -x /usr/sbin/visudo ]; then + PATH="$PATH:/usr/sbin" +fi if command -v visudo >/dev/null 2>&1; then if ! visudo -c -f "$TEMP_SUDOERS" >/dev/null 2>&1; then echo "" @@ -96,6 +104,8 @@ if command -v visudo >/dev/null 2>&1; then rm -f "$TEMP_SUDOERS" exit 1 fi +else + echo "⚠ visudo not found; the rules below have not been validated" fi echo "" @@ -124,7 +134,7 @@ if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 0 fi -# Apply the configuration using visudo +# Apply the configuration echo "Applying sudoers configuration..." # Harden the helper script: root-owned, not writable by web user echo "Hardening safe_plugin_rm.sh ownership..." @@ -143,21 +153,29 @@ if ! sudo chmod 755 "$SAFE_PIP_INSTALL_PATH"; then fi if sudo cp "$TEMP_SUDOERS" /etc/sudoers.d/ledmatrix_web; then + # sudo reads /etc/sudoers.d files that are root-owned and not writable by + # group or other; 440 is the mode visudo and first_time_install.sh use. + if ! sudo chmod 440 /etc/sudoers.d/ledmatrix_web; then + echo "Warning: could not set mode 440 on /etc/sudoers.d/ledmatrix_web" + fi echo "Configuration applied successfully!" echo "" echo "Testing sudo access..." - # Test a few commands - if sudo -n systemctl status ledmatrix.service > /dev/null 2>&1; then + # Ask sudo whether two of the new rules let this user in without a + # password. `sudo -l CMD` answers from the rules without running CMD, so + # this does not depend on whether ledmatrix.service is running, and it + # tests commands the rules actually grant. + if sudo -n -l "$SYSTEMCTL_PATH" status ledmatrix.service > /dev/null 2>&1; then echo "✓ systemctl status ledmatrix.service - OK" else - echo "✗ systemctl status ledmatrix.service - Failed" + echo "✗ systemctl status ledmatrix.service - not allowed without a password" fi - - if sudo -n test -f "$PROJECT_ROOT/start_display.sh"; then - echo "✓ File access test - OK" + + if sudo -n -l "$BASH_PATH" "$SAFE_RM_PATH" "$PROJECT_ROOT/plugin-repos/example" > /dev/null 2>&1; then + echo "✓ safe_plugin_rm.sh helper - OK" else - echo "✗ File access test - Failed" + echo "✗ safe_plugin_rm.sh helper - not allowed without a password" fi echo "" diff --git a/scripts/install/configure_wifi_permissions.sh b/scripts/install/configure_wifi_permissions.sh index 37e2c7db..3db862d1 100755 --- a/scripts/install/configure_wifi_permissions.sh +++ b/scripts/install/configure_wifi_permissions.sh @@ -144,6 +144,11 @@ $WEB_USER ALL=(ALL) NOPASSWD: $MKDIR_PATH -p /etc/NetworkManager/dnsmasq-shared. $WEB_USER ALL=(ALL) NOPASSWD: /usr/bin/cp /tmp/hostapd.conf /etc/hostapd/hostapd.conf $WEB_USER ALL=(ALL) NOPASSWD: /usr/bin/cp /tmp/dnsmasq.conf /etc/dnsmasq.d/ledmatrix-captive.conf $WEB_USER ALL=(ALL) NOPASSWD: /usr/bin/rm -f /etc/dnsmasq.d/ledmatrix-captive.conf +# The same captive-portal DNS drop-in for NetworkManager's shared-mode dnsmasq +# (wifi_manager._write_nm_dnsmasq_captive_conf / _remove_nm_dnsmasq_captive_conf), +# exact paths. +$WEB_USER ALL=(ALL) NOPASSWD: /usr/bin/cp /tmp/ledmatrix-nm-dnsmasq.conf /etc/NetworkManager/dnsmasq-shared.d/ledmatrix-captive.conf +$WEB_USER ALL=(ALL) NOPASSWD: /usr/bin/rm -f /etc/NetworkManager/dnsmasq-shared.d/ledmatrix-captive.conf EOF echo "Generated sudoers configuration:" @@ -151,6 +156,21 @@ echo "--------------------------------" cat "$TEMP_SUDOERS" echo "--------------------------------" +# Never install rules we have not parsed. A malformed drop-in in +# /etc/sudoers.d makes sudo refuse every command for every user, which on a +# headless Pi leaves no way in at all. first_time_install.sh and +# configure_web_sudo.sh check their rules the same way. +if command -v visudo >/dev/null 2>&1; then + if ! visudo -c -f "$TEMP_SUDOERS" >/dev/null 2>&1; then + echo "✗ The generated sudoers rules did not parse:" >&2 + visudo -c -f "$TEMP_SUDOERS" >&2 || true + echo " Leaving $SUDOERS_FILE unchanged." >&2 + exit 1 + fi +else + echo "⚠ visudo not found; installing the sudoers rules unvalidated" +fi + # Apply the sudoers configuration echo "" echo "Applying sudoers configuration..." @@ -213,11 +233,14 @@ rm -f "$TEMP_POLKIT" echo "" echo "Step 3: Testing permissions..." -# Test sudo access -if sudo -n "$NMCLI_PATH" device status > /dev/null 2>&1; then - echo "✓ nmcli device status - OK" +# Ask sudo whether one of the new rules lets this user in without a password. +# `sudo -l CMD` answers from the rules without running CMD, so the radio is +# left alone. (This used to run `nmcli device status`, which is not granted, +# so it could only ever report a failure.) +if sudo -n -l "$NMCLI_PATH" radio wifi on > /dev/null 2>&1; then + echo "✓ nmcli radio wifi on - OK" else - echo "✗ nmcli device status - Failed (this is expected if not connected)" + echo "✗ nmcli radio wifi on - not allowed without a password" fi echo "" diff --git a/scripts/install/install_dns_fix.sh b/scripts/install/install_dns_fix.sh index 1edf2785..99060517 100755 --- a/scripts/install/install_dns_fix.sh +++ b/scripts/install/install_dns_fix.sh @@ -10,6 +10,10 @@ set -e PROJECT_ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd) + +# shellcheck source=scripts/install/lib_systemd_render.sh +source "$PROJECT_ROOT_DIR/scripts/install/lib_systemd_render.sh" + SERVICE_NAME="ledmatrix-dns-fix" UNIT_SRC="$PROJECT_ROOT_DIR/systemd/$SERVICE_NAME.service" UNIT_DEST="/etc/systemd/system/$SERVICE_NAME.service" @@ -34,7 +38,8 @@ fi chmod +x "$PROJECT_ROOT_DIR/scripts/utils/apply_dns_single_request.sh" echo "Installing $UNIT_DEST..." -sed "s|__PROJECT_ROOT_DIR__|$PROJECT_ROOT_DIR|g" "$UNIT_SRC" \ +ESCAPED_PROJECT_ROOT_DIR=$(sed_escape_replacement "$PROJECT_ROOT_DIR") +sed "s|__PROJECT_ROOT_DIR__|$ESCAPED_PROJECT_ROOT_DIR|g" "$UNIT_SRC" \ | $SUDO tee "$UNIT_DEST" > /dev/null # Order ledmatrix.service after the fix. `Before=` in the unit itself only diff --git a/scripts/install/install_mqtt_bridge.sh b/scripts/install/install_mqtt_bridge.sh index 7e65ed08..86eea9bd 100755 --- a/scripts/install/install_mqtt_bridge.sh +++ b/scripts/install/install_mqtt_bridge.sh @@ -9,6 +9,10 @@ set -e PROJECT_ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd) + +# shellcheck source=scripts/install/lib_systemd_render.sh +source "$PROJECT_ROOT_DIR/scripts/install/lib_systemd_render.sh" + BRIDGE_DIR="$PROJECT_ROOT_DIR/integrations/mqtt_bridge" SERVICE_NAME="ledmatrix-mqtt-bridge" UNIT_SRC="$PROJECT_ROOT_DIR/systemd/$SERVICE_NAME.service" @@ -40,7 +44,8 @@ python3 -m pip install -r "$BRIDGE_DIR/requirements.txt" 2>/dev/null \ || python3 -m pip install --break-system-packages -r "$BRIDGE_DIR/requirements.txt" echo "Installing $UNIT_DEST..." -sed "s|__PROJECT_ROOT_DIR__|$PROJECT_ROOT_DIR|g" "$UNIT_SRC" \ +ESCAPED_PROJECT_ROOT_DIR=$(sed_escape_replacement "$PROJECT_ROOT_DIR") +sed "s|__PROJECT_ROOT_DIR__|$ESCAPED_PROJECT_ROOT_DIR|g" "$UNIT_SRC" \ | $SUDO tee "$UNIT_DEST" > /dev/null $SYSTEMCTL_CMD daemon-reload diff --git a/scripts/install/install_wifi_monitor.sh b/scripts/install/install_wifi_monitor.sh index 0c24b0b8..af934aa4 100755 --- a/scripts/install/install_wifi_monitor.sh +++ b/scripts/install/install_wifi_monitor.sh @@ -51,20 +51,25 @@ if [ ${#MISSING_PACKAGES[@]} -gt 0 ]; then # Install packages automatically (no prompt) # Use apt directly if running as root, otherwise use sudo + PACKAGES_OK=true if [ "$EUID" -eq 0 ]; then apt update || echo "⚠ apt update failed, continuing anyway..." apt install -y "${MISSING_PACKAGES[@]}" || { + PACKAGES_OK=false echo "⚠ Package installation failed, but continuing with WiFi monitor setup" echo " You may need to install packages manually: apt install -y ${MISSING_PACKAGES[*]}" } else sudo apt update || echo "⚠ apt update failed, continuing anyway..." sudo apt install -y "${MISSING_PACKAGES[@]}" || { + PACKAGES_OK=false echo "⚠ Package installation failed, but continuing with WiFi monitor setup" echo " You may need to install packages manually: sudo apt install -y ${MISSING_PACKAGES[*]}" } fi - echo "✓ Package installation completed" + if [ "$PACKAGES_OK" = true ]; then + echo "✓ Package installation completed" + fi fi # Render the unit from systemd/ledmatrix-wifi-monitor.service rather than diff --git a/scripts/install/lib_systemd_render.sh b/scripts/install/lib_systemd_render.sh index d07dcec1..1431b06e 100644 --- a/scripts/install/lib_systemd_render.sh +++ b/scripts/install/lib_systemd_render.sh @@ -2,9 +2,10 @@ # # Shared helper for rendering systemd unit templates via sed. # -# Sourced by install_service.sh, install_web_service.sh and -# install_wifi_monitor.sh so all three escape sed replacement text the same -# way instead of carrying three copies of the same fix. +# Sourced by install_service.sh, install_web_service.sh, +# install_wifi_monitor.sh, install_dns_fix.sh and install_mqtt_bridge.sh so +# every unit renderer escapes sed replacement text the same way instead of +# carrying its own copy of the fix. # sed_escape_replacement VALUE # diff --git a/scripts/install/one-shot-install.sh b/scripts/install/one-shot-install.sh index 37276027..cec278b0 100755 --- a/scripts/install/one-shot-install.sh +++ b/scripts/install/one-shot-install.sh @@ -205,27 +205,6 @@ check_sudo() { print_success "Sudo access confirmed" } -# Fix /tmp permissions if needed (common issue when running via curl | bash) -# Note: /tmp permission fixing is now done inline before running first_time_install.sh -# This function is kept for backward compatibility but not actively used -fix_tmp_permissions() { - CURRENT_STEP="TMP directory check" - # Only fix if /tmp is actually not writable (don't preemptively fix) - if [ ! -w /tmp ]; then - print_warning "/tmp is not writable, attempting to fix..." - if [ "$EUID" -eq 0 ]; then - chmod 1777 /tmp 2>/dev/null || true - else - sudo chmod 1777 /tmp 2>/dev/null || true - fi - fi - - # Ensure TMPDIR is set correctly - if [ -z "${TMPDIR:-}" ] || [ ! -w "${TMPDIR:-/tmp}" ]; then - export TMPDIR=/tmp - fi -} - # Main installation function main() { print_step "LED Matrix One-Shot Installation" @@ -429,6 +408,13 @@ main() { print_step "Installation Complete!" print_success "LED Matrix has been successfully installed!" echo "" + # first_time_install.sh -y reboots as its last action, so by now the + # reboot is under way (unless LEDMATRIX_SKIP_REBOOT_PROMPT=1 was set). + if [ "${LEDMATRIX_SKIP_REBOOT_PROMPT:-0}" != "1" ]; then + echo "The installer has just started a reboot to finish setup, so this" + echo "session may disconnect now. Give the Pi a few minutes to come back, then:" + echo "" + fi echo "Next steps:" echo " 1. Configure your settings: sudo nano $REPO_DIR/config/config.json" if command -v hostname >/dev/null 2>&1; then @@ -449,7 +435,7 @@ main() { else echo " 2. Or use the web interface: http://:5000" fi - echo " 3. Start the service: sudo systemctl start ledmatrix.service" + echo " 3. The display service starts on boot; to start it by hand: sudo systemctl start ledmatrix.service" echo "" else print_error "Main installation script exited with code $INSTALL_EXIT_CODE" diff --git a/scripts/utils/README.md b/scripts/utils/README.md index 9339d65d..6df34130 100644 --- a/scripts/utils/README.md +++ b/scripts/utils/README.md @@ -9,6 +9,7 @@ This directory contains utility scripts for maintenance and system operations. - **`wifi_monitor_daemon.py`** - Background daemon that monitors WiFi/Ethernet connection and manages access point mode - **`pixlet_config_editor.sh`** - Opens Pixlet's own config UI for one installed Starlark app - **`apply_dns_single_request.sh`** - Adds `options single-request` to the resolver (run by `ledmatrix-dns-fix.service`) +- **`auto_update_verify.py`** - Health check after an automatic update, rolling back if it fails (the updater copies it to `data/` before pulling and `ledmatrix-update-verify.service` runs that copy) ## Usage diff --git a/src/adaptive_images.py b/src/adaptive_images.py index 72c0907a..c7a35a36 100644 --- a/src/adaptive_images.py +++ b/src/adaptive_images.py @@ -29,13 +29,9 @@ from typing import Any, Optional, Tuple from PIL import Image -# The one Pillow >= 9.1 compat shim (replaces the per-plugin copies). -try: - RESAMPLE_LANCZOS = Image.Resampling.LANCZOS - RESAMPLE_NEAREST = Image.Resampling.NEAREST -except AttributeError: # Pillow < 9.1 - RESAMPLE_LANCZOS = Image.LANCZOS - RESAMPLE_NEAREST = Image.NEAREST +# Re-exported by src.common for plugins, which import them from there. +RESAMPLE_LANCZOS = Image.Resampling.LANCZOS +RESAMPLE_NEAREST = Image.Resampling.NEAREST FIT_MODES = ("contain", "cover", "fill_height", "stretch") diff --git a/src/background_data_service.py b/src/background_data_service.py index a41aef45..15ac2b4b 100644 --- a/src/background_data_service.py +++ b/src/background_data_service.py @@ -131,19 +131,14 @@ class BackgroundDataService: # Thread management self.executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="BackgroundData") - # cache_key -> request_id for fetches currently in flight. Submitting - # the same key twice used to start two identical fetches: request_id - # carries a millisecond timestamp, so every submit looked new, and - # active_requests is keyed by it rather than by what is being fetched. - # On a real board the season-schedule key is requested by both the - # Recent and the Upcoming manager, which miss the cache in the same - # millisecond and each download and parse the same payload. + # cache_key -> request_id for fetches currently in flight, so a second + # submit for the same key joins the running fetch instead of starting + # another. It is the normal case: a sport's Recent and Upcoming + # managers miss the cache for the same season schedule together. self._inflight_by_cache_key: Dict[str, str] = {} - # request_id was sport_year_milliseconds, which is not unique: two - # submits inside the same millisecond produced the SAME id, so one - # silently replaced the other in active_requests and completed_requests. - # Rare before, but dedupe hands this id back to every joiner as their - # handle for get_result(), so it has to be unique. A counter is enough. + # Makes every request_id unique. The id also carries a millisecond + # timestamp, but two submits can share a millisecond, and a joiner + # uses the id as its handle for get_result(). self._request_seq = itertools.count() self.active_requests: Dict[str, FetchRequest] = {} self.completed_requests: Dict[str, FetchResult] = {} @@ -186,9 +181,9 @@ class BackgroundDataService: This ensures Recent/Upcoming managers and background service use the same cache keys. """ - # Same format as CacheManager.generate_sport_cache_key(). This used to - # build a whole CacheManager to call it -- config load, cache-dir - # probing with test writes -- on every submit without a cache_key. + # Same format as CacheManager.generate_sport_cache_key(), built here + # rather than by constructing a CacheManager (config load, cache-dir + # probing) on every submit without a cache_key. if date_str is None: date_str = datetime.now(pytz.utc).strftime('%Y%m%d') return f"{sport}_{date_str}" @@ -331,10 +326,8 @@ class BackgroundDataService: try: with self._lock: - # A request cancelled while it sat in the executor queue must - # stay cancelled. Overwriting the status here undid the cancel - # outright: the worker went on to download, cache and call back - # for work the caller had already withdrawn. + # A request cancelled while it sat in the executor queue stays + # cancelled: no download, no cache write, no callback. if request.status == FetchStatus.CANCELLED: cancelled_before_start = True else: @@ -463,10 +456,9 @@ class BackgroundDataService: logger.error(f"Failed to fetch {request.sport} {request.year} data: {error_msg}") with self._lock: - # Don't relabel a cancelled request. The callback gate in the - # finally block only suppresses CANCELLED, so promoting it to - # FAILED here delivered an error callback for a fetch nobody - # was waiting on any more. + # A cancelled request stays CANCELLED even when its fetch + # failed: the finally block skips callbacks only for + # CANCELLED, and nobody is waiting on this fetch any more. if request.status != FetchStatus.CANCELLED: request.status = FetchStatus.FAILED request.error = error_msg @@ -526,20 +518,13 @@ class BackgroundDataService: except Exception as e: logger.error(f"Error in callback for request {request.id}: {e}") - # Released AFTER the loop, not inside it. Every callback here holds - # the same FetchResult, so releasing per-delivery handed the first - # one the data and every joiner `result.data is None` -- which is - # not a quiet degradation: they read `result.data.get('events')` and - # raise AttributeError, which this very loop catches and logs, so - # the symptom was one ERROR line and a manager that silently never - # got its schedule. Deduplication is the normal case, not a corner: - # a sport's recent, upcoming and live managers all ride one season - # fetch. + # Released after the loop, never inside it: every callback holds + # the same FetchResult (a sport's recent, upcoming and live + # managers usually share one fetch), so a release between + # deliveries would hand the later ones `result.data is None`. # - # Guarded on `callbacks`, because a request submitted without one - # has no other way to collect its payload than polling get_result(). - # The old per-delivery release got that right by accident: an empty - # list never entered the loop body. + # Only when there were callbacks: a request submitted without one + # collects its payload by polling get_result(). if callbacks: self._release_payload(result) request.result = None @@ -721,9 +706,6 @@ class BackgroundDataService: 'completed_requests_count': len(self.completed_requests), 'max_completed_requests': self._max_completed_requests, 'completed_requests_usage_percent': (len(self.completed_requests) / self._max_completed_requests * 100) if self._max_completed_requests > 0 else 0, - # Nothing is queued outside the executor; kept for callers - # that read the key. - 'queue_size': 0, 'last_cleanup': self._last_completed_requests_cleanup, 'cleanup_interval': self._completed_requests_cleanup_interval } @@ -793,27 +775,6 @@ class BackgroundDataService: return removed_count - def clear_completed_requests(self, older_than_hours: int = 24): - """ - Clear completed requests older than specified time. - - Args: - older_than_hours: Clear requests older than this many hours - """ - cutoff_time = time.time() - (older_than_hours * 3600) - - with self._lock: - to_remove = [] - for request_id, result in self.completed_requests.items(): - if result.completed_at < cutoff_time: - to_remove.append(request_id) - - for request_id in to_remove: - del self.completed_requests[request_id] - - if to_remove: - logger.info(f"Cleared {len(to_remove)} old completed requests") - def shutdown(self, wait: bool = True): """ Shutdown the background data service. diff --git a/src/backup_manager.py b/src/backup_manager.py index 13114114..490db4dd 100644 --- a/src/backup_manager.py +++ b/src/backup_manager.py @@ -83,14 +83,25 @@ BUNDLED_FONTS: frozenset[str] = frozenset({ _CONFIG_REL = Path("config/config.json") _SECRETS_REL = Path("config/config_secrets.json") _WIFI_REL = Path("config/wifi_config.json") -# Sits in config/ next to the three above and is pure user state — a -# YouTube Music session that has to be re-authenticated by hand if lost. -# It was omitted from backups, so a restore silently signed the user out. +# A YouTube Music session: pure user state that has to be re-authenticated by +# hand if lost, so a restore must bring it back. _YTM_REL = Path("config/ytm_auth.json") _FONTS_REL = Path("assets/fonts") _PLUGIN_UPLOADS_REL = Path("assets/plugins") _STATE_REL = Path("data/plugin_state.json") +#: The sections that are one file each: (section name, path, the +#: RestoreOptions flag that restores it). create, preview, validate and +#: restore all walk this table. ytm_auth follows restore_wifi: it is +#: device-local auth like the Wi-Fi settings, and a toggle of its own for one +#: file would be noise in the restore dialog. +_SINGLE_FILE_SECTIONS: Tuple[Tuple[str, Path, str], ...] = ( + ("config", _CONFIG_REL, "restore_config"), + ("secrets", _SECRETS_REL, "restore_secrets"), + ("wifi", _WIFI_REL, "restore_wifi"), + ("ytm_auth", _YTM_REL, "restore_wifi"), +) + MANIFEST_NAME = "manifest.json" PLUGINS_MANIFEST_NAME = "plugins.json" @@ -140,34 +151,18 @@ class RestoreResult: # --------------------------------------------------------------------------- -def _ledmatrix_version(project_root: Path) -> str: - """Best-effort version string for the current install.""" - version_file = project_root / "VERSION" - if version_file.exists(): - try: - return version_file.read_text(encoding="utf-8").strip() or "unknown" - except OSError: - pass - head_file = project_root / ".git" / "HEAD" - if head_file.exists(): - try: - head = head_file.read_text(encoding="utf-8").strip() - if head.startswith("ref: "): - ref = head[5:] - ref_path = project_root / ".git" / ref - if ref_path.exists(): - return ref_path.read_text(encoding="utf-8").strip()[:12] or "unknown" - return head[:12] or "unknown" - except OSError: - pass - return "unknown" +def _ledmatrix_version() -> str: + """The release of the running core (``src.__version__``), recorded in the + manifest so a restore can tell which release wrote the backup.""" + from src import __version__ + return __version__ -def _build_manifest(contents: List[str], project_root: Path) -> Dict[str, Any]: +def _build_manifest(contents: List[str]) -> Dict[str, Any]: return { "schema_version": SCHEMA_VERSION, "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "ledmatrix_version": _ledmatrix_version(project_root), + "ledmatrix_version": _ledmatrix_version(), "hostname": socket.gethostname(), "contents": contents, } @@ -178,13 +173,34 @@ def _build_manifest(contents: List[str], project_root: Path) -> Dict[str, Any]: # --------------------------------------------------------------------------- +def _plugins_directory(project_root: Path) -> Path: + """The plugin install directory: ``plugin_system.plugins_directory`` from + config/config.json (relative to ``project_root`` unless absolute), or + ``plugin-repos`` when the config does not say or cannot be read.""" + configured: Any = None + try: + with (project_root / _CONFIG_REL).open("r", encoding="utf-8") as f: + config = json.load(f) + if isinstance(config, dict): + plugin_system = config.get("plugin_system") + if isinstance(plugin_system, dict): + configured = plugin_system.get("plugins_directory") + except (OSError, json.JSONDecodeError): + pass + if not isinstance(configured, str) or not configured.strip(): + configured = "plugin-repos" + path = Path(configured) + return path if path.is_absolute() else project_root / path + + def list_installed_plugins(project_root: Path) -> List[Dict[str, Any]]: """ Return a list of currently-installed plugins suitable for the backup manifest. Each entry has ``plugin_id`` and ``version``. - Reads ``data/plugin_state.json`` if present; otherwise walks the plugin - directory and reads each ``manifest.json``. + Reads ``data/plugin_state.json`` if present, then adds any plugin it + does not list from the ``manifest.json`` files in the configured plugin + directory (see :func:`_plugins_directory`). """ plugins: Dict[str, Dict[str, Any]] = {} @@ -206,8 +222,7 @@ def list_installed_plugins(project_root: Path) -> List[Dict[str, Any]]: except (OSError, json.JSONDecodeError) as e: logger.warning("Could not read plugin_state.json: %s", e) - # Fall back to scanning plugin-repos/ for manifests. - plugins_root = project_root / "plugin-repos" + plugins_root = _plugins_directory(project_root) if plugins_root.exists(): for entry in sorted(plugins_root.iterdir()): if not entry.is_dir(): @@ -298,19 +313,10 @@ def create_backup( tmp_path = zip_path.with_suffix(".zip.tmp") try: with zipfile.ZipFile(tmp_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: - # Config files. - if (project_root / _CONFIG_REL).exists(): - zf.write(project_root / _CONFIG_REL, _CONFIG_REL.as_posix()) - contents.append("config") - if (project_root / _SECRETS_REL).exists(): - zf.write(project_root / _SECRETS_REL, _SECRETS_REL.as_posix()) - contents.append("secrets") - if (project_root / _WIFI_REL).exists(): - zf.write(project_root / _WIFI_REL, _WIFI_REL.as_posix()) - contents.append("wifi") - if (project_root / _YTM_REL).exists(): - zf.write(project_root / _YTM_REL, _YTM_REL.as_posix()) - contents.append("ytm_auth") + for section, rel, _flag in _SINGLE_FILE_SECTIONS: + if (project_root / rel).exists(): + zf.write(project_root / rel, rel.as_posix()) + contents.append(section) # User-uploaded fonts. user_fonts = iter_user_fonts(project_root) @@ -338,7 +344,7 @@ def create_backup( contents.append("plugins") # Manifest goes last so that `contents` reflects what we actually wrote. - manifest = _build_manifest(contents, project_root) + manifest = _build_manifest(contents) zf.writestr(MANIFEST_NAME, json.dumps(manifest, indent=2)) os.replace(tmp_path, zip_path) @@ -352,15 +358,16 @@ def create_backup( def preview_backup_contents(project_root: Path) -> Dict[str, Any]: """Return a summary of what ``create_backup`` would include.""" project_root = Path(project_root).resolve() - return { - "has_config": (project_root / _CONFIG_REL).exists(), - "has_secrets": (project_root / _SECRETS_REL).exists(), - "has_wifi": (project_root / _WIFI_REL).exists(), - "has_ytm_auth": (project_root / _YTM_REL).exists(), + preview: Dict[str, Any] = { + f"has_{section}": (project_root / rel).exists() + for section, rel, _flag in _SINGLE_FILE_SECTIONS + } + preview.update({ "user_fonts": [p.name for p in iter_user_fonts(project_root)], "plugin_uploads": len(iter_plugin_uploads(project_root)), "plugins": list_installed_plugins(project_root), - } + }) + return preview # --------------------------------------------------------------------------- @@ -431,15 +438,10 @@ def validate_backup(zip_path: Path) -> Tuple[bool, str, Dict[str, Any]]: {}, ) - detected: List[str] = [] - if _CONFIG_REL.as_posix() in names: - detected.append("config") - if _SECRETS_REL.as_posix() in names: - detected.append("secrets") - if _WIFI_REL.as_posix() in names: - detected.append("wifi") - if _YTM_REL.as_posix() in names: - detected.append("ytm_auth") + detected: List[str] = [ + section for section, rel, _flag in _SINGLE_FILE_SECTIONS + if rel.as_posix() in names + ] if any(n.startswith(_FONTS_REL.as_posix() + "/") for n in names): detected.append("fonts") if any( @@ -584,55 +586,18 @@ def restore_backup( result.errors.append("Failed to extract backup") return result - # Main config. - if options.restore_config and (tmp_dir / _CONFIG_REL).exists(): + for section, rel, flag in _SINGLE_FILE_SECTIONS: + if not (tmp_dir / rel).exists(): + continue + if not getattr(options, flag): + result.skipped.append(section) + continue try: - _copy_file(tmp_dir / _CONFIG_REL, project_root / _CONFIG_REL) - result.restored.append("config") + _copy_file(tmp_dir / rel, project_root / rel) + result.restored.append(section) except OSError as e: - logger.error("[Backup] Failed to restore config.json: %s", e, exc_info=True) - result.errors.append("Failed to restore config.json") - elif (tmp_dir / _CONFIG_REL).exists(): - result.skipped.append("config") - - # Secrets. - if options.restore_secrets and (tmp_dir / _SECRETS_REL).exists(): - try: - _copy_file(tmp_dir / _SECRETS_REL, project_root / _SECRETS_REL) - result.restored.append("secrets") - except OSError as e: - logger.error( - "[Backup] Failed to restore config_secrets.json: %s", e, exc_info=True - ) - result.errors.append("Failed to restore config_secrets.json") - elif (tmp_dir / _SECRETS_REL).exists(): - result.skipped.append("secrets") - - # WiFi. - if options.restore_wifi and (tmp_dir / _WIFI_REL).exists(): - try: - _copy_file(tmp_dir / _WIFI_REL, project_root / _WIFI_REL) - result.restored.append("wifi") - except OSError as e: - logger.error( - "[Backup] Failed to restore wifi_config.json: %s", e, exc_info=True - ) - result.errors.append("Failed to restore wifi_config.json") - elif (tmp_dir / _WIFI_REL).exists(): - result.skipped.append("wifi") - - # YouTube Music session. Follows restore_wifi rather than getting its - # own flag: it is device-local auth in the same sense, and a separate - # toggle for one file would be noise in the restore dialog. - if options.restore_wifi and (tmp_dir / _YTM_REL).exists(): - try: - _copy_file(tmp_dir / _YTM_REL, project_root / _YTM_REL) - result.restored.append("ytm_auth") - except OSError as e: - logger.error("[Backup] Failed to restore ytm_auth.json: %s", e, exc_info=True) - result.errors.append("Failed to restore ytm_auth.json") - elif (tmp_dir / _YTM_REL).exists(): - result.skipped.append("ytm_auth") + logger.error("[Backup] Failed to restore %s: %s", rel.name, e, exc_info=True) + result.errors.append(f"Failed to restore {rel.name}") # User fonts — skip anything that collides with a bundled font. tmp_fonts = tmp_dir / _FONTS_REL diff --git a/src/base_odds_manager.py b/src/base_odds_manager.py index efb10bba..548dcfba 100644 --- a/src/base_odds_manager.py +++ b/src/base_odds_manager.py @@ -18,6 +18,8 @@ import requests import json from typing import Dict, Any, Optional, List +from src.common.api_helper import DEFAULT_HTTP_HEADERS + class BaseOddsManager: """ @@ -45,22 +47,15 @@ class BaseOddsManager: self.logger = logging.getLogger(__name__) self.base_url = "https://sports.core.api.espn.com/v2/sports" - # This path used a bare requests.get, so it identified itself as - # python-requests/x.y -- the one thing ESPN is known to reject. Around - # 2026-08-04 it began 403ing browser strings and bare custom tokens - # alike; what it accepts is a token with a URL that says who is - # calling. Every other ESPN caller in the tree already sends this - # (src/common/api_helper.py); the odds path was simply missed, and it is the one whose failures cost - # the caller its whole update budget. + # Core's shared headers: ESPN rejects requests' default User-Agent + # (see api_helper.USER_AGENT), and a rejected odds request costs the + # calling plugin its update budget. # # Deliberately no retry adapter, unlike api_helper: retries multiply # request_timeout, which is set to 5s precisely to stay inside that # budget. One try, then the cooldown below. self.session = requests.Session() - self.session.headers.update({ - 'User-Agent': 'LEDMatrix/1.0 (+https://github.com/ChuckBuilds/LEDMatrix)', - 'Accept': 'application/json', - }) + self.session.headers.update(DEFAULT_HTTP_HEADERS) # Configuration with defaults self.update_interval = 3600 # 1 hour default @@ -72,7 +67,6 @@ class BaseOddsManager: self.request_timeout = 5 # Set when a request fails; until then, skip the network entirely. self._skip_network_until = 0.0 - self.cache_ttl = 1800 # 30 minutes default # Load configuration if available if config_manager: @@ -89,12 +83,10 @@ class BaseOddsManager: self.update_interval = odds_config.get('update_interval', self.update_interval) self.request_timeout = odds_config.get('timeout', self.request_timeout) - self.cache_ttl = odds_config.get('cache_ttl', self.cache_ttl) - + self.logger.debug(f"BaseOddsManager configuration loaded: " f"update_interval={self.update_interval}s, " - f"timeout={self.request_timeout}s, " - f"cache_ttl={self.cache_ttl}s") + f"timeout={self.request_timeout}s") except Exception as e: self.logger.warning(f"Failed to load BaseOddsManager configuration: {e}") @@ -172,15 +164,12 @@ class BaseOddsManager: odds_data = self._extract_espn_data(raw_data) if odds_data: self.logger.info(f"Successfully extracted odds data: {odds_data}") - else: - self.logger.debug("No odds data available for this game") - - if odds_data: self.cache_manager.set(cache_key, odds_data, ttl=interval) self.logger.info(f"Saved odds data to cache for {cache_key} with TTL {interval}s") else: self.logger.debug(f"No odds data available for {cache_key}") - # Cache the fact that no odds are available to avoid repeated API calls + # Cache the absence too, so the game is not re-requested + # on every update until the interval passes. self.cache_manager.set(cache_key, {"no_odds": True}, ttl=interval) return odds_data diff --git a/src/cache_manager.py b/src/cache_manager.py index d064a37f..ee4f46a6 100644 --- a/src/cache_manager.py +++ b/src/cache_manager.py @@ -32,7 +32,6 @@ from typing import Any, Dict, List, Optional import logging import threading import tempfile -from src.exceptions import CacheError from src.cache.memory_cache import MemoryCache, default_max_size from src.cache.disk_cache import DiskCache from src.cache.cache_strategy import CacheStrategy @@ -272,12 +271,9 @@ class CacheManager: # Update memory cache first self._memory_cache_component.set(key, data) - # Save to disk cache - try: - self._disk_cache_component.set(key, data) - except CacheError: - # Disk cache errors are already logged and raised by DiskCache - raise + # DiskCache logs a failed write and raises CacheError, which the + # caller gets as is. + self._disk_cache_component.set(key, data) def load_cache(self, key: str) -> Optional[Dict[str, Any]]: """Load data from cache with memory caching.""" diff --git a/src/common/README.md b/src/common/README.md index 4c682574..68c22404 100644 --- a/src/common/README.md +++ b/src/common/README.md @@ -1,62 +1,242 @@ -# Common Utilities +# src/common -This directory contains reusable utilities and helpers for LEDMatrix plugins and core modules. +Helpers shared by core and plugins. This page lists every module, what it is +for, and whether plugins are expected to import it. -## Adaptive Layout & Images (`src/adaptive_layout.py`, `src/adaptive_images.py`) +Rules for the package: -The recommended way to lay out plugins that render legibly on **any** panel -size (64x32 through 256x128+) without hand-tuned coordinates. Re-exported -from `src.common` for convenience; canonical import paths are -`src.adaptive_layout` / `src.adaptive_images`. +- Every module must import without display hardware: nothing here may import + `src.display_manager` or `src.plugin_system` at module level + ([`test/test_common_is_hardware_free.py`](../../test/test_common_is_hardware_free.py)). + That keeps plugins that use it loadable by the web preview, + `scripts/check_plugin.py` and tests on a laptop. +- A plugin that imports a module added in a given core release must declare + that release as its minimum (`ledmatrix_min_version` in the manifest's + `versions` entry). The "Since" column gives the release; "—" means it + predates 3.1.0, "n/a" that plugins should not import it. +- `from src.common import ...` re-exports `APIHelper`, `ScrollHelper`, + `LogoHelper`, `TextHelper`, `scroll_config` (plus `ScrollSettings`, + `configure_scroll`, `resolve_scroll_settings`, `refresh_hz_from_config`) and + the adaptive layout names below ([`__init__.py`](__init__.py)). + +## Summary + +| Module | For | Plugins import it? | Since | +|---|---|---|---| +| [`api_helper`](#api_helper) | HTTP GET/POST with caching and rate limiting | Yes | — | +| [`bdf_font`](#bdf_font) | Load and draw BDF bitmap fonts | Yes, if drawing BDF text directly | Unreleased | +| [`espn_dates`](#espn_dates) | Fetch ESPN scoreboards across a date range | Yes (scoreboards) | 3.5.0 | +| [`font_layout`](#font_layout) | Reproducible TrueType loading, crisp sizes | Yes | 3.4.0 | +| [`logo_helper`](#logo_helper) | Load, resize and cache team logos | Yes | — | +| [`path_safety`](#path_safety) | Turn request-supplied names into safe paths | No, core-internal | n/a | +| [`permission_utils`](#permission_utils) | File modes and shared-group ownership | Rarely | — | +| [`scroll_config`](#scroll_config) | Plugin scroll config → configured `ScrollHelper` | Yes (scrollers) | 3.4.0 | +| [`scroll_helper`](#scroll_helper) | Pre-rendered horizontal scrolling | Yes | — | +| [`snapshot_policy`](#snapshot_policy) | When to write the web preview frame | No, core-internal | n/a | +| [`sports_card`](#sports_card) | Scoreboard card settings, colours, fonts, dates | Yes (scoreboards) | 3.3.0 | +| [`sports_game_renderer`](#sports_game_renderer) | Scoreboard scroll/Vegas card geometry | Yes (scoreboards) | 3.3.0 | +| [`sports_helpers`](#sports_helpers) | Small helpers every scoreboard `sports.py` copies | Yes (scoreboards) | 3.5.0 | +| [`sports_scroll`](#sports_scroll) | Scoreboard scroll-display orchestration | Yes (scoreboards) | 3.2.0 | +| [`sports_shared`](#sports_shared) | Sport-independent `sports.py` methods | Yes (scoreboards) | 3.3.0 | +| [`sync_manager`](#sync_manager) | Leader/follower sync between two displays | No, core-internal | n/a | +| [`text_helper`](#text_helper) | Outlined text, wrapping, measurement | Yes | — | + +The four `sports_*` mixin and card modules hold code the scoreboard plugins +used to carry as identical copies. Each module docstring lists what a host +class must provide. The plan behind them is in +[docs/SPORTS_UNIFICATION.md](../../docs/SPORTS_UNIFICATION.md). + +## Adaptive layout and images + +`src/adaptive_layout.py` and `src/adaptive_images.py` live outside this +package but are re-exported from `src.common`. They are the recommended way +to lay out a plugin that renders legibly on any panel size. Every +`BasePlugin` already has `self.layout`, `self.draw_fit()` and +`self.draw_image()`: ```python -# Every BasePlugin already has self.layout and the draw helpers: regs = scoreboard_regions(self.layout.bounds, ctx=self.layout) self.draw_image(away_logo, regs.away_slot, mode="fill_height", crop_to_ink=True, cache_key=f"logo:{abbr}") self.draw_fit(score_text, regs.score_area) # largest crisp font that fits -self.draw_fit(status, regs.status_band) ``` -Key pieces: `Region` (rect algebra: bands/columns/splits/offset), -font ladders (`LADDER_GRID`, `LADDER_ARCADE` — discrete crisp sizes, never -fractional scaling), `LayoutContext` (`fit_text`, `fit_image`, `by_tier`, -`px`), and composite carvers `scoreboard_regions()` / `media_row()`. -Full guide: [docs/ADAPTIVE_LAYOUT.md](../../docs/ADAPTIVE_LAYOUT.md). +Key pieces: `Region`, the font ladders `LADDER_GRID` / `LADDER_ARCADE`, +`LayoutContext` (`fit_text`, `fit_image`, `by_tier`, `px`), and +`scoreboard_regions()` / `media_row()`. Guide: +[docs/ADAPTIVE_LAYOUT.md](../../docs/ADAPTIVE_LAYOUT.md). -## API Helpers (`api_helper.py`) +## Modules -Utilities for making HTTP requests and handling API responses. +### api_helper -## Logo Helpers (`logo_helper.py`) +[`api_helper.py`](api_helper.py). `APIHelper(cache_manager=None, ...)`: +`get()` and `post()` with retries, optional caching through the cache +manager, and a minimum interval between requests (`set_rate_limit()`). Also has +`fetch_espn_scoreboard()`, `fetch_espn_standings()` and +`fetch_espn_rankings()`. -Utilities for loading and managing team logos. +### bdf_font -## Text Helpers (`text_helper.py`) +[`bdf_font.py`](bdf_font.py). The one BDF loader and rasterizer. +`load_bdf_face(path, size)` returns `(face, realised_px)`, falling back to +the file's native strike when it has none at `size`; +`draw_bdf_text(draw, text, x, y, face, color)` draws top-left anchored onto a +PIL `ImageDraw` the same way the panel does. `read_bdf_native_size(path)` +and `clear_face_cache()` round it out. Faces are cached per thread (FreeType +faces are not thread-safe). `DisplayManager`, `FontManager`, `element_style` +and the plugin test harness all use it. Most plugins get BDF text through +`display_manager.draw_text()` or `FontManager` and never import this. -Utilities for text processing and formatting. +### espn_dates -## BDF Fonts (`bdf_font.py`) +[`espn_dates.py`](espn_dates.py). ESPN's site API rejects `dates=` ranges +and truncates results when `limit` is above 500. `fetch_espn_scoreboard()` +splits a range into month and day requests ESPN accepts and merges the +results; `espn_date_chunks()`, `fetch_espn_date_chunks()`, +`clamp_espn_limit()` and `merge_scoreboard_payloads()` are the pieces. +Scoreboard plugins also bundle a copy for older cores. -The one way to load and draw BDF bitmap fonts. `load_bdf_face(path, size)` -returns `(face, realised_px)`, falling back to the file's native strike when -it has none at `size`; `draw_bdf_text(draw, text, x, y, face, color)` draws -top-left anchored onto a PIL `ImageDraw` exactly as the panel does. -`DisplayManager`, `FontManager`, `element_style` and the plugin test harness -all go through it. +### font_layout -## Scroll Helpers (`scroll_helper.py`) +[`font_layout.py`](font_layout.py). `load_truetype(path, size)` is +`ImageFont.truetype` with PIL's Basic layout engine pinned, so text lays out +the same whether or not the host Pillow has libraqm; use it for anything +drawn to the panel or compared against a golden image. `crisp_size()` gives +the size a bundled face renders on whole pixels at. `resolve_asset_path()` +resolves `assets/fonts/...` against the install root rather than the +working directory. -Utilities for scrolling text on the display. +### logo_helper -## Permission Utilities (`permission_utils.py`) +[`logo_helper.py`](logo_helper.py). `LogoHelper(display_width, +display_height, ...)`: `load_logo()`, `load_logo_with_download()`, +`get_logo_variations()`, `normalize_abbreviation()`, with an in-memory cache. -Helpers for ensuring directory permissions and ownership are correct -when running as a service (used by `CacheManager` to set up its -persistent cache directory). +### path_safety -## Best Practices +[`path_safety.py`](path_safety.py). Core-internal, used by web handlers that +open files named in a request. `safe_path_component(value)` returns the +value if it is one harmless path segment, else `None`; +`resolve_under(base, *parts)` returns the resolved path, or `None` if a part +is unsafe or the result would leave `base`; `safe_relative_parts()` splits a +relative path the same way. Both return the sanitised value rather than a +boolean, so a caller cannot check one string and open another. -1. **Use centralized logging**: Import from `src.logging_config` instead of creating loggers directly -2. **Reuse utilities**: Check existing utilities before creating new ones -3. **Document additions**: Add documentation when adding new utilities +### permission_utils + +[`permission_utils.py`](permission_utils.py). The modes and ownership that +let the root display service and the web user share files: +`ensure_directory_permissions()`, `ensure_file_permissions()`, the +`get_*_mode()` functions, `ensure_shared_group_ownership()`, +`sudo_remove_directory()` and `install_requirements_file()` (the sudo +`safe_pip_install.sh` path). `ConfigManager`, `CacheManager` and the store +already call these; a plugin needs them only when it creates its own files +outside the cache. See [docs/PERMISSIONS.md](../../docs/PERMISSIONS.md). + +### scroll_config + +[`scroll_config.py`](scroll_config.py). `configure(scroll_helper, +plugin_config=, global_config=, display_manager=, plugin_logger=)` reads a +plugin's scroll settings, snaps the speed to a whole number of pixels per +panel refresh, puts the helper in fixed-step mode and returns +`ScrollSettings`. Pass `settings.frame_hold` to +`display_manager.set_scrolling_state(True, frame_hold=...)` or the scroll +runs too fast. `resolve()` does the calculation without touching a helper. +See [docs/SCROLL_PERFORMANCE.md](../../docs/SCROLL_PERFORMANCE.md). + +### scroll_helper + +[`scroll_helper.py`](scroll_helper.py). `ScrollHelper(display_width, +display_height, logger=None)`: build a wide image once +(`create_scrolling_image()` or `set_scrolling_image()`), then per frame +`update_scroll_position()` and `get_visible_portion()`; +`is_scroll_complete()`, `calculate_dynamic_duration()` and +`get_dynamic_duration()` for timing. Configure it with `scroll_config` +rather than the `set_*` methods. Vegas mode reads a plugin's +`scroll_helper` image when the plugin has no `get_vegas_content()`. + +### snapshot_policy + +[`snapshot_policy.py`](snapshot_policy.py). Core-internal. `decide()` +tells `DisplayManager` whether to write `/tmp/led_matrix_preview.png`, only +touch its mtime, or skip, based on whether a browser is watching the preview. +The web health check reads the file's age. + +### sports_card + +[`sports_card.py`](sports_card.py). Free functions taking `config`, `fonts` +and `logger` explicitly: card options (`scroll_card_option()`, +`vs_text()`, `upcoming_center_mode()`), colours (`element_color()`, +`font_color()`, `score_color_for()`, `recent_score_color()`), favourite-team +rules (`favorite_teams_for()`, `side_is_favorite()`, `favorite_result()`), +dates (`format_game_date()`, `format_game_time()`, `card_tzinfo()`) and font +sizes (`schema_font_size()`, `resolve_font_size()`). A plugin keeps its own +method and delegates the body. + +### sports_game_renderer + +[`sports_game_renderer.py`](sports_game_renderer.py). +`SportsGameRendererMixin`: the scroll/Vegas card geometry (centre gap, logo +slot, layout offsets, upcoming-card date and time). No `__init__` and no +state; add it as a base class of the plugin's game renderer and override +what differs. + +### sports_helpers + +[`sports_helpers.py`](sports_helpers.py). Free functions `clamp_window()`, +`clamp_seconds()`, `logo_needs_refresh()`, `spread_weighted_order()`, and +`SportsHelpersMixin` with the scoreboards' `_mode_customization`, +`_setting_int`, `_reset_dwell_on_reentry`, `_next_switch_index`, +`_odds_color` and `_upcoming_date_and_time_text` under their existing names. +Nothing in core uses it. + +### sports_scroll + +[`sports_scroll.py`](sports_scroll.py). `SportsScrollDisplay` and +`SportsScrollDisplayManager`: the scroll-display orchestration the +scoreboards share (Vegas items, dynamic duration, frame loop), paced through +`scroll_config`. Subclasses supply `prepare_scroll_content()` and set +`SCROLL_LEAGUE_KEYS`; see the module docstring for an example. + +### sports_shared + +[`sports_shared.py`](sports_shared.py). `SportsCoreSharedMixin`, +`SportsLiveSharedMixin`, `SportsRecentSharedMixin`: the `sports.py` methods +that were identical in every scoreboard (game selection and rotation, +fonts, colours, dates, the switch-mode upcoming card). The docstring lists +the attributes the host class must have and the three methods deliberately +left out. + +### sync_manager + +[`sync_manager.py`](sync_manager.py). Core-internal. `DisplaySyncManager` +links two displays as leader and follower (`sync.role` in config) over UDP +port 5765, plus TCP on the next port for scroll images. The leader drives the +scroll and sends the follower its part of each frame; a follower falls back +to its own plugins when the leader goes quiet. Rows and columns must match. +Created by `DisplayController`; works with any plugin. + +### text_helper + +[`text_helper.py`](text_helper.py). `TextHelper(font_dir=None, ...)`: +`load_fonts()`, `draw_text_with_outline()`, `get_text_width()`, +`get_text_dimensions()`, `center_text()`, `wrap_text()`, +`draw_multiline_text()`, `create_text_image()`. + +## Logging + +Modules here create their logger with `logging.getLogger(__name__)`, which is +the same logger `src.logging_config.get_logger(__name__)` returns. The helper +classes (`APIHelper`, `LogoHelper`, `ScrollHelper`, `TextHelper`) and +`espn_dates` take an optional `logger`. In a plugin, pass `self.logger`: it is +created by `get_logger(..., plugin_id=...)` in `BasePlugin`, so messages carry +the plugin id. + +## Adding a module + +- Keep it importable without hardware (see the test above). +- Give it a module docstring that says what it is for and, if it is a mixin, + what the host class must provide. +- Add it to the table on this page and, if plugins may import it, to the + CHANGELOG with the release to floor on. diff --git a/src/common/api_helper.py b/src/common/api_helper.py index d025b378..2e5603f9 100644 --- a/src/common/api_helper.py +++ b/src/common/api_helper.py @@ -1,8 +1,9 @@ """ API Helper -Handles HTTP requests, caching, and ESPN API integration for LED matrix plugins. -Extracted from LEDMatrix core to provide reusable functionality for plugins. +HTTP requests, response caching and ESPN fetch helpers for plugins +(``from src.common import APIHelper``), plus the headers every core request +sends (:data:`USER_AGENT`, :data:`DEFAULT_HTTP_HEADERS`). """ import logging @@ -36,13 +37,20 @@ DEFAULT_HTTP_HEADERS: Mapping[str, str] = MappingProxyType({ class APIHelper: """ - Helper class for HTTP requests, caching, and ESPN API integration. - - Provides functionality for: - - HTTP requests with retry logic and timeouts - - Response caching with TTL support - - ESPN API integration for sports data - - Request rate limiting and throttling + HTTP requests with retries, response caching and ESPN helpers. + + - Requests go through one ``requests.Session`` that retries GET, HEAD + and OPTIONS on 429 and 5xx with exponential backoff, and sends + :data:`DEFAULT_HTTP_HEADERS`. + - Consecutive requests from one helper are spaced at least + ``set_rate_limit()`` seconds apart (1 second by default). A cache hit + does not count. + - With a ``cache_manager``, :meth:`get` caches the parsed JSON under + ``cache_key`` for ``cache_ttl`` seconds. The lifetime is stored with + the entry, so CacheManager honours it on every later read, whatever + max_age that read asks for. + - Failed requests are logged and return None; nothing here raises for a + network or HTTP error. """ def __init__(self, cache_manager=None, default_timeout: int = 30, @@ -73,13 +81,7 @@ class APIHelper: self.session.mount("https://", adapter) self.session.mount("http://", adapter) - # Default headers - self.session.headers.update({ - 'User-Agent': USER_AGENT, - 'Accept': 'application/json', - 'Accept-Language': 'en-US,en;q=0.9', - 'Connection': 'keep-alive' - }) + self.session.headers.update({**DEFAULT_HTTP_HEADERS, 'Connection': 'keep-alive'}) # Rate limiting self._last_request_time = 0 @@ -102,9 +104,8 @@ class APIHelper: Returns: Response data as dictionary or None if request fails """ - # Check cache first if cache_key and self.cache_manager: - cached = self._get_from_cache(cache_key) + cached = self._get_from_cache(cache_key, cache_ttl) if cached is not None: self.logger.debug(f"Using cached response for {cache_key}") return cached @@ -268,33 +269,31 @@ class APIHelper: Args: key: Cache key data: Data to cache - ttl: Time-to-live in seconds (ignored - CacheManager doesn't support TTL) + ttl: Seconds the entry stays valid. Stored with the entry, so + it applies to every later read of ``key``. """ - if self.cache_manager: - self.cache_manager.set(key, data) + self._set_cache(key, data, ttl) def get_cache(self, key: str) -> Optional[Any]: """ Get cached data. - + Args: key: Cache key - + Returns: - Cached data or None if not found + Cached data, or None if there is none or it has expired. An + entry written with a ttl (set_cache, get) expires after that ttl; + one written without expires after CacheManager's default max_age. """ - if self.cache_manager: - return self.cache_manager.get(key) - return None + return self._get_from_cache(key) def clear_cache(self, pattern: Optional[str] = None) -> None: """ Clear cache data. - Uses CacheManager's real surface (clear_cache / delete / - list_cache_files); safely no-ops on managers without it. The old - implementation guarded on a nonexistent ``clear`` method, so it - silently never cleared anything. + Uses CacheManager's clear_cache(), or list_cache_files() and delete() + for a pattern. A cache manager without those methods is left alone. Args: pattern: Optional substring to match cache keys; only matching @@ -315,21 +314,22 @@ class APIHelper: "cannot clear by pattern") elif hasattr(self.cache_manager, 'clear_cache'): self.cache_manager.clear_cache() - elif hasattr(self.cache_manager, 'clear'): - self.cache_manager.clear() else: self.logger.debug("Cache manager exposes no clear method; no-op") - def _get_from_cache(self, key: str) -> Optional[Any]: - """Get data from cache.""" - if self.cache_manager: + def _get_from_cache(self, key: str, max_age: Optional[int] = None) -> Optional[Any]: + """Cached data for ``key``, or None. ``max_age`` only matters for an + entry stored without a ttl; one stored with a ttl uses that.""" + if not self.cache_manager: + return None + if max_age is None: return self.cache_manager.get(key) - return None - - def _set_cache(self, key: str, data: Any, ttl: int) -> None: - """Set data in cache.""" + return self.cache_manager.get(key, max_age=max_age) + + def _set_cache(self, key: str, data: Any, ttl: Optional[int]) -> None: + """Store ``data`` under ``key`` for ``ttl`` seconds.""" if self.cache_manager: - self.cache_manager.set(key, data) + self.cache_manager.set(key, data, ttl=ttl) def _enforce_rate_limit(self) -> None: """Enforce rate limiting between requests.""" diff --git a/src/common/font_layout.py b/src/common/font_layout.py index 097e604a..9441b0ef 100644 --- a/src/common/font_layout.py +++ b/src/common/font_layout.py @@ -67,10 +67,11 @@ _INSTALL_ROOT = Path(__file__).resolve().parents[2] def resolve_asset_path(relative_path: str) -> str: """Resolve a repo-relative asset path independently of the process cwd. - Prefers the path as given — so an absolute path is returned untouched and - behaviour is unchanged wherever the cwd already happened to be the install - root — then the install root derived above, then the original string so a - caller that wants to raise and fall back still can. + In order: an absolute path that exists is returned untouched; otherwise + ``relative_path`` under the install root derived above, if that exists; + otherwise ``relative_path`` unchanged, so a caller that wants to raise + and fall back still can. The cwd is never consulted, so a relative path + means the same file whichever directory the process started in. Without the fallback, any process started outside the install root (the plugin safety harness, a manual ``python run.py`` from ``$HOME``, a unit diff --git a/src/common/logo_helper.py b/src/common/logo_helper.py index a574b89c..9ead2536 100644 --- a/src/common/logo_helper.py +++ b/src/common/logo_helper.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Dict, List, Optional, Union import requests -from PIL import Image +from PIL import Image, ImageDraw from src.common.api_helper import USER_AGENT from src.common.permission_utils import ( ensure_directory_permissions, @@ -32,35 +32,14 @@ from src.common.permission_utils import ( # trade for not re-warning about a file nobody is going to add. MISSING_LOGO_RECHECK_SECONDS = 3600.0 -#: Bounds on a user-supplied logo scale. Wide enough to be useful, closed -#: enough that a typo cannot ask for a 4000px image on a 64px panel. -MIN_LOGO_SCALE = 0.05 -MAX_LOGO_SCALE = 8.0 - - -def _usable_scale(scale) -> float: - """A scale that can be applied, or 1.0. - - Anything unusable -- None, a string, zero, a negative, NaN, infinity -- - means "as shipped", because the alternative is a blank panel from a - mistyped number. - """ - try: - value = float(scale) - except (TypeError, ValueError): - return 1.0 - if value != value or value in (float('inf'), float('-inf')): - return 1.0 - if value < MIN_LOGO_SCALE or value > MAX_LOGO_SCALE: - return 1.0 - return value - - - # Well above any real team logo; bounds what a remote URL can write to disk. # The cap for every logo download: src.logo_downloader.fetch_logo uses it too. MAX_LOGO_BYTES = 10 * 1024 * 1024 +#: A logo's default bounding box, as a multiple of the panel's width and +#: height, when the caller gives no max_width / max_height. +DEFAULT_LOGO_BOX_FACTOR = 1.5 + class LogoHelper: """ @@ -119,12 +98,16 @@ class LogoHelper: Args: team_abbr: Team abbreviation for caching logo_path: Path to the logo file - max_width: Maximum width (defaults to display_width * 1.5) - max_height: Maximum height (defaults to display_height * 1.5) + max_width: Maximum width (default display_width * + DEFAULT_LOGO_BOX_FACTOR) + max_height: Maximum height (default display_height * + DEFAULT_LOGO_BOX_FACTOR) scale: User's size multiplier for this image, from - ``customization.layout..scale``. 1.0 is untouched and - takes exactly the path it always did. Callers hold the config, - so they resolve the element name; this only applies the number. + ``customization.layout..scale``; 1.0 leaves the box + as is. Callers hold the config, so they resolve the element + name; this only applies the number, clamped to + src.element_style's MIN_ELEMENT_SCALE..MAX_ELEMENT_SCALE. + A value that is not a finite positive number means 1.0. Returns: PIL Image object or None if loading fails @@ -138,10 +121,13 @@ class LogoHelper: # key is size-qualified — a panel-size change must not return a # logo resized for the old dimensions. if max_width is None: - max_width = int(self.display_width * 1.5) + max_width = int(self.display_width * DEFAULT_LOGO_BOX_FACTOR) if max_height is None: - max_height = int(self.display_height * 1.5) - scale = _usable_scale(scale) + max_height = int(self.display_height * DEFAULT_LOGO_BOX_FACTOR) + # Imported here: src.element_style imports src.common (for bdf_font), + # whose __init__ imports this module. + from src.element_style import coerce_scale + scale = coerce_scale(scale, 1.0) if scale != 1.0: max_width = max(1, int(round(max_width * scale))) max_height = max(1, int(round(max_height * scale))) @@ -374,9 +360,9 @@ class LogoHelper: nobody asked to grow would change every existing render. """ if max_width is None: - max_width = int(self.display_width * 1.5) + max_width = int(self.display_width * DEFAULT_LOGO_BOX_FACTOR) if max_height is None: - max_height = int(self.display_height * 1.5) + max_height = int(self.display_height * DEFAULT_LOGO_BOX_FACTOR) # Only resize if necessary if logo.width <= max_width and logo.height <= max_height: @@ -429,31 +415,26 @@ class LogoHelper: max_width: Optional[int] = None, max_height: Optional[int] = None) -> Optional[Image.Image]: """ - Create a placeholder logo with team abbreviation. - + A stand-in for a logo that could not be loaded or downloaded: a + translucent grey box with a light outline, filling the logo box. + No text is drawn; ``team_abbr`` is only used in log messages. + Args: - team_abbr: Team abbreviation to display - max_width: Maximum width - max_height: Maximum height - + team_abbr: Team the placeholder stands in for + max_width: Width (default display_width * DEFAULT_LOGO_BOX_FACTOR) + max_height: Height (default display_height * DEFAULT_LOGO_BOX_FACTOR) + Returns: - PIL Image with placeholder logo + The RGBA placeholder, or None if it could not be created """ try: if max_width is None: - max_width = int(self.display_width * 1.5) + max_width = int(self.display_width * DEFAULT_LOGO_BOX_FACTOR) if max_height is None: - max_height = int(self.display_height * 1.5) + max_height = int(self.display_height * DEFAULT_LOGO_BOX_FACTOR) - # Create placeholder image placeholder = Image.new('RGBA', (max_width, max_height), (0, 0, 0, 0)) - - # This would require a font, so we'll create a simple colored rectangle - # In a real implementation, you'd want to add text rendering here - from PIL import ImageDraw draw = ImageDraw.Draw(placeholder) - - # Draw a simple rectangle with team abbreviation draw.rectangle([0, 0, max_width-1, max_height-1], fill=(100, 100, 100, 200), outline=(200, 200, 200, 255)) diff --git a/src/common/permission_utils.py b/src/common/permission_utils.py index f59b31cc..5e959577 100644 --- a/src/common/permission_utils.py +++ b/src/common/permission_utils.py @@ -241,7 +241,8 @@ def get_assets_dir_mode() -> int: Return permission mode for asset directories. Returns: - Permission mode: 0o2775 (rwxrwxr-x + sticky bit) for group-writable directories + Permission mode: 0o2775 (rwxrwsr-x): group-writable, and setgid so + entries created in it take the directory's group """ return 0o2775 # rwxrwsr-x (setgid + group writable) @@ -251,7 +252,8 @@ def get_config_dir_mode() -> int: Return permission mode for config directory. Returns: - Permission mode: 0o2775 (rwxrwxr-x + sticky bit) for group-writable directories + Permission mode: 0o2775 (rwxrwsr-x): group-writable, and setgid so + entries created in it take the directory's group """ return 0o2775 # rwxrwsr-x (setgid + group writable) @@ -271,7 +273,8 @@ def get_plugin_dir_mode() -> int: Return permission mode for plugin directories. Returns: - Permission mode: 0o2775 (rwxrwxr-x + sticky bit) for group-writable directories + Permission mode: 0o2775 (rwxrwsr-x): group-writable, and setgid so + entries created in it take the directory's group """ return 0o2775 # rwxrwsr-x (setgid + group writable) @@ -281,7 +284,8 @@ def get_cache_dir_mode() -> int: Return permission mode for cache directories. Returns: - Permission mode: 0o2775 (rwxrwxr-x + sticky bit) for group-writable cache directories + Permission mode: 0o2775 (rwxrwsr-x): group-writable, and setgid so + entries created in it take the directory's group """ return 0o2775 # rwxrwsr-x (setgid + group writable) diff --git a/src/common/snapshot_policy.py b/src/common/snapshot_policy.py index 20cc8b0d..b9399dd3 100644 --- a/src/common/snapshot_policy.py +++ b/src/common/snapshot_policy.py @@ -5,7 +5,7 @@ serves two consumers with different needs: - The web UI's live preview (SSE reader in web_interface/app.py) wants fresh frames — but only while a browser is actually watching. -- The health check (web_interface/blueprints/api_v3.py, hardware status) +- The health check (web_interface/blueprints/api_v3/misc.py, hardware status) uses the file's AGE as a liveness proxy: age >= 60s reads as degraded. PNG-encoding every frame at 5 fps forever — identical frames, no viewers — @@ -27,7 +27,7 @@ Policy: TOUCH_INTERVAL so the health check (60s threshold) never degrades. If any constant here changes, re-check the health threshold in -api_v3.py (get_hardware_status) — TOUCH_INTERVAL must stay well under it. +api_v3/misc.py (get_hardware_status) — TOUCH_INTERVAL must stay well under it. """ from enum import Enum @@ -37,7 +37,7 @@ VIEWER_INTERVAL = 0.2 # Snapshot cadence with no viewers — cheap freshness for page-open (seconds). IDLE_INTERVAL = 30.0 # Max age of the last write/touch before bumping mtime for the health -# check. MUST stay well under api_v3's 60s degraded threshold. +# check. MUST stay well under get_hardware_status's 60s degraded threshold. TOUCH_INTERVAL = 20.0 # A viewer marker older than this no longer counts as a live viewer. VIEWER_MARKER_FRESH_SEC = 5.0 diff --git a/src/common/sports_card.py b/src/common/sports_card.py index 2c5c88fb..3389f090 100644 --- a/src/common/sports_card.py +++ b/src/common/sports_card.py @@ -101,11 +101,10 @@ def element_color(config: Optional[Dict[str, Any]], element: str, mode: Optional[str] = None): """Per-element text colour from customization..text_color. - Delegated rather than reimplemented: there were two copies of this - read and three of the offset read, and the shared one also resolves - the element under the names plugins actually use (the layout block - says `score` where the style block says `score_text`) and honours a - per-mode override. Hex strings are still accepted. + Delegates to src.element_style.element_color, which also resolves the + element under the names plugins actually use (the layout block says + `score` where the style block says `score_text`) and honours a per-mode + override. Hex strings are accepted. """ from src.element_style import element_color as _shared return _shared(config, element, default, mode) @@ -125,12 +124,11 @@ def resolve_font_color(config: Optional[Dict[str, Any]], One object can legitimately belong to several elements -- a size resolver can land two of them on the same face, and a BDF face cannot be un-shared at all because ``freetype.Face`` objects cannot be rebuilt from a path. - Those draws used to go out white, which is how an element rendered in any - of the 32 shipped bitmap fonts could silently lose a colour the user had - set. So ambiguity is now narrowed before it is given up on: among the + Ambiguity is therefore narrowed before it is given up on: among the elements sharing a face, a single configured colour is the only thing the user can have meant, and several that agree mean the same thing. Only a - genuine disagreement falls back to *default*. + genuine disagreement falls back to *default* -- otherwise an element + drawn in any of the shipped bitmap fonts could lose a colour the user set. The element vocabulary is a parameter because the two callers disagree about it -- the mixin's map says ``team_text`` where this module's says @@ -483,7 +481,7 @@ def unshare_element_fonts(logger, fonts, element_for_font=None): with identical metrics, so nothing about the rendering changes; only the ability to tell two elements apart does. Faces that cannot be rebuilt (a BDF loaded through freetype.Face, anything without a usable - path) are left shared, and their draws stay white as before. + path) are left shared; resolve_font_color then picks their colour. *element_for_font* names the font keys to consider, in order (the first holder of a face keeps it); it defaults to this module's @@ -491,10 +489,8 @@ def unshare_element_fonts(logger, fonts, element_for_font=None): which names different keys -- see ``resolve_font_color`` for why the two vocabularies are kept apart. """ - try: - from src.common.font_layout import load_truetype as _load - except ImportError: # pragma: no cover - return fonts + # Looked up at call time so tests can spy on the pinned loader. + from src.common.font_layout import load_truetype if element_for_font is None: element_for_font = ELEMENT_FOR_FONT seen = {} @@ -509,7 +505,7 @@ def unshare_element_fonts(logger, fonts, element_for_font=None): if not path or not size: continue try: - fonts[key] = _load(path, size) + fonts[key] = load_truetype(path, size) except (OSError, ValueError, TypeError): logger.debug( "Could not un-share the %s face; it keeps the default colour", key) diff --git a/src/common/sports_game_renderer.py b/src/common/sports_game_renderer.py index c890e45c..60367542 100644 --- a/src/common/sports_game_renderer.py +++ b/src/common/sports_game_renderer.py @@ -56,21 +56,13 @@ class SportsGameRendererMixin: # ---- geometry ------------------------------------------------------ - # Non-finite settings are rejected before any int()/round(): "inf" reaches - # these from config as a float or a string, passes an `isinstance` plus - # `>= 0` check unharmed, and then raises OverflowError out of int() -- - # which the old `except (TypeError, ValueError)` did not catch, so it - # aborted the whole card render. Present in all eight plugins before this - # moved to the core; fixing it here fixes it in all eight. - def _score_reserve_width(self) -> int: - """Centre strip the score actually needs, measured rather than assumed. + """Centre strip the score needs: the width of _SCORE_PROBE in the + score font plus a gutter each side, or 0 if it cannot be measured. - The gap was derived from the card width alone (width x - CENTER_GAP_RATIO, clamped to CENTER_GAP_MAX_PX) while the score's size - comes from config and the element-style resolver. Nothing compared the - two, so any score wider than the clamp was drawn over the logos. - Measuring it keeps the strip wide enough for whatever font is in play. + Measured rather than derived from the card width, because the score's + size comes from config and the element-style resolver: a strip sized + from the width alone lets a large score run over the logos. """ try: probe = ImageDraw.Draw(Image.new("RGB", (4, 4))) @@ -87,6 +79,10 @@ class SportsGameRendererMixin: the card width between the configurable min and max. 0 restores edge-to-edge logos. """ + # Non-finite settings are rejected before any int()/round(): "inf" + # arrives from config as a float or a string, passes the isinstance + # and >= 0 checks, and int() then raises OverflowError, which would + # abort the whole card render. configured = self._scroll_card_option("center_gap") if (isinstance(configured, (int, float)) and math.isfinite(configured) and configured >= 0): @@ -110,10 +106,9 @@ class SportsGameRendererMixin: def _logo_slot_width(self) -> int: """Per-side logo slot, leaving the center gap clear. - No longer capped at display_height: the card is sized as two - full-height logos plus the measured gap, so what is left after the gap - is exactly the logo's share. The cap was what froze the logos at 46px - on the old flat 128px card. + Not capped at display_height: the card is sized as two full-height + logos plus the measured gap, so what is left after the gap is exactly + the logo's share. At least 8 px. """ available = (self.display_width - self._center_gap_width()) // 2 return max(8, available) @@ -130,9 +125,8 @@ class SportsGameRendererMixin: """X/Y nudge for one element, from customization.layout. Same block the full-screen scorebug reads (sports.py - _get_layout_offset), so a nudge configured in the web UI now moves - the element on the scroll/Vegas card too -- previously the schema - advertised these offsets but this renderer ignored them. + _get_layout_offset), so a nudge configured in the web UI moves the + element on the scroll/Vegas card as well as on the scorebug. """ from src.element_style import layout_offset return layout_offset(self.config, element, axis, default, diff --git a/src/common/sports_scroll.py b/src/common/sports_scroll.py index d32fa76a..85e8bd6a 100644 --- a/src/common/sports_scroll.py +++ b/src/common/sports_scroll.py @@ -487,9 +487,9 @@ class SportsScrollDisplayManager: ) except Exception: # prepare_scroll_content is subclass-implemented and builds cards - # straight from feed data, which is exactly where this PR's other - # crashes came from. One sport's bad payload must not take down the - # shared orchestration for the others. + # straight from feed data, so it can raise on a malformed payload. + # One sport's bad payload must not take down the shared + # orchestration for the others. self.logger.exception( "Error preparing scroll content for game_type=%s", game_type ) diff --git a/src/common/sports_shared.py b/src/common/sports_shared.py index e9112ec6..af025a73 100644 --- a/src/common/sports_shared.py +++ b/src/common/sports_shared.py @@ -97,11 +97,11 @@ from datetime import datetime, timedelta, timezone from typing import Any, ClassVar, Dict, List, Optional, Tuple import pytz -from src.common.espn_dates import fetch_espn_scoreboard +from src.common.espn_dates import ESPN_MAX_LIMIT, fetch_espn_scoreboard import requests -from PIL import Image, ImageDraw, ImageFont +from PIL import Image, ImageDraw from src.common import sports_card as _card -from src.common.font_layout import load_truetype +from src.common.font_layout import load_truetype, resolve_asset_path logger = logging.getLogger(__name__) @@ -132,36 +132,16 @@ def _resolve_font_path(path: str) -> str: load raises, the caller falls back, and the scoreboard renders in PIL's default face instead of the pixel font it was laid out for. - Resolution order matches the core's own resolver: the path as given - first, so behaviour is unchanged wherever it already worked and a - configured absolute path is returned untouched, then the core install - root, then the original string so callers still raise and fall back - exactly as they do today. + Resolution order: the path as given, relative to the cwd, when it + exists -- the order the scoreboards' own sports.py copies used, so a + process running from another checkout keeps that checkout's fonts -- + then :func:`src.common.font_layout.resolve_asset_path` (the install + root), which returns the original string when neither exists so callers + still raise and fall back. """ if os.path.exists(path): return path - try: - import src.font_manager as _core_fonts - - # The core grew this resolver in ChuckBuilds/LEDMatrix#425. Use it - # when it is there so both repos stay on one definition of "install - # root"; older cores fall through to the equivalent derivation below. - manager = getattr(_core_fonts, "FontManager", None) - resolver = getattr(manager, "_resolve_asset_path", None) - if resolver is not None: - resolved = resolver(path) - if resolved and os.path.exists(resolved): - return resolved - root = os.path.dirname(os.path.dirname(os.path.abspath(_core_fonts.__file__))) - candidate = os.path.join(root, path) - if os.path.exists(candidate): - return candidate - except (ImportError, AttributeError, OSError): - # No core on the path (standalone tooling), a core laid out - # differently, or an unreadable install. Returning the original keeps - # the caller's existing fallback intact. - return path - return path + return resolve_asset_path(path) class SportsCoreSharedMixin: @@ -201,9 +181,6 @@ class SportsCoreSharedMixin: #: How long to stay quiet between ranking-coverage warnings. _RANKING_COVERAGE_SECONDS: ClassVar[int] = 60 * 60 - def _get_season_schedule_dates(self) -> tuple[str, str]: - return "", "" - def _draw_scorebug_layout(self, game: Dict, force_clear: bool = False) -> None: """Placeholder draw method - subclasses should override.""" # This base method will be simple, subclasses provide specifics @@ -889,7 +866,10 @@ class SportsCoreSharedMixin: draw.text((x, y), text, font=font, fill=fill) def _should_log(self, warning_type: str, cooldown: int = 60) -> bool: - """Check if we should log a warning based on cooldown period.""" + """True at most once per ``cooldown`` seconds, for rate-limiting a + warning. The cooldown is shared by every warning on this manager: + ``warning_type`` is part of the signature scoreboards inherit, but + does not give each type its own cooldown.""" current_time = time.time() if current_time - self._last_warning_time > cooldown: self._last_warning_time = current_time @@ -904,8 +884,6 @@ class SportsCoreSharedMixin: try: # Fetch current week and next few days for immediate display now = datetime.now(pytz.utc) - immediate_events = [] - start_date = now - timedelta(days=self.schedule_lookback_days) end_date = now + timedelta(days=self.schedule_lookahead_days) date_str = f"{start_date.strftime('%Y%m%d')}-{end_date.strftime('%Y%m%d')}" @@ -913,7 +891,7 @@ class SportsCoreSharedMixin: data = fetch_espn_scoreboard( self.session, url, - params={"dates": date_str, "limit": 1000}, + params={"dates": date_str, "limit": ESPN_MAX_LIMIT}, headers=self.headers, timeout=10, logger=self.logger, diff --git a/src/common/sync_manager.py b/src/common/sync_manager.py index 3df6ca5c..bec3d7f0 100644 --- a/src/common/sync_manager.py +++ b/src/common/sync_manager.py @@ -32,7 +32,7 @@ from typing import Callable, Optional import numpy as np from PIL import Image -from src.display_geometry import DEFAULT_CHAIN_LENGTH +from src.display_geometry import DEFAULT_CHAIN_LENGTH, DEFAULT_COLS, DEFAULT_ROWS # Raw-frame wire format: 8-byte magic + 4-byte header + raw RGB pixels # Much faster than PNG: no encode/decode, negligible CPU, same UDP packet size @@ -75,9 +75,12 @@ class FollowerState(Enum): class DisplaySyncManager: """ Core sync manager. Instantiated by DisplayController based on config['sync']. - Leader sends compressed PNG frames to the follower after each render cycle. - Follower renders received frames; returns to own plugin stack when leader - goes offline. + + The leader sends each rendered frame to the follower over UDP as raw RGB + bytes (send_frame), and for Vegas scrolling sends the whole scroll image + once per cycle as a PNG over TCP on port + 1 (send_scroll_image), then + only the scroll position. The follower draws what it receives and goes + back to its own plugins when the leader stops sending. """ def __init__( @@ -192,8 +195,8 @@ class DisplaySyncManager: def _handle_hello(self, msg: dict, sender_ip: str) -> None: hw = self._hw_config - local_rows = hw.get("rows", 32) - local_cols = hw.get("cols", 64) + local_rows = hw.get("rows", DEFAULT_ROWS) + local_cols = hw.get("cols", DEFAULT_COLS) peer_rows = int(msg.get("rows", 0)) peer_cols = int(msg.get("cols", 0)) peer_chain = int(msg.get("chain", DEFAULT_CHAIN_LENGTH)) @@ -469,16 +472,23 @@ class DisplaySyncManager: """Record a decoded leader frame and enter follower mode if needed.""" with self._frame_lock: self._latest_frame = img + self._enter_follower_mode(sender_ip) + + def _enter_follower_mode(self, sender_ip: str) -> bool: + """Note that the leader at ``sender_ip`` just sent something, and + switch from standalone to follower mode if not already following. + Returns True if this call made the switch.""" self._last_leader_frame_time = time.time() self._leader_ip = sender_ip - - if self._follower_state == FollowerState.STANDALONE: - self._follower_state = FollowerState.FOLLOWER - self.logger.info( - "Sync: leader active at %s — switching to follower mode", - sender_ip, - ) - self.write_status_file() + if self._follower_state != FollowerState.STANDALONE: + return False + self._follower_state = FollowerState.FOLLOWER + self.logger.info( + "Sync: leader active at %s — switching to follower mode", + sender_ip, + ) + self.write_status_file() + return True def _follower_recv_loop(self) -> None: while self._running: @@ -559,15 +569,7 @@ class DisplaySyncManager: # back from. Treat it as malformed. raise ValueError(f"non-finite scroll x: {msg['x']!r}") self._latest_scroll_x = scroll_x - self._last_leader_frame_time = time.time() - self._leader_ip = sender_ip - if self._follower_state == FollowerState.STANDALONE: - self._follower_state = FollowerState.FOLLOWER - self.logger.info( - "Sync: leader active at %s — switching to follower mode", - sender_ip, - ) - self.write_status_file() + if self._enter_follower_mode(sender_ip): fire_new_cycle = True # build initial scroll image elif t == "nc": # Leader started a new scroll cycle — rebuild local image @@ -589,8 +591,8 @@ class DisplaySyncManager: hw = self._hw_config hello = json.dumps({ "t": "hello", - "rows": hw.get("rows", 32), - "cols": hw.get("cols", 64), + "rows": hw.get("rows", DEFAULT_ROWS), + "cols": hw.get("cols", DEFAULT_COLS), "chain": hw.get("chain_length", DEFAULT_CHAIN_LENGTH), }).encode("utf-8") heartbeat = json.dumps({"t": "hb"}).encode("utf-8") @@ -660,8 +662,8 @@ class DisplaySyncManager: base = { "role": self.role.value, "port": self.port, - "local_rows": hw.get("rows", 32), - "local_cols": hw.get("cols", 64), + "local_rows": hw.get("rows", DEFAULT_ROWS), + "local_cols": hw.get("cols", DEFAULT_COLS), "local_chain": hw.get("chain_length", DEFAULT_CHAIN_LENGTH), } diff --git a/src/common/text_helper.py b/src/common/text_helper.py index 703476eb..be27772c 100644 --- a/src/common/text_helper.py +++ b/src/common/text_helper.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple, Union from PIL import Image, ImageDraw, ImageFont -from src.common.font_layout import load_truetype +from src.common.font_layout import load_truetype, resolve_asset_path # Shared throwaway draw surface for measuring text without a target canvas. _measure_draw = ImageDraw.Draw(Image.new("RGB", (1, 1))) @@ -18,13 +18,14 @@ _measure_draw = ImageDraw.Draw(Image.new("RGB", (1, 1))) class TextHelper: """ - Helper class for text rendering with outlines and font management. - - Provides functionality for: - - Loading and managing fonts - - Drawing text with outlines for better readability - - Calculating text dimensions and positioning - - Managing font resources + Font loading, outlined text and text measurement for plugins. + + - :meth:`load_fonts` loads TrueType fonts from ``font_dir`` (the install's + assets/fonts by default) with the layout engine pinned + (font_layout.load_truetype). Each (file, size) is loaded once per helper + and reused; a missing or unloadable file becomes PIL's default font. + - :meth:`draw_text_with_outline` and friends draw onto a caller's + ``ImageDraw``; the measuring methods need no canvas. """ def __init__(self, font_dir: Optional[Union[str, Path]] = None, @@ -33,22 +34,26 @@ class TextHelper: Initialize the TextHelper. Args: - font_dir: Directory containing font files (defaults to assets/fonts) + font_dir: Directory containing font files. Defaults to the + install's assets/fonts, whatever the process cwd is. logger: Optional logger instance """ self.logger = logger or logging.getLogger(__name__) - self.font_dir = Path(font_dir) if font_dir else Path("assets/fonts") + self.font_dir = Path(font_dir) if font_dir else Path(resolve_asset_path("assets/fonts")) + # ":" -> loaded font; see load_fonts. self._font_cache: Dict[str, ImageFont.ImageFont] = {} def load_fonts(self, font_config: Optional[Dict[str, Dict]] = None) -> Dict[str, ImageFont.ImageFont]: """ Load fonts for different text elements. - + Args: - font_config: Custom font configuration dictionary - + font_config: ``{name: {"file": , "size": }}``; + defaults to the scoreboard set in _get_default_font_config. + Returns: - Dictionary mapping font names to PIL ImageFont objects + Dictionary mapping font names to PIL ImageFont objects. A font + already loaded by this helper at the same size is reused. """ if font_config is None: font_config = self._get_default_font_config() @@ -61,9 +66,13 @@ class TextHelper: size = config['size'] if font_path.exists(): - font = load_truetype(str(font_path), size) + cache_key = f"{font_path}:{size}" + font = self._font_cache.get(cache_key) + if font is None: + font = load_truetype(str(font_path), size) + self._font_cache[cache_key] = font + self.logger.debug(f"Loaded font: {font_name} ({font_path}, size {size})") fonts[font_name] = font - self.logger.debug(f"Loaded font: {font_name} ({font_path}, size {size})") else: # Fallback to default font font = ImageFont.load_default() @@ -115,12 +124,7 @@ class TextHelper: Returns: Width in pixels """ - try: - return int(_measure_draw.textlength(text, font=font)) - except AttributeError: - # Fallback for older PIL versions - bbox = _measure_draw.textbbox((0, 0), text, font=font) - return bbox[2] - bbox[0] + return int(_measure_draw.textlength(text, font=font)) def get_text_height(self, text: str, font: ImageFont.ImageFont) -> int: """ diff --git a/src/config_manager.py b/src/config_manager.py index d9800d4d..292f6026 100644 --- a/src/config_manager.py +++ b/src/config_manager.py @@ -16,8 +16,10 @@ additionally keeps rotating backups in ``config/backups/``. Plugin configuration -------------------- Plugin configs are stored inside ``config.json`` under the plugin's ID key -and survive plugin reinstalls. Use :meth:`ConfigManager.update_plugin_config` -to write plugin settings; never write directly to the plugin directory. +and survive plugin reinstalls. Write them by saving the whole config with +:meth:`ConfigManager.save_config_atomic` (or +:meth:`ConfigManager.save_raw_file_content`); never write settings into the +plugin directory, which a reinstall deletes. Hot-reload ---------- @@ -127,13 +129,10 @@ class ConfigManager: # Update in-memory config if save was successful if result.status == SaveResultStatus.SUCCESS: self.config = new_config_data - # In-memory config now matches what was just written; refresh - # the load signature so the fast path stays valid. NOTE: the - # in-memory copy includes merged secrets; the on-disk file has - # them stripped — the fast path returning self.config preserves - # exactly the pre-cache behavior (load-after-save also returned - # the secret-merged self.config only after re-reading secrets; - # here secrets file is unchanged, so contents are equivalent). + # In-memory config now matches what was just written, so the + # load_config fast path may return it. It still carries the + # merged secrets that were stripped on disk; that matches a full + # reload, because the secrets file was not changed by the save. self._loaded_sig = self._files_signature() self.logger.info(f"Configuration successfully saved atomically to {os.path.abspath(self.config_path)}") elif result.status == SaveResultStatus.ROLLED_BACK: @@ -253,11 +252,11 @@ class ConfigManager: return self.config except FileNotFoundError as e: - if str(e).find('config_secrets.json') == -1: # Only raise if main config is missing - error_msg = f"Configuration file not found at {os.path.abspath(self.config_path)}" - self.logger.error(error_msg, exc_info=True) - raise ConfigError(error_msg, config_path=self.config_path) from e - return self.config + # Only config.json can get here: a missing or unreadable secrets + # file is handled where it is read. + error_msg = f"Configuration file not found at {os.path.abspath(self.config_path)}" + self.logger.error(error_msg, exc_info=True) + raise ConfigError(error_msg, config_path=self.config_path) from e except json.JSONDecodeError as e: error_msg = f"Error parsing configuration file {os.path.abspath(self.config_path)}" self.logger.error(error_msg, exc_info=True) @@ -320,10 +319,9 @@ class ConfigManager: A missing secrets file is fine (nothing to strip). But a file that EXISTS and cannot be read or parsed means stripping is impossible — and the in-memory config being saved has secrets deep-merged into it, - so proceeding would write them into config.json in plaintext. That - was the historical behavior; it is now a hard refusal. The save - raises so the caller (and user) fixes the secrets file instead of - silently leaking its contents into the world-readable main config. + so proceeding would write them into config.json in plaintext. The + save raises instead, so the caller (and user) fixes the secrets file + rather than leaking its contents into the world-readable main config. """ if not os.path.exists(self.secrets_path): return {} @@ -465,9 +463,8 @@ class ConfigManager: # Merge template defaults into current config self._merge_template_defaults(self.config, template_config) - # Save migrated config using atomic save to preserve permissions - # Use atomic save to preserve file permissions - # Note: save_config_atomic handles secrets internally + # save_config_atomic strips the merged secrets back out and + # keeps the file's owner and mode. result = self.save_config_atomic( new_config_data=self.config, create_backup=False, # Already created backup above @@ -600,20 +597,16 @@ class ConfigManager: self.logger.info(f"{file_type.capitalize()} configuration successfully saved to {os.path.abspath(path_to_save)}") - # If we just saved the main config or secrets, the merged self.config might be stale. - # Reload it to reflect the new state. - # Note: We wrap this in try-except because reload failures (e.g., migration errors) - # should not cause the save operation to fail - the file was saved successfully. - if file_type == "main" or file_type == "secrets": - try: - self.load_config() - except Exception as reload_error: - # Log the reload error but don't fail the save operation - # The file was saved successfully, reload is just for in-memory consistency - self.logger.warning( - f"Configuration file saved successfully, but reload failed: {reload_error}. " - f"The file on disk is valid, but in-memory config may be stale." - ) + # The merged self.config is now stale; reload it. A reload failure + # (a migration error, say) is logged, not raised: the file itself + # was saved. + try: + self.load_config() + except Exception as reload_error: + self.logger.warning( + f"Configuration file saved successfully, but reload failed: {reload_error}. " + f"The file on disk is valid, but in-memory config may be stale." + ) except PermissionError as e: # Provide helpful error message with fix instructions @@ -670,7 +663,7 @@ class ConfigManager: try: # Load current configs main_config = self.get_raw_file_content('main') - secrets_config = self.get_raw_file_content('secrets') if os.path.exists(self.secrets_path) else {} + secrets_config = self.get_raw_file_content('secrets') # {} when there is no file # Remove plugin from main config if plugin_id in main_config: @@ -703,7 +696,7 @@ class ConfigManager: try: # Load current configs main_config = self.get_raw_file_content('main') - secrets_config = self.get_raw_file_content('secrets') if os.path.exists(self.secrets_path) else {} + secrets_config = self.get_raw_file_content('secrets') # {} when there is no file valid_set = set(valid_plugin_ids) diff --git a/src/dynamic_team_resolver.py b/src/dynamic_team_resolver.py index 7cbe09cd..ef666a20 100644 --- a/src/dynamic_team_resolver.py +++ b/src/dynamic_team_resolver.py @@ -21,6 +21,8 @@ import time import requests from typing import Dict, List +from src.common.api_helper import DEFAULT_HTTP_HEADERS + logger = logging.getLogger(__name__) class DynamicTeamResolver: @@ -141,7 +143,9 @@ class DynamicTeamResolver: self.logger.info("Fetching fresh NCAA Football rankings from ESPN API") rankings_url = "https://site.api.espn.com/apis/site/v2/sports/football/college-football/rankings" - response = requests.get(rankings_url, timeout=self.request_timeout) + # ESPN rejects requests' default User-Agent; see api_helper.USER_AGENT. + response = requests.get(rankings_url, headers=dict(DEFAULT_HTTP_HEADERS), + timeout=self.request_timeout) response.raise_for_status() data = response.json() diff --git a/src/element_style.py b/src/element_style.py index 50c45660..15e1af4b 100644 --- a/src/element_style.py +++ b/src/element_style.py @@ -90,6 +90,17 @@ def _cache_put(key: Tuple[str, int], value: Tuple[Any, int]) -> None: # Config keys a style element block carries, in schema/UI order. _STYLE_KEYS = ('font', 'font_size', 'text_color', 'visible', 'align') +# Title of every generated `layout` (x/y offset) group in the config form. +_LAYOUT_TITLE = 'Layout Offsets' + +#: Bounds on a user-set ``customization.layout..scale``. They are the +#: Scale field's minimum and maximum in the generated schema, and every reader +#: (coerce_scale, element_scale, LogoHelper.load_logo) clamps to them, so the +#: web form and the renderer agree. Below 0.1 a logo is a dot; ten times a +#: panel-sized box is already far off the panel. +MIN_ELEMENT_SCALE = 0.1 +MAX_ELEMENT_SCALE = 10.0 + @dataclass(frozen=True) class ElementStyle: @@ -301,7 +312,7 @@ def expand_style_elements(schema: Dict[str, Any]) -> Dict[str, Any]: if layout_props: layout = props.setdefault('layout', { 'type': 'object', - 'title': 'Layout Offsets', + 'title': _LAYOUT_TITLE, 'description': 'Pixel offsets applied to each element ' '(positive x moves right, positive y moves down)', 'x-advanced': True, @@ -345,16 +356,14 @@ def _element_block_from_spec(element_key: str, 'type': 'string', 'title': 'Font Family', 'x-advanced': True, - # The core already ships this widget and the config form already - # allowlists it; without the hint the field rendered as a bare - # text box the user had to type a filename into. + # The core's font picker; without the hint the form renders a + # bare text box the user has to type a filename into. 'x-widget': 'font-selector', } # A bitmap font ignores font_size and renders at its own baked-in # size, so the size ceiling has to be enforced when picking the # font, not when setting the size. - max_size = (size_spec or {}).get('max') if isinstance( - spec.get('size'), dict) else None + max_size = size_spec.get('max') if size_spec else None if isinstance(max_size, (int, float)): font_prop['x-options'] = {'maxFixedSize': max_size} if 'default' in font_spec: @@ -457,8 +466,8 @@ def _offset_block_from_spec(element_key: str, 'title': 'Scale', 'description': 'Size multiplier; 1 is the shipped size.', 'default': 1.0, - 'minimum': 0.1, - 'maximum': 10.0, + 'minimum': MIN_ELEMENT_SCALE, + 'maximum': MAX_ELEMENT_SCALE, 'x-advanced': True, } if isinstance(scale_spec, dict): @@ -539,7 +548,7 @@ def _modes_block(declaration: Dict[str, Any], if layout_props: element_props['layout'] = { 'type': 'object', - 'title': 'Layout Offsets', + 'title': _LAYOUT_TITLE, 'x-advanced': True, 'additionalProperties': False, 'properties': layout_props, @@ -680,7 +689,7 @@ def _modes_block_from_properties(props: Dict[str, Any], element_keys: list, if layout_props: element_props['layout'] = { 'type': 'object', - 'title': 'Layout Offsets', + 'title': _LAYOUT_TITLE, 'x-advanced': True, 'additionalProperties': False, 'properties': layout_props, @@ -1010,12 +1019,15 @@ def _coerce_align(value: Any) -> Optional[str]: return None -def _coerce_scale(value: Any, default: float) -> float: - """A positive size multiplier, or ``default``. +def coerce_scale(value: Any, default: float = 1.0) -> float: + """A usable size multiplier: ``value`` clamped to + [MIN_ELEMENT_SCALE, MAX_ELEMENT_SCALE], or ``default``. - Clamped rather than merely validated: a scale of 0 or a negative one is - a zero-or-inverted image, and the panel is 32 pixels tall -- a typo - should cost a wrong size, not a crash inside PIL. + ``default`` is returned for anything that is not a finite positive number + (None, a bool, a string, 0, a negative, NaN, infinity): those are typos, + and a typo should cost the shipped size, not a blank or inverted image or + a crash inside PIL. A positive number outside the range is a real request + for "smaller" or "bigger", so it is clamped rather than ignored. """ if isinstance(value, bool) or value is None: return default @@ -1023,9 +1035,9 @@ def _coerce_scale(value: Any, default: float) -> float: scale = float(value) except (TypeError, ValueError): return default - if scale <= 0: + if not math.isfinite(scale) or scale <= 0: return default - return min(scale, 10.0) + return min(max(scale, MIN_ELEMENT_SCALE), MAX_ELEMENT_SCALE) def _coerce_offset(value: Any, default: int, element_key: str, @@ -1194,7 +1206,7 @@ def element_scale(config: Any, element_key: str, default: float = 1.0, try: value = _element_field(config, element_key, 'scale', mode, in_layout=True) - return default if value is None else _coerce_scale(value, default) + return default if value is None else coerce_scale(value, default) except Exception as e: logger.warning("Error reading scale for %s: %s", element_key, e) return default @@ -1371,12 +1383,6 @@ class ElementStyleResolver: return {} return _lookup_element(block.get('layout'), element_key) - @classmethod - def _layout_axis(cls, block: Dict[str, Any], element_key: str, - axis: str) -> Any: - """``block['layout'][element][axis]``, or None if absent anywhere.""" - return cls._layout_element(block, element_key).get(axis) - # -- resolution internals ----------------------------------------------- @@ -1497,7 +1503,7 @@ class ElementStyleResolver: user_forced_color=bool(color_forced), visible=_coerce_bool(visible, True), align=_coerce_align(align), - scale=_coerce_scale(scale, 1.0), + scale=coerce_scale(scale, 1.0), ) def _classic_style(self, classic_font: str, classic_size: int, diff --git a/src/error_aggregator.py b/src/error_aggregator.py index 55fc7b1e..49b656a2 100644 --- a/src/error_aggregator.py +++ b/src/error_aggregator.py @@ -26,6 +26,19 @@ from src.exceptions import LEDMatrixError from src.redaction import redact_credentials +def _format_trace(error: BaseException) -> str: + """The traceback carried by ``error`` itself. + + Callers often record an exception after its ``except`` block has ended, + or from another thread than the one that raised it (plugin_executor runs + plugins on worker threads), where ``traceback.format_exc()`` has nothing + to report. The exception object keeps its own ``__traceback__``, so the + trace is built from that. An exception that was created but never raised + has no traceback, and the result is just its type and message. + """ + return "".join(traceback.format_exception(type(error), error, error.__traceback__)) + + @dataclass class ErrorRecord: """Record of a single error occurrence.""" @@ -145,8 +158,8 @@ class ErrorAggregator: with self._lock: error_type = type(error).__name__ - # Extract additional context from LEDMatrixError subclasses - error_context = context or {} + # A copy, so the caller's dict is not changed behind its back. + error_context = dict(context) if context else {} if isinstance(error, LEDMatrixError) and error.context: error_context.update(error.context) @@ -157,7 +170,7 @@ class ErrorAggregator: context=error_context, plugin_id=plugin_id, operation=operation, - stack_trace=traceback.format_exc() + stack_trace=_format_trace(error) ) # Add record (with size limit) diff --git a/src/font_manager.py b/src/font_manager.py index f013aecc..38f2f173 100644 --- a/src/font_manager.py +++ b/src/font_manager.py @@ -40,6 +40,11 @@ from pathlib import Path from PIL import ImageFont from src.common.bdf_font import load_bdf_face, read_bdf_native_size from src.common.font_layout import load_truetype, resolve_asset_path +from src.common.permission_utils import ( + ensure_directory_permissions, + get_assets_dir_mode, + get_config_dir_mode, +) from typing import Dict, Tuple, Optional, Union, Any, List from src.deprecation import deprecated @@ -57,7 +62,6 @@ class FontManager: def __init__(self, config: Dict[str, Any]): self.config = config - self.fonts_config = config.get("fonts", {}) # Font discovery and catalog self.font_catalog: Dict[str, str] = {} # family_name -> file_path @@ -73,10 +77,8 @@ class FontManager: # Plugin font management self.plugin_fonts: Dict[str, Dict[str, Any]] = {} # plugin_id -> font_manifest self.plugin_font_catalogs: Dict[str, Dict[str, str]] = {} # plugin_id -> {family_name -> file_path} - self.font_metadata: Dict[str, Dict[str, Any]] = {} # family_name -> metadata - self.font_dependencies: Dict[str, List[str]] = {} # family_name -> [required_families] - # Manager font registration - NEW for manager-centric model + # Fonts managers and plugins report using (register_manager_font). self.manager_fonts: Dict[str, Dict[str, Any]] = {} # manager_id -> {element_key: {family, size_px, color}} self.detected_fonts: Dict[str, Dict[str, Any]] = {} # element_key -> {family, size_px, color, manager_id, usage_count} # Bumped when a manager's registered families change (not when one @@ -88,13 +90,10 @@ class FontManager: self.temp_font_dir = Path(tempfile.gettempdir()) / "ledmatrix_fonts" self.temp_font_dir.mkdir(exist_ok=True) - # Performance monitoring + # Counters behind get_performance_stats(). self.performance_stats = { - "font_load_times": {}, "cache_hits": 0, "cache_misses": 0, - "render_times": {}, - "total_renders": 0, "failed_loads": 0, "start_time": time.time() } @@ -105,9 +104,6 @@ class FontManager: "four_by_six": "assets/fonts/4x6-font.ttf", "five_by_seven": "assets/fonts/5x7.bdf", "tom_thumb": "assets/fonts/tom-thumb.bdf" - # Note: cozette_bdf removed - font file not available - # To re-enable: download cozette.bdf from https://github.com/the-moonwitch/Cozette - # and add: "cozette_bdf": "assets/fonts/cozette.bdf" } # Size tokens for convenience @@ -116,7 +112,10 @@ class FontManager: } # Font overrides storage (for manual overrides) - self.font_overrides_file = "config/font_overrides.json" + # Under the install root's config/ (which always exists), not the + # cwd: the file itself may not exist yet, and resolve_asset_path + # hands back a missing path unchanged. + self.font_overrides_file = os.path.join(resolve_asset_path("config"), "font_overrides.json") self.font_overrides: Dict[str, Dict[str, Any]] = {} # Bumped whenever cached font objects are invalidated, so holders of @@ -128,7 +127,6 @@ class FontManager: def reload_config(self, new_config: Dict[str, Any]): """Reload configuration and refresh font catalog.""" self.config = new_config - self.fonts_config = new_config.get("fonts", {}) self.font_cache.clear() # Clear cache to force reload self.metrics_cache.clear() # Clear metrics cache self.cache_generation += 1 @@ -136,7 +134,6 @@ class FontManager: logger.info("FontManager configuration reloaded successfully") # ==================== Manager Font Registration ==================== - # NEW: Support for managers to register their font choices dynamically def register_manager_font(self, manager_id: str, element_key: str, family: str, size_px: int, color: Optional[Tuple[int, int, int]] = None): @@ -209,16 +206,23 @@ class FontManager: # ==================== Plugin Font Management ==================== - def register_plugin_fonts(self, plugin_id: str, font_manifest: Dict[str, Any]) -> bool: + def register_plugin_fonts(self, plugin_id: str, font_manifest: Dict[str, Any], + plugin_dir: Optional[Union[str, Path]] = None) -> bool: """ Register fonts for a specific plugin. Args: plugin_id: Unique identifier for the plugin - font_manifest: Font manifest from plugin's manifest.json + font_manifest: The ``fonts`` block of the plugin's manifest.json + plugin_dir: The plugin's directory, which ``plugin://`` sources + are relative to. PluginManager passes the directory it loaded + the plugin from. When omitted, the plugin is looked up in the + configured ``plugin_system.plugins_directory`` and then in + ``plugins/``. Returns: - True if registration successful, False otherwise + True if the manifest was valid (individual fonts that fail to load + are logged and skipped), False otherwise """ try: # Validate font manifest structure @@ -235,7 +239,7 @@ class FontManager: # Process font definitions fonts = font_manifest.get("fonts", []) for font_def in fonts: - if self._register_plugin_font(plugin_id, font_def): + if self._register_plugin_font(plugin_id, font_def, plugin_dir): logger.info(f"Successfully registered font {font_def.get('family')} for plugin {plugin_id}") logger.info(f"Registered {len(fonts)} fonts for plugin {plugin_id}") @@ -270,7 +274,8 @@ class FontManager: return True - def _register_plugin_font(self, plugin_id: str, font_def: Dict[str, Any]) -> bool: + def _register_plugin_font(self, plugin_id: str, font_def: Dict[str, Any], + plugin_dir: Optional[Union[str, Path]] = None) -> bool: """Register a single font from a plugin.""" try: family = font_def["family"] @@ -284,7 +289,7 @@ class FontManager: elif source.startswith("plugin://"): # Relative to plugin directory relative_path = source.replace("plugin://", "") - font_path = self._resolve_plugin_font_path(plugin_id, relative_path) + font_path = self._resolve_plugin_font_path(plugin_id, relative_path, plugin_dir) else: # Absolute or relative path font_path = source @@ -298,14 +303,6 @@ class FontManager: self.plugin_font_catalogs[plugin_id][family] = font_path self.font_catalog[namespaced_family] = font_path - # Store metadata - if "metadata" in font_def: - self.font_metadata[namespaced_family] = font_def["metadata"] - - # Store dependencies - if "dependencies" in font_def: - self.font_dependencies[namespaced_family] = font_def["dependencies"] - logger.info(f"Registered plugin font: {namespaced_family} -> {font_path}") return True @@ -367,11 +364,15 @@ class FontManager: return '.zip' return '.ttf' # default - def _resolve_plugin_font_path(self, plugin_id: str, relative_path: str) -> Optional[str]: - """Resolve a plugin-relative font path.""" - # Assume plugins are in a 'plugins' directory - plugin_dir = Path("plugins") / plugin_id - font_path = plugin_dir / relative_path + def _resolve_plugin_font_path(self, plugin_id: str, relative_path: str, + plugin_dir: Optional[Union[str, Path]] = None) -> Optional[str]: + """Resolve a ``plugin://`` font path against the plugin's directory.""" + if plugin_dir is None: + plugin_dir = self._find_plugin_dir(plugin_id) + if plugin_dir is None: + logger.error(f"Plugin font {relative_path}: directory for plugin {plugin_id} not found") + return None + font_path = Path(plugin_dir) / relative_path if font_path.exists(): return str(font_path) @@ -379,6 +380,18 @@ class FontManager: logger.error(f"Plugin font not found: {font_path}") return None + def _find_plugin_dir(self, plugin_id: str) -> Optional[Path]: + """The installed directory of ``plugin_id``, for callers of + register_plugin_fonts that do not pass one: the configured plugins + directory (relative paths are relative to the install root), then the + legacy ``plugins/`` directory.""" + # Imported here: src.plugin_system's package import loads PluginManager. + from src.plugin_system.plugin_dirs import resolve_plugin_dir + + configured = (self.config.get("plugin_system") or {}).get("plugins_directory") or "plugin-repos" + search_dirs = [Path(resolve_asset_path(configured)), Path(resolve_asset_path("plugins"))] + return resolve_plugin_dir(plugin_id, search_dirs, prefix=True) + @deprecated("3.7.0") def unregister_plugin_fonts(self, plugin_id: str) -> bool: """Unregister all fonts for a plugin.""" @@ -390,8 +403,6 @@ class FontManager: namespaced_family = f"{plugin_id}::{family}" if namespaced_family in self.font_catalog: del self.font_catalog[namespaced_family] - if namespaced_family in self.font_metadata: - del self.font_metadata[namespaced_family] del self.plugin_font_catalogs[plugin_id] @@ -442,8 +453,6 @@ class FontManager: Returns: Resolved font object """ - start_time = time.time() - try: # Check for manual overrides first if element_key in self.font_overrides: @@ -460,14 +469,7 @@ class FontManager: if plugin_id in self.plugin_font_catalogs and family in self.plugin_font_catalogs[plugin_id]: family = f"{plugin_id}::{family}" - # Get the font - font = self.get_font(family, size_px) - - # Record performance - duration = time.time() - start_time - self._record_performance_metric("resolve", f"{family}_{size_px}", duration) - - return font + return self.get_font(family, size_px) except Exception as e: logger.error(f"Error resolving font for {element_key}: {e}", exc_info=True) @@ -491,7 +493,6 @@ class FontManager: return self.font_cache[cache_key] self.performance_stats["cache_misses"] += 1 - start_time = time.time() # Load font font_path = self.font_catalog.get(family) @@ -506,15 +507,13 @@ class FontManager: else: font = load_truetype(font_path, size_px) except Exception as e: + # The one log line for a failed load: _load_bdf_font lets + # its error propagate to here. logger.error(f"Error loading font {font_path}: {e}") self.performance_stats["failed_loads"] += 1 font = ImageFont.load_default() - # Cache and record performance self.font_cache[cache_key] = font - duration = time.time() - start_time - self.performance_stats["font_load_times"][cache_key] = duration - return font def _load_bdf_font(self, font_path: str, size_px: int) -> freetype.Face: @@ -524,11 +523,7 @@ class FontManager: rather than failing over to PIL's default font, a different typeface (see :func:`src.common.bdf_font.load_bdf_face`). """ - try: - return load_bdf_face(font_path, size_px)[0] - except Exception as e: - logger.error(f"Error loading BDF font {font_path}: {e}") - raise + return load_bdf_face(font_path, size_px)[0] def get_native_bdf_size(self, family: str) -> Optional[int]: """The one true pixel size of a BDF family in the catalog, or None @@ -723,11 +718,6 @@ class FontManager: def _save_overrides(self): """Save current font overrides to file.""" try: - from pathlib import Path - from src.common.permission_utils import ( - ensure_directory_permissions, - get_config_dir_mode - ) font_overrides_path = Path(self.font_overrides_file) ensure_directory_permissions(font_overrides_path.parent, get_config_dir_mode()) with open(self.font_overrides_file, 'w') as f: @@ -754,12 +744,6 @@ class FontManager: """Get available size tokens.""" return self.size_tokens.copy() - def _record_performance_metric(self, operation: str, font_key: str, duration: float): - """Record a performance metric.""" - if operation not in self.performance_stats: - self.performance_stats[operation] = {} - self.performance_stats[operation][font_key] = duration - @deprecated("3.7.0") def get_performance_stats(self) -> Dict[str, Any]: """Get performance statistics.""" @@ -789,7 +773,8 @@ class FontManager: @deprecated("3.7.0") def add_font(self, font_file_path: str, family_name: str) -> bool: - """Add a new font to the catalog.""" + """Add ``font_file_path`` to the catalog as ``family_name``. The file + stays where it is; only assets/fonts is created if it is missing.""" try: # Validate font file if not os.path.exists(font_file_path): @@ -801,13 +786,7 @@ class FontManager: logger.warning(f"Font family '{family_name}' already exists") return False - # Copy font to assets/fonts directory - from pathlib import Path - from src.common.permission_utils import ( - ensure_directory_permissions, - get_assets_dir_mode - ) - fonts_dir = Path("assets/fonts") + fonts_dir = Path(resolve_asset_path("assets/fonts")) ensure_directory_permissions(fonts_dir, get_assets_dir_mode()) # Add to catalog diff --git a/src/logo_downloader.py b/src/logo_downloader.py index c74096df..c042188a 100644 --- a/src/logo_downloader.py +++ b/src/logo_downloader.py @@ -16,7 +16,7 @@ import json from typing import Dict, List, Optional, Tuple from pathlib import Path from PIL import Image, ImageDraw, ImageFont, UnidentifiedImageError -from src.common.font_layout import load_truetype +from src.common.font_layout import load_truetype, resolve_asset_path from PIL.PngImagePlugin import PngInfo from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry @@ -370,22 +370,15 @@ class LogoDownloader: @staticmethod def get_logo_filename_variations(abbreviation: str) -> list: - """Get possible filename variations for a team abbreviation.""" - variations = [] + """Filenames a logo for ``abbreviation`` may be stored under: the + upper-cased abbreviation as given, then its normalize_abbreviation() + form (``TA&M.png``, then ``TAANDM.png``).""" original = abbreviation.upper() normalized = LogoDownloader.normalize_abbreviation(abbreviation) - - # Add original and normalized versions - variations.extend([f"{original}.png", f"{normalized}.png"]) - - # Special handling for known cases - if original == 'TA&M': - # TA&M has a file named TA&M.png, but normalize creates TAANDM.png - variations = [f"{original}.png", f"{normalized}.png"] - - return variations + return [f"{original}.png", f"{normalized}.png"] - # Allowlist for league names used in filesystem paths: alphanumerics, underscores, dashes only + # Allowlist for a league name or code that goes into a filesystem path or + # an ESPN URL: lower-case alphanumerics, underscores and dashes only. _SAFE_LEAGUE_RE = re.compile(r'^[a-z0-9_-]+$') def get_logo_directory(self, league: str) -> str: @@ -462,15 +455,12 @@ class LogoDownloader: logger.error(f"Unexpected error downloading logo for {team_abbreviation}: {e}") return False - # Allowlist for the league_code segment interpolated into ESPN API URLs - _SAFE_LEAGUE_CODE_RE = re.compile(r'^[a-z0-9_-]+$') - def _resolve_api_url(self, league: str) -> Optional[str]: """Resolve the ESPN API teams URL for a league, with dynamic fallback for custom soccer leagues.""" api_url = self.API_ENDPOINTS.get(league) if not api_url and league.startswith('soccer_'): league_code = league[len('soccer_'):] - if not self._SAFE_LEAGUE_CODE_RE.match(league_code): + if not self._SAFE_LEAGUE_RE.match(league_code): logger.warning(f"Rejecting unsafe league_code for ESPN URL construction: {league_code!r}") return None api_url = f'https://site.api.espn.com/apis/site/v2/sports/soccer/{league_code}/teams' @@ -501,7 +491,8 @@ class LogoDownloader: return None def fetch_single_team(self, league: str, team_id: str) -> Optional[Dict]: - """Fetch team data from ESPN API for a specific league.""" + """Fetch one team's record (``/``) from the + ESPN API; None on any request or parse failure.""" api_url = self._resolve_api_url(league) if not api_url: logger.error(f"No API endpoint configured for league: {league}") @@ -520,7 +511,7 @@ class LogoDownloader: logger.error(f"Error fetching team data for {team_id} in {league}: {e}") return None except json.JSONDecodeError as e: - logger.error(f"Error parsing JSON response for{team_id} in {league}: {e}") + logger.error(f"Error parsing JSON response for {team_id} in {league}: {e}") return None def extract_teams_from_data(self, data: Dict, league: str) -> List[Dict[str, str]]: @@ -626,42 +617,6 @@ class LogoDownloader: # Default to FBS for unknown conferences return 'FBS' - def _get_team_name_variations(self, abbreviation: str) -> List[str]: - """Generate common variations of a team abbreviation for matching.""" - variations = set() - abbr = abbreviation.upper() - variations.add(abbr) - - # Add normalized version - variations.add(self.normalize_abbreviation(abbr)) - - # Common substitutions - substitutions = { - '&': ['AND', 'A'], - 'A&M': ['TAMU', 'TA&M', 'TEXASAM'], - 'STATE': ['ST', 'ST.'], - 'UNIVERSITY': ['U', 'UNIV'], - 'COLLEGE': ['C', 'COL'], - 'TECHNICAL': ['TECH', 'T'], - 'NORTHERN': ['NORTH', 'N'], - 'SOUTHERN': ['SOUTH', 'S'], - 'EASTERN': ['EAST', 'E'], - 'WESTERN': ['WEST', 'W'] - } - - # Apply substitutions - for original, replacements in substitutions.items(): - if original in abbr: - for replacement in replacements: - variations.add(abbr.replace(original, replacement)) - variations.add(abbr.replace(original, '')) # Remove the word entirely - - # Add common abbreviations for Texas A&M - if 'A&M' in abbr or 'TAMU' in abbr: - variations.update(['TAMU', 'TA&M', 'TEXASAM', 'TEXAS_A&M', 'TEXAS_AM']) - - return list(variations) - def download_missing_logos_for_league(self, league: str, force_download: bool = False) -> Tuple[int, int]: """Download missing logos for a specific league.""" logger.info(f"Starting logo download for league: {league}") @@ -794,7 +749,9 @@ class LogoDownloader: return False try: logo_url = data["team"]["logos"][0]["href"] - except KeyError: + except (KeyError, IndexError, TypeError): + # A team without logos comes back with an empty list. + logger.debug(f"No logo URL for team {team_id} in {league}") return False # Download the logo success = self.download_logo(logo_url, logo_path, team_abbreviation) @@ -824,24 +781,39 @@ class LogoDownloader: logger.info(f"Overall logo download results: {total_downloaded} downloaded, {total_failed} failed") return results - def create_placeholder_logo(self, team_abbreviation: str, logo_dir: str) -> bool: - """Create a placeholder logo when real logo cannot be downloaded.""" + def create_placeholder_logo(self, team_abbreviation: str, logo_dir: str, + filepath: Optional[Path] = None) -> bool: + """Write a grey placeholder with the team abbreviation on it, for when + the real logo cannot be downloaded. + + Args: + team_abbreviation: Drawn on the placeholder. + logo_dir: Directory for the file when ``filepath`` is not given; + the file is then ``.png``. + filepath: The exact path to write, which is where the caller will + look for the logo. ``logo_dir`` is ignored when it is given. + + Returns: + True if the placeholder was written, False otherwise (logged). + """ + if filepath is not None: + filepath = Path(filepath) + logo_dir = str(filepath.parent) try: - # Ensure the logo directory exists if not self.ensure_logo_directory(logo_dir): logger.error(f"Failed to create logo directory: {logo_dir}") return False - - filename = f"{self.normalize_abbreviation(team_abbreviation)}.png" - filepath = Path(logo_dir) / filename - # Create a simple placeholder logo - logo = Image.new('RGBA', (64, 64), (100, 100, 100, 255)) # Gray background + if filepath is None: + filename = f"{self.normalize_abbreviation(team_abbreviation)}.png" + filepath = Path(logo_dir) / filename + + logo = Image.new('RGBA', PLACEHOLDER_SIZE, PLACEHOLDER_BG) draw = ImageDraw.Draw(logo) # Try to load a font, fallback to default try: - font = load_truetype("assets/fonts/PressStart2P-Regular.ttf", 12) + font = load_truetype(resolve_asset_path("assets/fonts/PressStart2P-Regular.ttf"), 12) except (OSError, IOError): try: font = ImageFont.load_default() @@ -855,8 +827,8 @@ class LogoDownloader: bbox = draw.textbbox((0, 0), text, font=font) text_width = bbox[2] - bbox[0] text_height = bbox[3] - bbox[1] - x = (64 - text_width) // 2 - y = (64 - text_height) // 2 + x = (PLACEHOLDER_SIZE[0] - text_width) // 2 + y = (PLACEHOLDER_SIZE[1] - text_height) // 2 draw.text((x, y), text, font=font, fill=(255, 255, 255, 255)) else: # Fallback without font @@ -963,14 +935,19 @@ def download_missing_logo(league: str, team_id: str, team_abbreviation: str, log Convenience function to download a missing team logo. Args: - team_abbreviation: Team abbreviation (e.g., 'UGA', 'BAMA', 'TA&M') league: League identifier (e.g., 'ncaa_fb', 'nfl') - logo_path: Full path to where the logo should be saved + team_id: ESPN team id, used to look up the logo URL when + ``logo_url`` is not given + team_abbreviation: Team abbreviation (e.g., 'UGA', 'BAMA', 'TA&M') + logo_path: Where the logo (or placeholder) is written; relative paths + are relative to the install root logo_url: Optional direct URL to the logo create_placeholder: Whether to create a placeholder if download fails Returns: - True if logo exists or was successfully downloaded, False otherwise + True if a logo or placeholder is at ``logo_path`` afterwards (it was + already there, was downloaded, or a placeholder was written), + False otherwise. """ downloader = shared_downloader() @@ -1011,7 +988,7 @@ def download_missing_logo(league: str, team_id: str, team_abbreviation: str, log time.sleep(0.1) # Small delay if not success and create_placeholder: logger.info(f"Creating placeholder logo for {team_abbreviation}") - success = downloader.create_placeholder_logo(team_abbreviation, logo_dir) + success = downloader.create_placeholder_logo(team_abbreviation, logo_dir, filepath=filepath) return success success = downloader.download_missing_logo_for_team(league, team_id, team_abbreviation, logo_path) @@ -1019,7 +996,7 @@ def download_missing_logo(league: str, team_id: str, team_abbreviation: str, log if not success and create_placeholder: logger.info(f"Creating placeholder logo for {team_abbreviation}") # Create placeholder as fallback - success = downloader.create_placeholder_logo(team_abbreviation, logo_dir) + success = downloader.create_placeholder_logo(team_abbreviation, logo_dir, filepath=filepath) if success: logger.info(f"Successfully handled logo for {team_abbreviation}") diff --git a/src/plugin_system/base_plugin.py b/src/plugin_system/base_plugin.py index 8bd62635..5af7f7fe 100644 --- a/src/plugin_system/base_plugin.py +++ b/src/plugin_system/base_plugin.py @@ -11,7 +11,6 @@ Stability: Stable - maintains backward compatibility from abc import ABC, abstractmethod from enum import Enum from typing import Dict, Any, Optional, List -import logging import os import sys from src.logging_config import get_logger @@ -511,104 +510,53 @@ class BasePlugin(ABC): """ Get the display duration for this plugin instance. - Automatically detects duration from: - 1. self.display_duration instance variable (if exists) - 2. self.config.get("display_duration", 15.0) (fallback) + Uses, in order, the first positive number among: + 1. ``self.display_duration`` (a common pattern in scoreboard plugins) + 2. ``self.config["display_duration"]`` + 3. 15.0 - Can be overridden by plugins to provide dynamic durations based - on content (e.g., longer duration for more complex displays). + Numeric strings count as numbers. Can be overridden by plugins to + provide dynamic durations based on content (e.g., longer duration for + more complex displays). Returns: Duration in seconds to display this plugin's content """ - # Check for instance variable first (common pattern in scoreboard plugins) - if hasattr(self, 'display_duration'): - try: - duration = getattr(self, 'display_duration') - # Handle None case - if duration is None: - pass # Fall through to config - # Try to convert to float if it's a number or numeric string. - # bool is excluded: it's an int subclass, and True would - # otherwise read as a 1-second duration. - elif isinstance(duration, (int, float)) and not isinstance(duration, bool): - if duration > 0: - return float(duration) - else: - self.logger.debug( - "display_duration instance variable is non-positive (%s), using config fallback", - duration - ) - # Try converting string representations of numbers - elif isinstance(duration, str): - try: - duration_float = float(duration) - if duration_float > 0: - return duration_float - else: - self.logger.debug( - "display_duration string value is non-positive (%s), using config fallback", - duration - ) - except (ValueError, TypeError): - self.logger.warning( - "display_duration instance variable has invalid string value '%s', using config fallback", - duration - ) - else: - self.logger.warning( - "display_duration instance variable has unexpected type %s (value: %s), using config fallback", - type(duration).__name__, duration - ) - except (TypeError, ValueError, AttributeError) as e: - self.logger.warning( - "Error reading display_duration instance variable: %s, using config fallback", - e - ) - - # Fall back to config - config_duration = self.config.get("display_duration", 15.0) try: - # Ensure config value is also a valid float (bool excluded — an - # int subclass that would otherwise read True as 1 second) - if isinstance(config_duration, (int, float)) and not isinstance(config_duration, bool): - if config_duration > 0: - return float(config_duration) - else: - self.logger.debug( - "Config display_duration is non-positive (%s), using default 15.0", - config_duration - ) - return 15.0 - elif isinstance(config_duration, str): - try: - duration_float = float(config_duration) - if duration_float > 0: - return duration_float - else: - self.logger.debug( - "Config display_duration string is non-positive (%s), using default 15.0", - config_duration - ) - return 15.0 - except ValueError: - self.logger.warning( - "Config display_duration has invalid string value '%s', using default 15.0", - config_duration - ) - return 15.0 - else: - self.logger.warning( - "Config display_duration has unexpected type %s (value: %s), using default 15.0", - type(config_duration).__name__, config_duration - ) - except (ValueError, TypeError) as e: + duration = getattr(self, 'display_duration', None) + except (TypeError, ValueError, AttributeError) as e: + # A plugin may define display_duration as a property that raises. self.logger.warning( - "Error processing config display_duration: %s, using default 15.0", - e - ) + "Error reading display_duration instance variable: %s, using config fallback", e) + duration = None + if duration is not None: + seconds = self._positive_seconds(duration, "display_duration instance variable") + if seconds is not None: + return seconds - return 15.0 + seconds = self._positive_seconds( + self.config.get("display_duration", 15.0), "config display_duration") + return seconds if seconds is not None else 15.0 + + def _positive_seconds(self, value: Any, source: str) -> Optional[float]: + """``value`` as a positive float, or None (with a log line) if it is not one. + + bool is rejected although it is an int subclass: True would otherwise + read as a 1-second duration. + """ + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + self.logger.warning("%s has unexpected type %s (value: %s), ignoring it", + source, type(value).__name__, value) + return None + try: + seconds = float(value) + except ValueError: + self.logger.warning("%s has invalid value %r, ignoring it", source, value) + return None + if seconds > 0: + return seconds + self.logger.debug("%s is non-positive (%s), ignoring it", source, value) + return None # --------------------------------------------------------------------- # Dynamic duration support hooks @@ -926,25 +874,20 @@ class BasePlugin(ABC): config_mode, self.plugin_id ) - # Fall back to mapping legacy content_type - content_type = self.get_vegas_content_type() - if content_type == 'multi': + # Fall back to mapping legacy content_type. 'none' (excluded from + # Vegas) also maps to FIXED_SEGMENT: exclusion is decided by checking + # get_vegas_content_type() separately. + if self.get_vegas_content_type() == 'multi': return VegasDisplayMode.SCROLL - elif content_type == 'static': - return VegasDisplayMode.FIXED_SEGMENT - elif content_type == 'none': - # 'none' means excluded - return FIXED_SEGMENT as default - # The exclusion is handled by checking get_vegas_content_type() separately - return VegasDisplayMode.FIXED_SEGMENT - return VegasDisplayMode.FIXED_SEGMENT def get_supported_vegas_modes(self) -> List[VegasDisplayMode]: """ Return list of Vegas display modes this plugin supports. - Used by the web UI to show available mode options for user configuration. - Override to customize which modes are available for this plugin. + Not currently consulted by core: neither Vegas mode nor the web UI + calls it. It is kept, and plugins override it, as the declared set of + modes a future mode picker would offer. By default: - 'multi' content type plugins support SCROLL and FIXED_SEGMENT @@ -972,6 +915,10 @@ class BasePlugin(ABC): """ Get the preferred width for this plugin in Vegas FIXED_SEGMENT mode. + Not currently consulted by core: Vegas mode sizes a card from the + ``vegas_width_pct`` / ``vegas_scroll.render_width_pct`` settings + (see get_vegas_render_width()). Kept because plugins override it. + Returns the number of panels this plugin should occupy when displayed as a fixed segment. The actual pixel width is calculated as: width = panels * single_panel_width @@ -1025,7 +972,7 @@ class BasePlugin(ABC): required_fields = ['api_key', 'city'] for field in required_fields: if field not in self.config: - self.logger.error("Missing required field: %s", field) + self.logger.error("Missing required field: %s", field) return False return True """ diff --git a/src/plugin_system/operation_queue.py b/src/plugin_system/operation_queue.py index 8ac18080..b0e421b8 100644 --- a/src/plugin_system/operation_queue.py +++ b/src/plugin_system/operation_queue.py @@ -9,8 +9,6 @@ import threading import queue from typing import Dict, Optional, List, Callable, Any from datetime import datetime -from pathlib import Path -import json from src.plugin_system.operation_types import ( PluginOperation, OperationType, OperationStatus @@ -28,28 +26,22 @@ class PluginOperationQueue: - Prevents concurrent operations on same plugin - Operation status tracking - Operation cancellation - - Operation history + - In-memory history of finished operations + + The history is not persisted. The web UI's operation history comes from + OperationHistory (operation_history.py), which has its own file; a copy + written here was never read back by anything. """ - - def __init__( - self, - history_file: Optional[str] = None, - max_history: int = 100, - lazy_load: bool = False - ): + + def __init__(self, max_history: int = 100): """ Initialize operation queue. - + Args: - history_file: Optional path to file for persisting operation history max_history: Maximum number of operations to keep in history - lazy_load: If True, defer loading history file until first access """ self.logger = get_logger(__name__) - self.history_file = Path(history_file) if history_file else None self.max_history = max_history - self._lazy_load = lazy_load - self._history_loaded = False # Operation tracking self._operations: Dict[str, PluginOperation] = {} @@ -62,20 +54,8 @@ class PluginOperationQueue: self._worker_thread: Optional[threading.Thread] = None self._stop_event = threading.Event() - # Load history from file if it exists (unless lazy loading) - if not self._lazy_load and self.history_file and self.history_file.exists(): - self._load_history() - self._history_loaded = True - - # Start worker thread self._start_worker() - - def _ensure_loaded(self) -> None: - """Ensure history is loaded (for lazy loading).""" - if not self._history_loaded and self.history_file and self.history_file.exists(): - self._load_history() - self._history_loaded = True - + def enqueue_operation( self, operation_type: OperationType, @@ -139,7 +119,6 @@ class PluginOperationQueue: Returns: PluginOperation if found, None otherwise """ - self._ensure_loaded() with self._lock: return self._operations.get(operation_id) @@ -184,7 +163,6 @@ class PluginOperationQueue: Returns: List of operations, sorted by creation time (newest first) """ - self._ensure_loaded() with self._lock: # Sort by creation time (newest first) history = sorted( @@ -314,11 +292,7 @@ class PluginOperationQueue: if self._active_operations[operation.plugin_id].operation_id == operation.operation_id: del self._active_operations[operation.plugin_id] - # Add to history self._add_to_history(operation) - - # Save history to file - self._save_history() def _add_to_history(self, operation: PluginOperation) -> None: """Add operation to history, maintaining max_history limit.""" @@ -330,46 +304,6 @@ class PluginOperationQueue: self._operation_history.sort(key=lambda op: op.created_at) self._operation_history = self._operation_history[-self.max_history:] - def _save_history(self) -> None: - """Save operation history to file.""" - if not self.history_file: - return - - try: - with self._lock: - # Convert operations to dicts - history_data = [op.to_dict() for op in self._operation_history] - - # Ensure directory exists - self.history_file.parent.mkdir(parents=True, exist_ok=True) - - # Write to file - with open(self.history_file, 'w') as f: - json.dump(history_data, f, indent=2) - - except Exception as e: - self.logger.warning(f"Error saving operation history: {e}") - - def _load_history(self) -> None: - """Load operation history from file.""" - if not self.history_file or not self.history_file.exists(): - return - - try: - with open(self.history_file, 'r') as f: - history_data = json.load(f) - - with self._lock: - self._operation_history = [ - PluginOperation.from_dict(op_data) - for op_data in history_data - ] - - self.logger.info(f"Loaded {len(self._operation_history)} operations from history") - - except Exception as e: - self.logger.warning(f"Error loading operation history: {e}") - def shutdown(self) -> None: """Shutdown the operation queue and worker thread.""" self.logger.info("Shutting down plugin operation queue") @@ -377,7 +311,4 @@ class PluginOperationQueue: if self._worker_thread and self._worker_thread.is_alive(): self._worker_thread.join(timeout=5.0) - - # Save history one last time - self._save_history() diff --git a/src/plugin_system/plugin_executor.py b/src/plugin_system/plugin_executor.py index 6446fca0..a2014145 100644 --- a/src/plugin_system/plugin_executor.py +++ b/src/plugin_system/plugin_executor.py @@ -93,7 +93,7 @@ class PluginExecutor: if result_container['exception']: error = result_container['exception'] error_msg = f"{plugin_context} operation failed: {error}" - self.logger.error(error_msg, exc_info=True) + self.logger.error(error_msg, exc_info=error) record_error(error, plugin_id=plugin_id, operation="execute") raise PluginError(error_msg, plugin_id=plugin_id) from error diff --git a/src/plugin_system/plugin_loader.py b/src/plugin_system/plugin_loader.py index 66feabab..7547c8f0 100644 --- a/src/plugin_system/plugin_loader.py +++ b/src/plugin_system/plugin_loader.py @@ -5,6 +5,7 @@ Handles plugin module imports, dependency installation, and class instantiation. Extracted from PluginManager to improve separation of concerns. """ +import errno import importlib import importlib.metadata import importlib.util @@ -184,6 +185,46 @@ def find_trusted_subdir(trusted_dir: str, name: str) -> Optional[str]: return None +def contained_plugin_dir(plugin_dir: Path, plugins_dir: Path) -> Optional[str]: + """``plugin_dir`` rebuilt from an entry enumerated under ``plugins_dir``. + + Returns None when ``plugin_dir`` is not a subdirectory of ``plugins_dir``. + Callers derive ``plugin_dir`` from a manifest-declared id, so the path is + rebuilt from :func:`find_trusted_subdir`'s answer rather than trusted: a + name that came out of ``os.scandir()`` on the trusted root carries no + taint, which is a real containment guarantee (and one CodeQL's + path-injection query can follow), not a string sanitiser. + """ + plugin_dir_real = os.path.realpath(str(plugin_dir)) + plugins_dir_real = os.path.realpath(str(plugins_dir)) + matched_name = find_trusted_subdir(plugins_dir_real, os.path.basename(plugin_dir_real)) + if matched_name is None: + return None + return os.path.join(plugins_dir_real, matched_name) + + +def requirements_to_install(plugin_dir: str, logger: logging.Logger, + label: str) -> Optional[str]: + """The plugin's requirements.txt if pip has work to do, else None. + + None when there is no requirements.txt, when it lists nothing (plugins + whose dependencies ship with core often keep an all-comments file), or + when every requirement is already installed. Shared by the loader and the + store so both skip pip for the same reasons; they differ only in how they + run it. + """ + requirements_file = os.path.join(plugin_dir, "requirements.txt") + if not os.path.isfile(requirements_file): + return None + if not requirements_has_real_deps(requirements_file): + logger.debug("requirements.txt for %s has no real dependencies, skipping pip", label) + return None + if requirements_are_satisfied(requirements_file): + logger.debug("Dependencies for %s already satisfied, skipping pip", label) + return None + return requirements_file + + class PluginLoader: """Handles plugin module loading and class instantiation.""" @@ -273,43 +314,15 @@ class PluginLoader: if not plugin_id: return False - # Resolve to a canonical absolute path (normalises .. and symlinks) - plugin_dir_real = os.path.realpath(str(plugin_dir)) - plugins_dir_real = os.path.realpath(str(plugins_dir)) - requested_name = os.path.basename(plugin_dir_real) - - # Match the requested directory against an entry actually enumerated - # from the trusted plugins_dir, and build the path from that entry -- - # not from requested_name. A name that came out of os.scandir() on a - # trusted root carries no taint regardless of what the caller asked - # for, so this is a real containment guarantee (an allowlist check - # against a trusted source), not a string-sanitisation of untrusted - # input that a static analyzer has to trust blindly. - matched_name = find_trusted_subdir(plugins_dir_real, requested_name) - if matched_name is None: + safe_plugin_dir = contained_plugin_dir(plugin_dir, plugins_dir) + if safe_plugin_dir is None: self.logger.error( "Plugin directory for %s not found inside plugins dir", plugin_id ) return False - safe_plugin_dir = os.path.join(plugins_dir_real, matched_name) - requirements_file = os.path.join(safe_plugin_dir, "requirements.txt") - - if not os.path.isfile(requirements_file): - return True # No dependencies needed - - if not requirements_has_real_deps(requirements_file): - self.logger.debug( - "requirements.txt for %s has no real dependencies (comments/blank only), skipping pip", - plugin_id - ) - return True - - if requirements_are_satisfied(requirements_file): - self.logger.debug( - "Dependencies for %s already satisfied in current environment, skipping pip", - plugin_id - ) + requirements_file = requirements_to_install(safe_plugin_dir, self.logger, plugin_id) + if requirements_file is None: return True try: @@ -348,8 +361,8 @@ class PluginLoader: # below). try: # sys.executable is this process's own interpreter (not - # attacker-influenced), and requirements_file is a path - # built internally by find_plugin_directory, never raw + # attacker-influenced), and requirements_file is rebuilt + # by contained_plugin_dir() from a trusted listing, never raw # external input. retry_result = subprocess.run( # nosec B603 - no shell invoked (list-form argv) # nosemgrep [sys.executable, "-m", "pip", "install", "--break-system-packages", @@ -384,10 +397,10 @@ class PluginLoader: except FileNotFoundError: self.logger.warning("pip not found. Skipping dependency installation for %s", plugin_id) return True - except (BrokenPipeError, OSError) as e: - # Handle broken pipe errors (errno 32) which can occur during pip downloads - # Often caused by network interruptions or output buffer issues - if isinstance(e, OSError) and e.errno == 32: + except OSError as e: + # A broken pipe (EPIPE) happens when pip's output pipe closes + # mid-download, usually a network interruption. + if e.errno == errno.EPIPE: self.logger.error( "Broken pipe error during dependency installation for %s. " "This usually indicates a network interruption or pip output buffer issue. " @@ -528,7 +541,7 @@ class PluginLoader: plugin_id: str, plugin_dir: Path, entry_point: str - ) -> Optional[Any]: + ) -> Any: """ Load a plugin module from file. @@ -547,7 +560,12 @@ class PluginLoader: entry_point: Entry point filename (e.g., 'manager.py') Returns: - Loaded module or None on error + The loaded module + + Raises: + PluginError: If the plugin id, directory or entry point is + invalid. Whatever the module raises while executing + propagates unchanged. """ plugin_id = os.path.basename(plugin_id or '') if not plugin_id: @@ -782,9 +800,7 @@ class PluginLoader: # Load module entry_point = manifest.get('entry_point', 'manager.py') module = self.load_module(plugin_id, plugin_dir, entry_point) - if module is None: - raise PluginError(f"Failed to load module for plugin {plugin_id}", plugin_id=plugin_id) - + # Get plugin class class_name = manifest.get('class_name') if not class_name: diff --git a/src/plugin_system/plugin_manager.py b/src/plugin_system/plugin_manager.py index 866e99eb..90d7bc67 100644 --- a/src/plugin_system/plugin_manager.py +++ b/src/plugin_system/plugin_manager.py @@ -2,7 +2,8 @@ Plugin Manager Manages plugin discovery, loading, and lifecycle for the LEDMatrix system. -Handles dynamic plugin loading from the plugins/ directory. +Loads plugins from the configured plugins directory +(``plugin_system.plugins_directory``, ``plugin-repos/`` by default). API Version: 1.0.0 """ @@ -40,7 +41,7 @@ class PluginManager: Manages plugin discovery, loading, and lifecycle. The PluginManager is responsible for: - - Discovering plugins in the plugins/ directory + - Discovering plugins in the configured plugins directory - Loading plugin modules and instantiating plugin classes - Managing plugin lifecycle (load, unload, reload) - Providing access to loaded plugins @@ -99,10 +100,9 @@ class PluginManager: self.plugin_directories: Dict[str, Path] = {} self.plugin_last_update: Dict[str, float] = {} - # Cached data-fetch intervals per plugin_id. - # _get_plugin_update_interval falls back to config_manager.get_config() - # (a full dict copy) when the manifest lacks an interval — caching avoids - # that copy on every 30-fps tick. Cleared on load/unload. + # Cached static data-fetch intervals per plugin_id, so the render + # loop's scheduling tick does not repeat the manifest/config lookup + # for every plugin. Cleared on load/unload. self._update_interval_cache: Dict[str, Optional[float]] = {} # Health tracking (optional, set by display_controller if available) @@ -110,14 +110,12 @@ class PluginManager: self.resource_monitor = None # --- Asynchronous plugin updates ------------------------------- - # update() used to run inline in the render loop (execute_update's - # internal thread.join(timeout=30) blocked it), so one slow plugin - # HTTP fetch froze scrolling for the whole fetch. Scheduling still - # happens on the render thread (run_scheduled_updates), but - # execution moves to this single background worker. Per-plugin - # locks keep a plugin's update() and display() mutually exclusive — - # today's implicit guarantee, now explicit (and, unlike today, - # also held across the post-timeout window). + # Run inline in the render loop, one slow plugin HTTP fetch in + # update() freezes scrolling for the whole fetch. Scheduling happens + # on the render thread (run_scheduled_updates); execution happens on + # this single background worker. Per-plugin locks keep a plugin's + # update() and display() mutually exclusive, including across the + # post-timeout window. # Kill switch: plugin_system.synchronous_updates: true restores the # inline path. self._update_queue: "queue.Queue[Optional[Tuple[str, float]]]" = queue.Queue() @@ -396,7 +394,8 @@ class PluginManager: self.font_manager, 'register_plugin_fonts' ): try: - self.font_manager.register_plugin_fonts(plugin_id, font_manifest) + self.font_manager.register_plugin_fonts( + plugin_id, font_manifest, plugin_dir=plugin_dir) except Exception as e: self.logger.warning( "Failed to register fonts for plugin %s: %s", plugin_id, e @@ -661,9 +660,15 @@ class PluginManager: if not self.unload_plugin(plugin_id): return False - # Re-discover to get updated manifest - manifest_path = self.plugins_dir / plugin_id / "manifest.json" - if manifest_path.exists(): + # Re-read the manifest so an edit to it takes effect, from the + # directory discovery found the plugin in: a directory's name need not + # be the id its manifest declares. + with self._discovery_lock: + directories = dict(self.plugin_directories) + plugin_dir = self.plugin_loader.find_plugin_directory( + plugin_id, self.plugins_dir, directories) + manifest_path = plugin_dir / "manifest.json" if plugin_dir is not None else None + if manifest_path is not None and manifest_path.exists(): try: with open(manifest_path, 'r', encoding='utf-8') as f: manifest = json.load(f) @@ -881,11 +886,12 @@ class PluginManager: updating, since a scheduler that propagates a plugin bug stops every other plugin too. - The static result is cached per plugin_id after the first lookup to - avoid calling config_manager.get_config() — which returns a full dict - copy — on every tick of the 30-fps display loop. The cache is - invalidated when a plugin is loaded or unloaded. The dynamic hook is - deliberately *not* cached: caching it would defeat its only purpose. + The static result is cached per plugin_id after the first lookup, so + the manifest/config resolution is not repeated on every scheduling + tick of the display loop. A change to ``update_interval`` in + config.json therefore takes effect when the plugin is next loaded or + unloaded, which clears the cache. The dynamic hook is deliberately + *not* cached: caching it would defeat its only purpose. """ dynamic = self._dynamic_update_interval(plugin_id, plugin_instance) if dynamic is not None: diff --git a/src/plugin_system/plugin_state.py b/src/plugin_system/plugin_state.py index 68087e1d..e1e26db1 100644 --- a/src/plugin_system/plugin_state.py +++ b/src/plugin_system/plugin_state.py @@ -41,7 +41,6 @@ class PluginStateManager: self._state_transition_counts: Dict[str, int] = {} self._error_info: Dict[str, Dict[str, Any]] = {} self._last_update: Dict[str, datetime] = {} - self._last_display: Dict[str, datetime] = {} def _record_transition(self, plugin_id: str) -> None: """Count a state transition. Callers must already hold ``_lock``.""" @@ -181,11 +180,7 @@ class PluginStateManager: def get_last_update(self, plugin_id: str) -> Optional[datetime]: """Get timestamp of last update() call.""" return self._last_update.get(plugin_id) - - def get_last_display(self, plugin_id: str) -> Optional[datetime]: - """Get timestamp of last display() call.""" - return self._last_display.get(plugin_id) - + def get_state_info(self, plugin_id: str) -> Dict[str, Any]: """ Get comprehensive state information for a plugin. @@ -212,7 +207,6 @@ class PluginStateManager: 'is_error': self.is_error(plugin_id), 'can_execute': self.can_execute(plugin_id), 'last_update': self.get_last_update(plugin_id), - 'last_display': self.get_last_display(plugin_id), 'error_info': self.get_error_info(plugin_id), 'state_history_count': self._state_transition_counts.get(plugin_id, 0) } @@ -231,5 +225,4 @@ class PluginStateManager: self._state_transition_counts.pop(plugin_id, None) self._error_info.pop(plugin_id, None) self._last_update.pop(plugin_id, None) - self._last_display.pop(plugin_id, None) diff --git a/src/plugin_system/repo_urls.py b/src/plugin_system/repo_urls.py new file mode 100644 index 00000000..0bfad8ae --- /dev/null +++ b/src/plugin_system/repo_urls.py @@ -0,0 +1,70 @@ +""" +Repository URL helpers shared by the plugin store and saved repositories. + +One definition of "the same repository URL", of how a GitHub URL maps to +``owner/repo``, and of the headers sent to the GitHub API. +""" + +from typing import Dict, Optional, Tuple +from urllib.parse import urlparse + +#: Hosts whose URLs name a GitHub repository. Matched against +#: ``urlparse(url).hostname``, never by substring: a substring test accepts +#: ``https://github.com.example.org/...`` as GitHub. +GITHUB_HOSTS = frozenset({'github.com', 'www.github.com'}) + +#: Sent on every request the store makes to GitHub. +USER_AGENT = 'LEDMatrix-Plugin-Manager/1.0' + + +def normalize_repo_url(url: str) -> str: + """``url`` without surrounding whitespace, trailing slashes or a trailing ``.git``. + + Only a *trailing* ``.git`` is removed. The unanchored + ``url.replace('.git', '')`` this replaces turned + ``https://github.com/user/my.github.io`` into ``.../myhub.io``. + Case is preserved; compare with :func:`same_repo`. + """ + url = url.strip().rstrip('/') + if url.endswith('.git'): + url = url[:-4] + return url + + +def same_repo(url_a: str, url_b: str) -> bool: + """Whether two URLs name the same repository. + + GitHub owner and repository names are case-insensitive, so + ``ChuckBuilds/LEDMatrix-Plugins`` and ``chuckbuilds/ledmatrix-plugins`` + are one repository. + """ + return normalize_repo_url(url_a).lower() == normalize_repo_url(url_b).lower() + + +def github_owner_repo(url: str) -> Optional[Tuple[str, str]]: + """``(owner, repo)`` for a github.com repository URL, else None. + + The first two path segments, so a URL that points inside the repository + (``.../owner/repo/tree/main/plugins/x``) still names ``owner/repo``. + """ + parsed = urlparse(normalize_repo_url(url)) + if parsed.hostname not in GITHUB_HOSTS: + return None + parts = [part for part in parsed.path.split('/') if part] + if len(parts) < 2: + return None + return parts[0], normalize_repo_url(parts[1]) + + +def github_api_headers(token: Optional[str] = None) -> Dict[str, str]: + """Headers for a GitHub REST API request, authenticated when ``token`` is set. + + An authenticated request gets 5000 requests an hour instead of 60. + """ + headers = { + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': USER_AGENT, + } + if token: + headers['Authorization'] = f'token {token}' + return headers diff --git a/src/plugin_system/resource_monitor.py b/src/plugin_system/resource_monitor.py index b024e726..ed11e9f3 100644 --- a/src/plugin_system/resource_monitor.py +++ b/src/plugin_system/resource_monitor.py @@ -33,7 +33,14 @@ class ResourceLimits: @dataclass class ResourceMetrics: - """Resource usage metrics for a plugin.""" + """Resource usage metrics for a plugin. + + ``memory_mb`` is the largest growth in this *process's* resident memory + seen across a single monitored call -- a high-water mark, not current + usage, and not the plugin's own footprint (another thread allocating + during the call counts too). ``cpu_percent`` is the whole process's CPU + use since the previous sample. + """ memory_mb: float = 0.0 cpu_percent: float = 0.0 execution_time: float = 0.0 @@ -42,11 +49,6 @@ class ResourceMetrics: max_execution_time: float = 0.0 min_execution_time: float = float('inf') last_update_time: float = field(default_factory=time.time) - - def update_average_execution_time(self): - """Update average execution time.""" - if self.call_count > 0: - self.total_execution_time = self.total_execution_time / self.call_count #: How often a plugin's metrics are written to the cache, in seconds. @@ -287,7 +289,8 @@ class PluginResourceMonitor: # Calculate execution time execution_time = time.time() - start_time - + memory_growth_mb = 0.0 + # Update metrics with self._lock: metrics.execution_time = execution_time @@ -302,17 +305,17 @@ class PluginResourceMonitor: # Update memory and CPU if monitoring enabled if self.enable_monitoring: - end_memory = self._get_process_memory_mb() - metrics.memory_mb = max(metrics.memory_mb, end_memory - start_memory) + memory_growth_mb = self._get_process_memory_mb() - start_memory + metrics.memory_mb = max(metrics.memory_mb, memory_growth_mb) # CPU is harder to measure per-call, so we track it separately metrics.cpu_percent = self._get_process_cpu_percent() - + # Persist metrics, at most once per interval per plugin. self._persist_metrics(plugin_id, metrics) - - # Check limits + if limits: - self._check_limits(plugin_id, metrics, limits, execution_time) + self._check_limits(plugin_id, metrics, limits, execution_time, + memory_growth_mb) return result @@ -326,9 +329,17 @@ class PluginResourceMonitor: metrics.last_update_time = time.time() raise - def _check_limits(self, plugin_id: str, metrics: ResourceMetrics, - limits: ResourceLimits, execution_time: float) -> None: - """Check if plugin has exceeded resource limits.""" + def _check_limits(self, plugin_id: str, metrics: ResourceMetrics, + limits: ResourceLimits, execution_time: float, + memory_growth_mb: float) -> None: + """Raise ResourceLimitExceeded if this call went over a limit. + + Execution time and memory growth are this call's own; CPU is the + latest process sample. Judging memory by the stored high-water mark + (``metrics.memory_mb``) instead would fail every call after the first + expensive one, so the health tracker's circuit breaker would reopen + on every recovery probe and the plugin would never update again. + """ warnings = [] errors = [] @@ -343,13 +354,13 @@ class PluginResourceMonitor: ) # Check memory - if limits.max_memory_mb and metrics.memory_mb > limits.max_memory_mb: + if limits.max_memory_mb and memory_growth_mb > limits.max_memory_mb: errors.append( - f"Memory usage {metrics.memory_mb:.2f}MB exceeds limit {limits.max_memory_mb:.2f}MB" + f"Memory growth {memory_growth_mb:.2f}MB exceeds limit {limits.max_memory_mb:.2f}MB" ) - elif limits.max_memory_mb and metrics.memory_mb > limits.max_memory_mb * limits.warning_threshold: + elif limits.max_memory_mb and memory_growth_mb > limits.max_memory_mb * limits.warning_threshold: warnings.append( - f"Memory usage {metrics.memory_mb:.2f}MB approaching limit {limits.max_memory_mb:.2f}MB" + f"Memory growth {memory_growth_mb:.2f}MB approaching limit {limits.max_memory_mb:.2f}MB" ) # Check CPU diff --git a/src/plugin_system/saved_repositories.py b/src/plugin_system/saved_repositories.py index c8da3c5b..64794d8c 100644 --- a/src/plugin_system/saved_repositories.py +++ b/src/plugin_system/saved_repositories.py @@ -10,6 +10,8 @@ import os from pathlib import Path from typing import List, Dict, Optional +from src.plugin_system.repo_urls import normalize_repo_url + class SavedRepositoriesManager: """Manages saved GitHub repository URLs.""" @@ -71,18 +73,6 @@ class SavedRepositoriesManager: pass return False - @staticmethod - def _clean_url(repo_url: str) -> str: - """Normalize a repo URL: strip whitespace, trailing slashes, and a - trailing ``.git`` suffix ONLY. (The old ``.replace('.git', '')`` - was an unanchored substring replace that mangled URLs merely - containing ``.git``, e.g. ``https://github.com/user/my.github.io``.) - """ - repo_url = repo_url.strip().rstrip('/') - if repo_url.endswith('.git'): - repo_url = repo_url[:-4] - return repo_url - def get_all(self) -> List[Dict[str, str]]: """Get all saved repositories.""" return self.repositories.copy() @@ -98,7 +88,7 @@ class SavedRepositoriesManager: Returns: True if added successfully """ - repo_url = self._clean_url(repo_url) + repo_url = normalize_repo_url(repo_url) # Check if already exists for repo in self.repositories: @@ -138,7 +128,7 @@ class SavedRepositoriesManager: Returns: True if removed successfully """ - repo_url = self._clean_url(repo_url) + repo_url = normalize_repo_url(repo_url) previous = self.repositories remaining = [r for r in previous if r.get('url') != repo_url] @@ -156,7 +146,7 @@ class SavedRepositoriesManager: def has(self, repo_url: str) -> bool: """Check if a repository is already saved.""" - repo_url = self._clean_url(repo_url) + repo_url = normalize_repo_url(repo_url) return any(r.get('url') == repo_url for r in self.repositories) def get_registry_repositories(self) -> List[Dict[str, str]]: diff --git a/src/plugin_system/schema_manager.py b/src/plugin_system/schema_manager.py index 0042f75e..8b267e7c 100644 --- a/src/plugin_system/schema_manager.py +++ b/src/plugin_system/schema_manager.py @@ -14,6 +14,7 @@ import jsonschema from jsonschema import Draft7Validator, ValidationError from src.core_config_keys import CORE_CONFIG_KEYS +from src.element_style import expand_style_elements def _renders_as_object(prop: Dict[str, Any]) -> bool: @@ -452,11 +453,7 @@ class SchemaManager: # full per-element style blocks (font/size/color + layout # offsets) the web-UI config form renders. No-op for schemas # without the declaration; never raises. - try: - from src.element_style import expand_style_elements - schema = expand_style_elements(schema) - except ImportError: - pass + schema = expand_style_elements(schema) # Cache the schema self._schema_cache[plugin_id] = schema @@ -643,20 +640,12 @@ class SchemaManager: [name for name in CORE_PLUGIN_PROPERTIES if name not in declared] ) - # Create validator with enhanced schema + # iter_errors reports every violation, including one ``required`` + # error per missing field at every depth. validator = Draft7Validator(enhanced_schema) - - # Collect all validation errors for error in validator.iter_errors(config): - error_msg = self._format_validation_error(error, plugin_id) - errors.append(error_msg) - - # Check required fields - required_fields = enhanced_schema.get('required', []) - for field in required_fields: - if field not in config: - errors.append(f"Missing required field: '{field}'") - + errors.append(self._format_validation_error(error, plugin_id)) + if errors: return False, errors @@ -687,7 +676,15 @@ class SchemaManager: field_path = f"'{path}'" if path else "root" if error.validator == 'required': - missing = error.validator_value + # validator_value is the schema's whole ``required`` list; the + # error itself is about one field, which jsonschema names only in + # its message ("'api_key' is a required property"). + missing = next( + (name for name in error.validator_value + if error.message.startswith(f"{name!r} ")), + None) + if missing is None: + return f"Field {field_path}: {error.message}" return f"Field {field_path}: Missing required property '{missing}'" elif error.validator == 'type': expected = error.validator_value diff --git a/src/plugin_system/state_manager.py b/src/plugin_system/state_manager.py index 16151f74..da8ea449 100644 --- a/src/plugin_system/state_manager.py +++ b/src/plugin_system/state_manager.py @@ -34,7 +34,9 @@ class PluginState: version: Optional[str] = None installed_at: Optional[datetime] = None last_updated: Optional[datetime] = None - config_version: int = 1 # For detecting state corruption + # Bumped on every update_plugin_state(). Nothing reads it; it stays so + # plugin_state.json keeps the shape older releases load with cls(**data). + config_version: int = 1 metadata: Dict[str, Any] = None def __post_init__(self): @@ -100,6 +102,8 @@ class PluginStateManager: # State storage self._states: Dict[str, PluginState] = {} + # The file's top-level "version", written back as read. Nothing + # checks it yet; it is there for a future format change to branch on. self._state_version = 1 # Threading @@ -193,7 +197,6 @@ class PluginStateManager: current_state.metadata = {} current_state.metadata.update(updates['metadata']) - # Increment config version current_state.config_version += 1 # Store updated state diff --git a/src/plugin_system/store_manager.py b/src/plugin_system/store_manager.py index 5e543b1e..1a2a2168 100644 --- a/src/plugin_system/store_manager.py +++ b/src/plugin_system/store_manager.py @@ -5,6 +5,7 @@ Handles plugin discovery, installation, updates, and uninstallation from both the official registry and custom GitHub repositories. """ +import errno import os import re import json @@ -22,21 +23,19 @@ from pathlib import Path from typing import List, Dict, Optional, Any, Tuple, Set import logging -from urllib.parse import urlparse +from jsonschema import Draft7Validator, ValidationError -from src.common.permission_utils import sudo_remove_directory, install_requirements_file -from src.plugin_system.plugin_loader import ( - requirements_has_real_deps, requirements_are_satisfied, find_trusted_subdir +from src.common.permission_utils import ( + ensure_directory_permissions, get_plugin_dir_mode, install_requirements_file, + sudo_remove_directory, ) +from src.plugin_system.plugin_loader import contained_plugin_dir, requirements_to_install from src.plugin_system.plugin_dirs import ( BACKUP_MARKER, PluginDirectoryIndex, resolve_plugin_dir, store_search_dirs, ) - -try: - from jsonschema import Draft7Validator, ValidationError - JSONSCHEMA_AVAILABLE = True -except ImportError: - JSONSCHEMA_AVAILABLE = False +from src.plugin_system.repo_urls import ( + USER_AGENT, github_api_headers, github_owner_repo, normalize_repo_url, same_repo, +) class PluginStoreManager: @@ -93,11 +92,11 @@ class PluginStoreManager: self.registry_cache_timeout = 900 self.commit_info_cache = {} # Cache for latest commit info: {key: (timestamp, data)} # 30 minutes for commit/manifest caches. Plugin Store users browse - # the catalog via /plugins/store/list which fetches commit info and - # manifest data per plugin. 5-min TTLs meant every fresh browse on - # a Pi4 paid for ~3 HTTP requests x N plugins (30-60s serial). 30 - # minutes keeps the cache warm across a realistic session while - # still picking up upstream updates within a reasonable window. + # the catalog via /plugins/store/list, which fetches commit info per + # plugin; with a 5-minute TTL nearly every browse on a Pi4 paid for + # an HTTP request per plugin again. 30 minutes keeps the cache warm + # across a realistic session while still picking up upstream updates + # within a reasonable window. self.commit_cache_timeout = 1800 self.manifest_cache = {} # Cache for GitHub manifest fetches: {key: (timestamp, data)} self.manifest_cache_timeout = 1800 @@ -127,9 +126,9 @@ class PluginStoreManager: # where ``signature`` is a tuple of (head_mtime, resolved_ref_mtime, # head_contents) so a fast-forward update to the current branch # (which touches .git/refs/heads/ but NOT .git/HEAD) still - # invalidates the cache. Before this cache, every - # /plugins/installed request fired 4 git subprocesses per plugin, - # which pegged the CPU on a Pi4 with a dozen plugins. The cached + # invalidates the cache. Without it every /plugins/installed request + # runs a git subprocess per plugin, which adds up on a Pi4 with a + # dozen plugins. The cached # ``data`` dict is the same shape returned by ``_get_local_git_info`` # itself (sha / short_sha / branch / optional remote_url, date_iso, # date) — all string-keyed strings. @@ -368,13 +367,7 @@ class PluginStoreManager: # Validate token by making a lightweight API call to /user endpoint try: api_url = "https://api.github.com/user" - headers = { - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'LEDMatrix-Plugin-Manager/1.0', - 'Authorization': f'token {token}' - } - - response = requests.get(api_url, headers=headers, timeout=5) + response = requests.get(api_url, headers=github_api_headers(token), timeout=5) if response.status_code == 200: # Token is valid @@ -502,9 +495,6 @@ class PluginStoreManager: Returns: List of validation error messages (empty if valid or schema unavailable) """ - if not JSONSCHEMA_AVAILABLE: - return [] - try: # Load manifest schema schema_path = Path(__file__).parent.parent.parent / "schema" / "manifest_schema.json" @@ -535,134 +525,107 @@ class PluginStoreManager: self.logger.debug(f"Error validating manifest schema for {plugin_id}: {e}") return [] + _EMPTY_REPO_INFO: Dict[str, Any] = { + 'stars': 0, + 'forks': 0, + 'open_issues': 0, + 'updated_at_iso': '', + 'last_commit_iso': '', + 'last_commit_date': '', + 'language': '', + 'license': '', + 'default_branch': 'main', + } + def _get_github_repo_info(self, repo_url: str) -> Dict[str, Any]: - """Fetch GitHub repository information (stars, etc.)""" - # Extract owner/repo from URL + """GitHub metadata for a repository (stars, default branch, last push). + + Returns zeroed defaults (``_EMPTY_REPO_INFO``) for a non-GitHub URL or + when GitHub cannot be asked and nothing is cached. + """ try: - # Handle different URL formats - _parsed_url = urlparse(repo_url) - if _parsed_url.hostname in ('github.com', 'www.github.com'): - parts = repo_url.strip('/').split('/') - if len(parts) >= 2: - owner = parts[-2] - repo = parts[-1] - if repo.endswith('.git'): - repo = repo[:-4] + owner_repo = github_owner_repo(repo_url) + if owner_repo is None: + return dict(self._EMPTY_REPO_INFO) + owner, repo = owner_repo + cache_key = f"{owner}/{repo}" - cache_key = f"{owner}/{repo}" + if cache_key in self.github_cache: + cached_time, cached_data = self.github_cache[cache_key] + if time.time() - cached_time < self.cache_timeout: + return cached_data - # Check cache first - if cache_key in self.github_cache: - cached_time, cached_data = self.github_cache[cache_key] - if time.time() - cached_time < self.cache_timeout: - return cached_data + api_url = f"https://api.github.com/repos/{owner}/{repo}" + try: + response = requests.get( + api_url, headers=github_api_headers(self.github_token), timeout=10) + except requests.RequestException as req_err: + # Network error: prefer a stale cache hit over an empty + # default so the UI keeps working on a flaky Pi WiFi link. + # Bump the cached entry's timestamp into a short backoff + # window so subsequent requests serve the stale payload + # cheaply instead of re-hitting the network on every request. + if cache_key in self.github_cache: + _, stale = self.github_cache[cache_key] + self._record_cache_backoff(self.github_cache, cache_key, self.cache_timeout, stale) + self.logger.warning( + "GitHub repo info fetch failed for %s (%s); serving stale cache.", + cache_key, req_err, + ) + return stale + raise - # Fetch from GitHub API - api_url = f"https://api.github.com/repos/{owner}/{repo}" - headers = { - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'LEDMatrix-Plugin-Manager/1.0' - } - - # Add authentication if token is available - if self.github_token: - headers['Authorization'] = f'token {self.github_token}' + if response.status_code == 200: + data = response.json() + pushed_at = data.get('pushed_at', '') or data.get('updated_at', '') + repo_info = { + 'stars': data.get('stargazers_count', 0), + 'forks': data.get('forks_count', 0), + 'open_issues': data.get('open_issues_count', 0), + 'updated_at_iso': data.get('updated_at', ''), + 'last_commit_iso': pushed_at, + 'last_commit_date': self._iso_to_date(pushed_at), + 'language': data.get('language', ''), + 'license': data.get('license', {}).get('name', '') if data.get('license') else '', + 'default_branch': data.get('default_branch', 'main') + } + self.github_cache[cache_key] = (time.time(), repo_info) + return repo_info - try: - response = requests.get(api_url, headers=headers, timeout=10) - except requests.RequestException as req_err: - # Network error: prefer a stale cache hit over an - # empty default so the UI keeps working on a flaky - # Pi WiFi link. Bump the cached entry's timestamp - # into a short backoff window so subsequent - # requests serve the stale payload cheaply instead - # of re-hitting the network on every request. - if cache_key in self.github_cache: - _, stale = self.github_cache[cache_key] - self._record_cache_backoff(self.github_cache, cache_key, self.cache_timeout, stale) - self.logger.warning( - "GitHub repo info fetch failed for %s (%s); serving stale cache.", - cache_key, req_err, - ) - return stale - raise + if response.status_code == 403: + # Rate limit or authentication issue. A stale star count is + # better than a reset to zero, and the backoff bump stops the + # store hammering the API while rate-limited. + if cache_key in self.github_cache: + _, stale = self.github_cache[cache_key] + self._record_cache_backoff(self.github_cache, cache_key, self.cache_timeout, stale) + self.logger.warning( + "GitHub API 403 for %s; serving stale cache.", cache_key, + ) + return stale + if not self.github_token: + self.logger.warning( + "GitHub API rate limit likely exceeded (403). " + "Add a GitHub personal access token to config/config_secrets.json " + "under 'github.api_token' to increase rate limits from 60 to 5000/hour." + ) + else: + self.logger.warning( + f"GitHub API request failed: 403 for {api_url}. " + f"Your token may have insufficient permissions or rate limit exceeded." + ) + else: + self.logger.warning(f"GitHub API request failed: {response.status_code} for {api_url}") + if cache_key in self.github_cache: + _, stale = self.github_cache[cache_key] + self._record_cache_backoff(self.github_cache, cache_key, self.cache_timeout, stale) + return stale - if response.status_code == 200: - data = response.json() - pushed_at = data.get('pushed_at', '') or data.get('updated_at', '') - repo_info = { - 'stars': data.get('stargazers_count', 0), - 'forks': data.get('forks_count', 0), - 'open_issues': data.get('open_issues_count', 0), - 'updated_at_iso': data.get('updated_at', ''), - 'last_commit_iso': pushed_at, - 'last_commit_date': self._iso_to_date(pushed_at), - 'language': data.get('language', ''), - 'license': data.get('license', {}).get('name', '') if data.get('license') else '', - 'default_branch': data.get('default_branch', 'main') - } - - # Cache the result - self.github_cache[cache_key] = (time.time(), repo_info) - return repo_info - elif response.status_code == 403: - # Rate limit or authentication issue. If we have a - # previously-cached value, serve it rather than - # returning empty defaults — a stale star count is - # better than a reset to zero. Apply the same - # failure-backoff bump as the network-error path - # so we don't hammer the API with repeat requests - # while rate-limited. - if cache_key in self.github_cache: - _, stale = self.github_cache[cache_key] - self._record_cache_backoff(self.github_cache, cache_key, self.cache_timeout, stale) - self.logger.warning( - "GitHub API 403 for %s; serving stale cache.", cache_key, - ) - return stale - if not self.github_token: - self.logger.warning( - "GitHub API rate limit likely exceeded (403). " - "Add a GitHub personal access token to config/config_secrets.json " - "under 'github.api_token' to increase rate limits from 60 to 5000/hour." - ) - else: - self.logger.warning( - f"GitHub API request failed: 403 for {api_url}. " - f"Your token may have insufficient permissions or rate limit exceeded." - ) - else: - self.logger.warning(f"GitHub API request failed: {response.status_code} for {api_url}") - if cache_key in self.github_cache: - _, stale = self.github_cache[cache_key] - self._record_cache_backoff(self.github_cache, cache_key, self.cache_timeout, stale) - return stale - - return { - 'stars': 0, - 'forks': 0, - 'open_issues': 0, - 'updated_at_iso': '', - 'last_commit_iso': '', - 'last_commit_date': '', - 'language': '', - 'license': '', - 'default_branch': 'main' - } + return dict(self._EMPTY_REPO_INFO) except Exception as e: self.logger.error(f"Error fetching GitHub repo info for {repo_url}: {e}") - return { - 'stars': 0, - 'forks': 0, - 'open_issues': 0, - 'updated_at_iso': '', - 'last_commit_iso': '', - 'last_commit_date': '', - 'language': '', - 'license': '', - 'default_branch': 'main' - } + return dict(self._EMPTY_REPO_INFO) def _http_get_with_retries(self, url: str, *, timeout: int = 10, stream: bool = False, headers: Dict[str, str] = None, max_retries: int = 3, backoff_sec: float = 0.75): """ @@ -697,27 +660,17 @@ class PluginStoreManager: Registry dict with plugins list, or None if not found/invalid """ try: - # Clean up URL - repo_url = repo_url.rstrip('/').replace('.git', '') - - # Try to find plugins.json in common locations - # First try root directory + repo_url = normalize_repo_url(repo_url) + + # plugins.json or registry.json at the root of main, then master. registry_urls = [] + owner_repo = github_owner_repo(repo_url) + if owner_repo is not None: + owner, repo = owner_repo + for branch in ['main', 'master']: + registry_urls.append(f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/plugins.json") + registry_urls.append(f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/registry.json") - # Extract owner/repo from URL - _parsed_repo_url = urlparse(repo_url) - if _parsed_repo_url.hostname in ('github.com', 'www.github.com'): - parts = repo_url.split('/') - if len(parts) >= 2: - owner = parts[-2] - repo = parts[-1] - - # Try common branch names - for branch in ['main', 'master']: - registry_urls.append(f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/plugins.json") - registry_urls.append(f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/registry.json") - - # Try each URL for url in registry_urls: try: response = self._http_get_with_retries(url, timeout=10) @@ -815,15 +768,20 @@ class PluginStoreManager: """ Search for plugins in the registry with enhanced metadata. - GitHub is now treated as the source of truth for live metadata like - stars and last commit timestamps. The registry provides descriptive - information (name, description, repo URL, etc.). + GitHub supplies live metadata such as stars and last commit + timestamps; the registry supplies descriptive information (name, + description, repo URL, etc.). Args: - query: Search query string (searches name, description, id) + query: Search query string (searches name, description, id, author) category: Filter by category (e.g., 'sports', 'weather', 'time') tags: Filter by tags (matches any tag in list) fetch_commit_info: If True (default), fetch commit metadata from GitHub. + include_saved_repos: If True (default), also search the + registry-style repositories the user saved. + saved_repositories_manager: The SavedRepositoriesManager holding + those repositories; without it only the official registry is + searched. Returns: List of matching plugin metadata enriched with GitHub information @@ -877,11 +835,11 @@ class PluginStoreManager: def _enrich(plugin: Dict) -> Dict: """Enrich a single plugin with GitHub metadata. - Called concurrently from a ThreadPoolExecutor. Each underlying - HTTP helper (``_get_github_repo_info`` / ``_get_latest_commit_info`` - / ``_fetch_manifest_from_github``) is thread-safe — they use - ``requests`` and write their own cache keys on Python dicts, - which is atomic under the GIL for single-key assignments. + Called concurrently from a ThreadPoolExecutor. Both HTTP helpers + (``_get_github_repo_info`` / ``_get_latest_commit_info``) are + thread-safe -- they use ``requests`` and write their own cache + keys on Python dicts, which is atomic under the GIL for + single-key assignments. """ enhanced_plugin = plugin.copy() repo_url = plugin.get('repo', '') @@ -912,24 +870,21 @@ class PluginStoreManager: # The registry's plugins.json already carries ``description`` # (it is generated from each plugin's manifest by # ``update_registry.py``), and ``last_updated`` is filled in - # from the commit info above. An earlier implementation - # fetched manifest.json per plugin anyway, which meant one - # extra HTTPS round trip per result; on a Pi4 with a flaky - # WiFi link the tail retries of that one extra call + # from the commit info above. Fetching manifest.json per + # plugin costs one extra HTTPS round trip per result; on a Pi4 + # with a flaky WiFi link the tail retries of that one call # (_http_get_with_retries does 3 attempts with exponential - # backoff) dominated wall time even after parallelization. + # backoff) dominate wall time even with the thread pool. return enhanced_plugin - # Fan out the per-plugin GitHub enrichment. The previous - # implementation did this serially, which on a Pi4 with ~15 plugins - # and a fresh cache meant 30+ HTTP requests in strict sequence (the - # "connecting to display" hang reported by users). With a thread + # Fan out the per-plugin GitHub enrichment. Serially, a Pi4 with ~15 + # plugins and a cold cache makes 30+ HTTP requests in strict sequence + # (the "connecting to display" hang users reported). With a thread # pool, latency is dominated by the slowest request rather than # their sum. Workers capped at 10 to stay well under the # unauthenticated GitHub rate limit burst and avoid overwhelming a - # Pi's WiFi link. For a small number of plugins the pool is - # essentially free. + # Pi's WiFi link. if not filtered: return [] @@ -959,46 +914,34 @@ class PluginStoreManager: Manifest data or None if not found """ try: - # Convert repo URL to raw content URL - # https://github.com/user/repo -> https://raw.githubusercontent.com/user/repo/branch/manifest.json - _parsed_manifest_url = urlparse(repo_url) - if _parsed_manifest_url.hostname in ('github.com', 'www.github.com'): - # Handle different URL formats - repo_url = repo_url.rstrip('/') - if repo_url.endswith('.git'): - repo_url = repo_url[:-4] + owner_repo = github_owner_repo(repo_url) + if owner_repo is None: + return None + owner, repo = owner_repo - parts = repo_url.split('/') - if len(parts) >= 2: - owner = parts[-2] - repo = parts[-1] + cache_key = f"{owner}/{repo}:{branch}:{manifest_path}" + if not force_refresh and cache_key in self.manifest_cache: + cached_time, cached_data = self.manifest_cache[cache_key] + if time.time() - cached_time < self.manifest_cache_timeout: + return cached_data - # Check cache first - cache_key = f"{owner}/{repo}:{branch}:{manifest_path}" - if not force_refresh and cache_key in self.manifest_cache: - cached_time, cached_data = self.manifest_cache[cache_key] - if time.time() - cached_time < self.manifest_cache_timeout: - return cached_data + raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{manifest_path}" + response = self._http_get_with_retries(raw_url, timeout=10) + if response.status_code == 200: + result = response.json() + self.manifest_cache[cache_key] = (time.time(), result) + return result + if response.status_code == 404 and branch != "main": + raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/main/{manifest_path}" + response = self._http_get_with_retries(raw_url, timeout=10) + if response.status_code == 200: + result = response.json() + self.manifest_cache[cache_key] = (time.time(), result) + return result - raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{manifest_path}" - - response = self._http_get_with_retries(raw_url, timeout=10) - if response.status_code == 200: - result = response.json() - self.manifest_cache[cache_key] = (time.time(), result) - return result - elif response.status_code == 404: - # Try main branch instead - if branch != "main": - raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/main/{manifest_path}" - response = self._http_get_with_retries(raw_url, timeout=10) - if response.status_code == 200: - result = response.json() - self.manifest_cache[cache_key] = (time.time(), result) - return result - - # Cache negative result - self.manifest_cache[cache_key] = (time.time(), None) + # Cache the miss too, so a plugin without a manifest at this path + # is not re-fetched on every browse. + self.manifest_cache[cache_key] = (time.time(), None) except Exception as e: self.logger.debug(f"Could not fetch manifest from GitHub for {repo_url}: {e}") @@ -1007,21 +950,11 @@ class PluginStoreManager: def _get_latest_commit_info(self, repo_url: str, branch: str = "main", force_refresh: bool = False) -> Optional[Dict[str, Any]]: """Return metadata about the latest commit on the given branch.""" try: - if 'github.com' not in repo_url: + owner_repo = github_owner_repo(repo_url) + if owner_repo is None: return None + owner, repo = owner_repo - repo_url = repo_url.rstrip('/') - if repo_url.endswith('.git'): - repo_url = repo_url[:-4] - - parts = repo_url.split('/') - if len(parts) < 2: - return None - - owner = parts[-2] - repo = parts[-1] - - # Check cache first cache_key = f"{owner}/{repo}:{branch}" if not force_refresh and cache_key in self.commit_info_cache: cached_time, cached_data = self.commit_info_cache[cache_key] @@ -1029,14 +962,7 @@ class PluginStoreManager: return cached_data branches_to_try = self._distinct_sequence([branch, 'main', 'master']) - - headers = { - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'LEDMatrix-Plugin-Manager/1.0' - } - - if self.github_token: - headers['Authorization'] = f'token {self.github_token}' + headers = github_api_headers(self.github_token) last_error = None for branch_name in branches_to_try: @@ -1254,46 +1180,57 @@ class PluginStoreManager: backup_path = plugin_path.with_name( f"{plugin_path.name}{BACKUP_MARKER}preinstall") - if backup_path.exists() and not self._safe_remove_directory(backup_path): - # Can't stage a safety net. Better to attempt the install than - # to refuse outright, which is what callers got before this - # existed. + problem = self._set_aside(plugin_path, backup_path) + if problem: + # Can't stage a safety net. Attempting the install anyway is + # what callers got before the net existed; refusing would be + # a new failure mode for a direct install. self.logger.warning( - "Could not clear stale pre-install backup for %s at %s; " - "installing without a rollback net", plugin_id, backup_path) - return self._install_plugin_impl(plugin_id, branch) - - try: - plugin_path.rename(backup_path) - except OSError as e: - self.logger.warning( - "Could not set aside existing install of %s (%s); " - "installing without a rollback net", plugin_id, e) + "Installing %s without a rollback net: %s", plugin_id, problem) return self._install_plugin_impl(plugin_id, branch) try: installed = self._install_plugin_impl(plugin_id, branch) except Exception: - self._restore_preinstall_backup(plugin_id, plugin_path, backup_path) + self._restore_backup(plugin_id, plugin_path, backup_path, "Install") raise if installed: - if not self._safe_remove_directory(backup_path): - self.logger.warning( - "Install of %s succeeded but the previous copy at %s " - "could not be removed; it will be cleared on the next " - "install", plugin_id, backup_path) + self._discard_backup(plugin_id, backup_path, "install") return True - self._restore_preinstall_backup(plugin_id, plugin_path, backup_path) + self._restore_backup(plugin_id, plugin_path, backup_path, "Install") return False - def _restore_preinstall_backup( - self, plugin_id: str, plugin_path: Path, backup_path: Path + def _set_aside(self, plugin_path: Path, backup_path: Path) -> Optional[str]: + """Rename an installed plugin to ``backup_path`` so a failed + (re)install can put it back. + + A stale backup left by a crash is cleared first, since it would block + the rename. Returns None on success, otherwise why it could not. + """ + if backup_path.exists() and not self._safe_remove_directory(backup_path): + return f"could not clear stale backup at {backup_path}" + try: + plugin_path.rename(backup_path) + except OSError as e: + return f"could not set aside {plugin_path}: {e}" + return None + + def _discard_backup(self, plugin_id: str, backup_path: Path, action: str) -> None: + """Remove the set-aside copy after a successful (re)install.""" + if not self._safe_remove_directory(backup_path): + self.logger.warning( + "%s of %s succeeded but the previous copy at %s could not be " + "removed; it will be cleared on the next %s", + action.capitalize(), plugin_id, backup_path, action) + + def _restore_backup( + self, plugin_id: str, plugin_path: Path, backup_path: Path, action: str ) -> None: - """Put the previous install back after a failed (re)install.""" + """Put the set-aside copy back after a failed (re)install.""" self.logger.error( - "Install of %s failed; restoring the previous version", plugin_id) + "%s of %s failed; restoring the previous version", action, plugin_id) try: if plugin_path.exists(): # Partial download debris from the failed install. @@ -1375,8 +1312,7 @@ class PluginStoreManager: return False else: branch_used = self._install_via_git(repo_url, plugin_path, branch_candidates) - if branch_used is None and not plugin_path.exists(): - # Git failed entirely; fall back to zip download + if branch_used is None: self.logger.info("Git not available or clone failed, attempting archive download...") for candidate in branch_candidates: download_url = f"{repo_url}/archive/refs/heads/{candidate}.zip" @@ -1384,7 +1320,7 @@ class PluginStoreManager: branch_used = candidate break - if branch_used is None and not plugin_path.exists(): + if branch_used is None: self.logger.error(f"Failed to install plugin {plugin_id} via git or archive download") return False @@ -1523,8 +1459,7 @@ class PluginStoreManager: branch_info = f" (branch: {branch})" if branch else "" self.logger.info(f"Installing plugin from custom URL: {repo_url}{branch_info}" + (f" (subpath: {plugin_path})" if plugin_path else "")) - # Clean up URL (remove .git suffix if present) - repo_url = repo_url.rstrip('/').replace('.git', '') + repo_url = normalize_repo_url(repo_url) temp_dir = None try: @@ -1549,24 +1484,22 @@ class PluginStoreManager: 'error': f'Failed to download or extract plugin from monorepo subdirectory: {plugin_path}' } else: - # Try git clone for direct plugin repos branch_used = self._install_via_git(repo_url, temp_dir, branch_candidates) - if branch_used: + if branch_used is not None: self.logger.info(f"Cloned via git (branch: {branch_used})") else: - # Git failed; try downloading as zip - branch_used = None + self.logger.info("Git not available or clone failed, attempting archive download...") for candidate in branch_candidates: download_url = f"{repo_url}/archive/refs/heads/{candidate}.zip" if self._install_via_download(download_url, temp_dir): branch_used = candidate break - - if branch_used is None: - return { - 'success': False, - 'error': 'Failed to clone or download repository' - } + + if branch_used is None: + return { + 'success': False, + 'error': 'Failed to clone or download repository' + } # Read manifest to get plugin ID manifest_path = temp_dir / "manifest.json" @@ -1634,8 +1567,10 @@ class PluginStoreManager: json.dump(manifest, f, indent=2) self.logger.info(f"Added missing entry_point field to {plugin_id} manifest (defaulted to manager.py)") - # Move to plugins directory - use manifest ID as source of truth - # This ensures directory name always matches manifest ID + # The directory is named for the caller's plugin_id when one was + # given (update_plugin passes the installed id), else for the + # manifest's id -- so it can differ from the manifest id, which + # discovery tolerates by reading the manifest. final_path = self.plugins_dir / plugin_id if final_path.exists(): self.logger.warning(f"Plugin {plugin_id} already exists, removing existing copy") @@ -1647,9 +1582,7 @@ class PluginStoreManager: shutil.move(str(temp_dir), str(final_path)) temp_dir = None # Prevent cleanup since we moved it - - # Note: plugin_id here is already from manifest (line 749), so directory name matches manifest ID - + # Install dependencies self._install_dependencies(final_path) @@ -1701,7 +1634,6 @@ class PluginStoreManager: Class name if found, None otherwise """ try: - import re with open(manager_file, 'r', encoding='utf-8') as f: content = f.read() @@ -1723,7 +1655,18 @@ class PluginStoreManager: return None def _install_via_git(self, repo_url: str, target_path: Path, branches: Optional[List[str]] = None) -> Optional[str]: - """Clone a repository into ``target_path``. Returns the branch name on success.""" + """Clone a repository into ``target_path``. + + Tries each of ``branches`` (default ``main``, ``master``), then the + repository's own default branch, so a repository whose only branch + is e.g. ``develop`` still installs. + + Returns: + The branch that was cloned, or None when every clone failed and + ``target_path`` has been removed. After a default-branch clone + this is the branch the clone checked out (``'HEAD'`` if the + remote's HEAD is detached), never None. + """ branches_to_try = self._distinct_sequence(branches or []) if not branches_to_try: branches_to_try = ['main', 'master'] @@ -1758,7 +1701,7 @@ class PluginStoreManager: timeout=60 ) self.logger.debug(f"Successfully cloned {repo_url} (git default branch) to {target_path}") - return None # Unknown branch name, git default used + return self._checked_out_branch(target_path) except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as e: last_error = e if target_path.exists(): @@ -1766,7 +1709,20 @@ class PluginStoreManager: self.logger.error(f"Git clone failed for all attempted branches: {last_error}") return None - + + @staticmethod + def _checked_out_branch(checkout: Path) -> str: + """The branch a fresh clone has checked out, read from ``.git/HEAD``. + + ``'HEAD'`` when HEAD is detached or unreadable. + """ + try: + head = (checkout / '.git' / 'HEAD').read_text(encoding='utf-8').strip() + except OSError: + return 'HEAD' + prefix = 'ref: refs/heads/' + return head[len(prefix):] if head.startswith(prefix) else 'HEAD' + def _install_from_monorepo(self, download_url: str, plugin_subpath: str, target_path: Path) -> bool: """ Install a plugin from a monorepo by downloading only the target subdirectory. @@ -1815,14 +1771,6 @@ class PluginStoreManager: pass return None, None - @staticmethod - def _normalize_repo_url(url: str) -> str: - """Normalize a GitHub repo URL for comparison (strip trailing / and .git).""" - url = url.rstrip('/') - if url.endswith('.git'): - url = url[:-4] - return url.lower() - def _install_from_monorepo_api(self, repo_url: str, branch: str, plugin_subpath: str, target_path: Path) -> bool: """ Install a plugin subdirectory using the GitHub Git Trees API. @@ -1841,25 +1789,15 @@ class PluginStoreManager: True if successful, False to trigger ZIP fallback """ try: - # Parse owner/repo from URL - clean_url = repo_url.rstrip('/') - if clean_url.endswith('.git'): - clean_url = clean_url[:-4] - parts = clean_url.split('/') - if len(parts) < 2: + owner_repo = github_owner_repo(repo_url) + if owner_repo is None: return False - owner, repo = parts[-2], parts[-1] + owner, repo = owner_repo # Step 1: Get the recursive tree listing (1 API call) api_url = f"https://api.github.com/repos/{owner}/{repo}/git/trees/{branch}?recursive=true" - headers = { - 'Accept': 'application/vnd.github.v3+json', - 'User-Agent': 'LEDMatrix-Plugin-Manager/1.0' - } - if self.github_token: - headers['Authorization'] = f'token {self.github_token}' - - tree_response = self._http_get_with_retries(api_url, timeout=15, headers=headers) + tree_response = self._http_get_with_retries( + api_url, timeout=15, headers=github_api_headers(self.github_token)) if tree_response.status_code != 200: self.logger.debug(f"Trees API returned {tree_response.status_code} for {owner}/{repo}") return False @@ -1892,10 +1830,6 @@ class PluginStoreManager: self.logger.info(f"Downloading {len(file_entries)} files for {plugin_subpath} via API") # Step 3: Create target directory and download each file - from src.common.permission_utils import ( - ensure_directory_permissions, - get_plugin_dir_mode - ) ensure_directory_permissions(target_path.parent, get_plugin_dir_mode()) target_path.mkdir(parents=True, exist_ok=True) @@ -1991,10 +1925,6 @@ class PluginStoreManager: source_plugin_dir = temp_extract / root_dir / plugin_subpath - from src.common.permission_utils import ( - ensure_directory_permissions, - get_plugin_dir_mode - ) ensure_directory_permissions(target_path.parent, get_plugin_dir_mode()) # Ensure target doesn't exist to prevent shutil.move nesting if target_path.exists(): @@ -2028,7 +1958,7 @@ class PluginStoreManager: try: self.logger.info(f"Downloading from: {download_url}") # Allow redirects (GitHub archive URLs redirect to codeload.github.com) - response = self._http_get_with_retries(download_url, timeout=60, stream=True, headers={'User-Agent': 'LEDMatrix-Plugin-Manager/1.0'}) + response = self._http_get_with_retries(download_url, timeout=60, stream=True, headers={'User-Agent': USER_AGENT}) response.raise_for_status() # Download to temporary file @@ -2065,10 +1995,6 @@ class PluginStoreManager: # Move contents from root_dir to target source_dir = temp_extract / root_dir if source_dir.exists(): - from src.common.permission_utils import ( - ensure_directory_permissions, - get_plugin_dir_mode - ) ensure_directory_permissions(target_path.parent, get_plugin_dir_mode()) shutil.move(str(source_dir), str(target_path)) else: @@ -2094,42 +2020,23 @@ class PluginStoreManager: """ Install Python dependencies from requirements.txt. + ``plugin_path`` is ultimately derived from a plugin-supplied manifest + ``id``, so it is only used after contained_plugin_dir() has rebuilt it + from a listing of ``self.plugins_dir``. + Args: plugin_path: Path to plugin directory Returns: True if successful or no requirements file """ - # Reconstruct the plugin path from the trusted self.plugins_dir base + - # an entry actually enumerated from it, rather than trusting - # plugin_path directly -- callers ultimately derive it from a - # plugin-supplied manifest "id" field (see install_plugin_from_url), - # so without this a malicious manifest could point requirements_file - # outside plugins_dir. find_trusted_subdir()'s return value always - # comes from os.scandir() on the trusted root, so building the path - # from it (not from the caller's string) is a real containment - # guarantee, matching the pattern in PluginLoader.install_dependencies(). - plugin_dir_real = os.path.realpath(str(plugin_path)) - plugins_dir_real = os.path.realpath(str(self.plugins_dir)) - requested_name = os.path.basename(plugin_dir_real) - matched_name = find_trusted_subdir(plugins_dir_real, requested_name) - if matched_name is None: + safe_plugin_dir = contained_plugin_dir(plugin_path, self.plugins_dir) + if safe_plugin_dir is None: self.logger.error("Plugin directory not found inside plugins dir for dependency install") return False - safe_plugin_path = Path(os.path.join(plugins_dir_real, matched_name)) - requirements_file = safe_plugin_path / "requirements.txt" - - if not requirements_file.exists(): - self.logger.debug(f"No requirements.txt found in {plugin_path.name}") - return True - - if not requirements_has_real_deps(str(requirements_file)): - self.logger.debug(f"requirements.txt for {plugin_path.name} has no real dependencies, skipping pip") - return True - - if requirements_are_satisfied(str(requirements_file)): - self.logger.debug(f"Dependencies for {plugin_path.name} already satisfied, skipping pip") + requirements_file = requirements_to_install(safe_plugin_dir, self.logger, plugin_path.name) + if requirements_file is None: return True try: @@ -2141,7 +2048,7 @@ class PluginStoreManager: # ledmatrix.service, so pip reports success while the package # stays invisible to the running plugin (e.g. missing `astral` # for the weather plugin even though "install" succeeded). - result = install_requirements_file(requirements_file, timeout=300) + result = install_requirements_file(Path(requirements_file), timeout=300) if result.returncode != 0: self.logger.error( f"Error installing dependencies for {plugin_path.name}: {result.stderr}" @@ -2153,10 +2060,10 @@ class PluginStoreManager: except subprocess.TimeoutExpired: self.logger.error("Dependency installation timed out") return False - except (BrokenPipeError, OSError) as e: - # Handle broken pipe errors (errno 32) which can occur during pip downloads - # Often caused by network interruptions or output buffer issues - if isinstance(e, OSError) and e.errno == 32: + except OSError as e: + # A broken pipe (EPIPE) happens when pip's output pipe closes + # mid-download, usually a network interruption. + if e.errno == errno.EPIPE: self.logger.error( f"Broken pipe error during dependency installation for {plugin_path.name}. " f"This usually indicates a network interruption or pip output buffer issue. " @@ -2166,7 +2073,6 @@ class PluginStoreManager: self.logger.error(f"OS error during dependency installation: {e}") return False except Exception as e: - # Catch any other unexpected errors self.logger.error(f"Unexpected error installing dependencies for {plugin_path.name}: {e}", exc_info=True) return False @@ -2241,9 +2147,9 @@ class PluginStoreManager: Results are cached keyed on a signature that includes HEAD contents plus the mtime of HEAD AND the resolved ref (or - packed-refs). Repeated calls skip the four ``git`` subprocesses - when nothing has changed, and a ``git pull`` that fast-forwards - the branch correctly invalidates the cache. + packed-refs). Repeated calls skip the ``git log`` subprocess when + nothing has changed, and a ``git pull`` that fast-forwards the + branch correctly invalidates the cache. """ git_dir = plugin_path / '.git' if not git_dir.exists(): @@ -2412,12 +2318,11 @@ class PluginStoreManager: No ``ledmatrix-`` prefix and no case folding here, unlike the loader: a store operation may delete what this returns, so it only accepts a - directory that names the id exactly or declares it. Note that this - leaves registry ids like `stocks` unresolved when the installed - plugin is `ledmatrix-stocks/` declaring `ledmatrix-stocks` (the - monorepo's leaderboard, music, stocks and weather); passing - ``prefix=True`` would resolve them, but update_plugin()'s reinstall - path has not been checked against that yet. + directory that names the id exactly or declares it. So a registry id + such as `stocks` does not resolve to an installed `ledmatrix-stocks/` + declaring `ledmatrix-stocks` (the monorepo's leaderboard, music, + stocks and weather); callers pass the installed id, and + update_plugin() maps it back to the registry id itself. Args: plugin_id: Plugin identifier @@ -2545,11 +2450,9 @@ class PluginStoreManager: The old install is renamed aside (not deleted) until the new install succeeds, then removed; on ANY install failure the old directory is - restored. This is the difference between a failed update and a - destroyed plugin: the previous delete-then-install flow permanently - removed plugins whenever the download failed mid-update (seen in the - field during the monorepo migration on a Pi with broken DNS — every - old-remote plugin was deleted and none could be re-downloaded). + restored. Deleting first turns a failed download into a destroyed + plugin: during the monorepo migration a Pi with broken DNS lost every + old-remote plugin that way, with none able to be re-downloaded. The aside name embeds BACKUP_MARKER ('.standalone-backup-') so every plugin directory lookup (src/plugin_system/plugin_dirs.py) ignores it @@ -2564,18 +2467,11 @@ class PluginStoreManager: with self._get_reinstall_lock(plugin_id): backup_path = plugin_path.with_name( f"{plugin_path.name}{BACKUP_MARKER}migrating") - # A stale aside from a previous crash would block the rename - if backup_path.exists(): - if not self._safe_remove_directory(backup_path): - self.logger.error( - f"Could not clear stale backup for {plugin_id} at " - f"{backup_path}; leaving old install in place") - return False - try: - plugin_path.rename(backup_path) - except OSError as e: + problem = self._set_aside(plugin_path, backup_path) + if problem: self.logger.error( - f"Could not set aside old plugin directory for {plugin_id}: {e}") + "Not updating %s: %s; the installed version is left in place", + plugin_id, problem) return False try: @@ -2585,27 +2481,11 @@ class PluginStoreManager: installed = False if installed: - if not self._safe_remove_directory(backup_path): - self.logger.warning( - f"Update of {plugin_id} succeeded but the old backup " - f"at {backup_path} could not be removed; it will be " - f"cleared on the next update") + self._discard_backup(plugin_id, backup_path, "update") return True - # Install failed (bad network, registry error...) — put the old - # version back so the user still has a working plugin. - self.logger.error( - f"Reinstall of {plugin_id} failed; restoring previous version") - try: - if plugin_path.exists(): - # partial download debris from the failed install - self._safe_remove_directory(plugin_path) - backup_path.rename(plugin_path) - self.logger.info(f"Restored previous install of {plugin_id}") - except OSError as e: - self.logger.error( - f"CRITICAL: could not restore {plugin_id} from {backup_path}: {e}. " - f"The previous install is preserved there — rename it back manually.") + # Bad network, registry error...: the user keeps a working plugin. + self._restore_backup(plugin_id, plugin_path, backup_path, "Reinstall") return False def update_plugin(self, plugin_id: str) -> bool: @@ -2665,7 +2545,7 @@ class PluginStoreManager: # while the registry now points to the monorepo. Detect this and reinstall. registry_repo = plugin_info_remote.get('repo', '') local_remote = git_info.get('remote_url', '') - if local_remote and registry_repo and self._normalize_repo_url(local_remote) != self._normalize_repo_url(registry_repo): + if local_remote and registry_repo and not same_repo(local_remote, registry_repo): self.logger.info( f"Plugin {resolved_id} git remote ({local_remote}) differs from registry ({registry_repo}). " f"Reinstalling from registry to migrate to new source." @@ -2814,7 +2694,6 @@ class PluginStoreManager: # If status check times out, assume there might be changes and proceed self.logger.warning(f"Git status check timed out for {plugin_id}, proceeding with update") has_changes = True - status_result = type('obj', (object,), {'stdout': '', 'stderr': 'Status check timed out'})() stash_info = "" # Whether the pull can be undone without destroying work. @@ -2898,7 +2777,7 @@ class PluginStoreManager: except subprocess.CalledProcessError as git_error: error_output = git_error.stderr or git_error.stdout or "Unknown error" - cmd_str = ' '.join(git_error.cmd) if hasattr(git_error, 'cmd') else 'unknown' + cmd_str = ' '.join(git_error.cmd) self.logger.error(f"Git update failed for {plugin_id}") self.logger.error(f"Command: {cmd_str}") self.logger.error(f"Return code: {git_error.returncode}") @@ -2914,7 +2793,7 @@ class PluginStoreManager: self.logger.error(f"Authentication failed for {plugin_id}. Check git credentials or repository permissions.") elif "not found" in error_lower or "does not exist" in error_lower: self.logger.error(f"Remote branch or repository not found for {plugin_id}. Check repository URL and branch name.") - elif "merge conflict" in error_lower or "conflict" in error_lower: + elif "conflict" in error_lower: self.logger.error(f"Merge conflict detected for {plugin_id}. Resolve conflicts manually or reinstall plugin.") return False @@ -2922,24 +2801,28 @@ class PluginStoreManager: self.logger.warning(f"Git update timed out for {plugin_id}") return False - # Not a git repository - try to get repo URL from git config if it exists - # (in case .git directory was removed but remote URL is still in config) + # A plugin with its own .git that _get_local_git_info could not + # read (e.g. no commits yet) may still name a remote to reinstall + # from. Without its own .git, `git -C ` walks up and finds + # the enclosing LEDMatrix checkout when plugins live in + # plugin-repos/ -- `--local` does not prevent that -- and the + # "plugin's" remote would be LEDMatrix itself. repo_url = None - try: - # Use --local to avoid inheriting the parent LEDMatrix repo's git config - # when the plugin directory lives inside the main repo (e.g. plugin-repos/). - remote_url_result = subprocess.run( - ['git', '-C', str(plugin_path), 'config', '--local', '--get', 'remote.origin.url'], - capture_output=True, - text=True, - timeout=10, - check=False - ) - if remote_url_result.returncode == 0: - repo_url = remote_url_result.stdout.strip() - self.logger.info(f"Found git remote URL for {plugin_id}: {repo_url}") - except Exception as e: - self.logger.debug(f"Could not get git remote URL: {e}") + if (plugin_path / '.git').exists(): + try: + remote_url_result = subprocess.run( + ['git', '-C', str(plugin_path), 'config', '--local', '--get', 'remote.origin.url'], + capture_output=True, + text=True, + timeout=10, + check=False + ) + if remote_url_result.returncode == 0: + repo_url = remote_url_result.stdout.strip() or None + if repo_url: + self.logger.info(f"Found git remote URL for {plugin_id}: {repo_url}") + except (OSError, subprocess.SubprocessError) as e: + self.logger.debug(f"Could not get git remote URL: {e}") # Try registry-based update self.logger.info(f"Plugin {plugin_id} is not a git repository, checking registry...") @@ -3027,9 +2910,7 @@ class PluginStoreManager: return self._reinstall_with_rollback(registry_id, plugin_path) except Exception as e: - import traceback - self.logger.error(f"Error updating plugin {plugin_id}: {e}") - self.logger.debug(traceback.format_exc()) + self.logger.error(f"Error updating plugin {plugin_id}: {e}", exc_info=True) return False def list_installed_plugins(self) -> List[str]: diff --git a/src/startup_validator.py b/src/startup_validator.py index 954c751f..14c4c227 100644 --- a/src/startup_validator.py +++ b/src/startup_validator.py @@ -1,8 +1,14 @@ """ Startup Validator -Validates system configuration, plugins, and dependencies on startup. -Fails fast with clear error messages to prevent runtime issues. +Checks configuration, the cache directory, plugins and the installed systemd +units when the display service starts, and reports what it finds. + +validate_all() never raises: it returns (is_valid, errors, warnings) and +DisplayController logs them. Startup continues either way, so a problem found +here shows up in the log rather than stopping the display. raise_on_errors() +turns the errors into exceptions for a caller that does want to stop; the +display service does not call it. """ import os @@ -180,17 +186,16 @@ class StartupValidator: try: config = self.config_manager.load_config() - # Check for required top-level keys required_keys = ['display', 'timezone'] for key in required_keys: if key not in config: self.errors.append(f"Missing required configuration key: {key}") - - # Validate display configuration - display_config = config.get('display', {}) - if not display_config: - self.errors.append("Display configuration is missing or empty") - + + # A missing display section is reported once, above, and an empty + # one here; _validate_display_config leaves both to this method. + if 'display' in config and not config['display']: + self.errors.append("Display configuration is empty") + except ConfigError as e: self.errors.append(f"Configuration error: {e}") except Exception as e: @@ -247,8 +252,7 @@ class StartupValidator: display_config = config.get('display', {}) if not display_config: - self.errors.append("Display configuration is missing") - return + return # reported by _validate_config hardware_config = display_config.get('hardware', {}) if not hardware_config: diff --git a/src/web_interface/config_arrays.py b/src/web_interface/config_arrays.py new file mode 100644 index 00000000..9dd5c32e --- /dev/null +++ b/src/web_interface/config_arrays.py @@ -0,0 +1,60 @@ +"""Putting a submitted plugin config's lists back into list shape. + +A list reaches a plugin-config save keyed by position more often than as a +list. The settings form posts one field per element (``feeds.custom_feeds.0.name``), +which ``_set_nested_value`` stores as ``{"0": {"name": ...}}``, and the JSON +path's dotToNested() in the browser builds the same dict. Validation expects +an array there, so the save converts them first. +""" +from typing import Any, Dict + + +def _is_index_dict(value: Any) -> bool: + """True for a dict keyed only by list positions ("0", "1", ...), or empty.""" + return isinstance(value, dict) and all(str(k).isdigit() for k in value) + + +def coerce_array_shapes(config: Dict[str, Any], schema_props: Dict[str, Any], + short_lists_take_default: bool = False) -> None: + """Turn position-keyed dicts into lists wherever the schema has an array. + + Walks ``config`` alongside the schema's ``properties``, in place: into + nested objects, and into the objects of an array's items. An empty dict + where an array belongs becomes ``[]``. + + ``short_lists_take_default`` is for form posts. A form draws a fixed-length + list (an RGB colour, say) as one input per element, and a blanked input + drops out of the parsed list; the schema default then stands in, as long as + it is itself long enough, instead of the save failing on ``minItems``. + + Element types are left alone: normalization after this converts numeric + strings to the numbers the schema asks for. + """ + if not isinstance(config, dict): + return + for key, prop_schema in schema_props.items(): + if key not in config or not isinstance(prop_schema, dict): + continue + prop_type = prop_schema.get('type') + value = config[key] + + if prop_type == 'array': + if _is_index_dict(value): + value = config[key] = [value[k] for k in sorted(value, key=lambda k: int(str(k)))] + if not isinstance(value, list): + continue + min_items = prop_schema.get('minItems') + default = prop_schema.get('default') + if (short_lists_take_default and min_items is not None + and len(value) < min_items + and isinstance(default, list) and len(default) >= min_items): + value = config[key] = list(default) + items_schema = prop_schema.get('items') + if (isinstance(items_schema, dict) and items_schema.get('type') == 'object' + and 'properties' in items_schema): + for element in value: + coerce_array_shapes(element, items_schema['properties'], + short_lists_take_default) + + elif prop_type == 'object' and 'properties' in prop_schema: + coerce_array_shapes(value, prop_schema['properties'], short_lists_take_default) diff --git a/src/wifi_manager.py b/src/wifi_manager.py index 9e3dc0fb..86dde71f 100644 --- a/src/wifi_manager.py +++ b/src/wifi_manager.py @@ -9,24 +9,18 @@ Tested and optimized for: - Raspberry Pi OS Bookworm (Debian 12) with NetworkManager - Raspberry Pi 3B+, 4, 5 with built-in WiFi -Sudoers Requirements: - The following sudoers entries are required for passwordless operation. - Add to /etc/sudoers.d/ledmatrix_wifi: - - ledpi ALL=(ALL) NOPASSWD: /usr/bin/nmcli - ledpi ALL=(ALL) NOPASSWD: /usr/bin/systemctl start hostapd - ledpi ALL=(ALL) NOPASSWD: /usr/bin/systemctl stop hostapd - ledpi ALL=(ALL) NOPASSWD: /usr/bin/systemctl start dnsmasq - ledpi ALL=(ALL) NOPASSWD: /usr/bin/systemctl stop dnsmasq - ledpi ALL=(ALL) NOPASSWD: /usr/bin/systemctl restart NetworkManager - ledpi ALL=(ALL) NOPASSWD: /usr/sbin/ip - ledpi ALL=(ALL) NOPASSWD: /sbin/ip - ledpi ALL=(ALL) NOPASSWD: /usr/sbin/rfkill - ledpi ALL=(ALL) NOPASSWD: /usr/sbin/iptables - ledpi ALL=(ALL) NOPASSWD: /usr/sbin/sysctl - ledpi ALL=(ALL) NOPASSWD: /usr/bin/cp /tmp/hostapd.conf /etc/hostapd/hostapd.conf - ledpi ALL=(ALL) NOPASSWD: /usr/bin/cp /tmp/dnsmasq.conf /etc/dnsmasq.d/ledmatrix-captive.conf - ledpi ALL=(ALL) NOPASSWD: /usr/bin/rm -f /etc/dnsmasq.d/ledmatrix-captive.conf +Privileges: + The web interface runs as an unprivileged user and reaches nmcli, + systemctl, sysctl, nft and rfkill through exact-command sudo rules. + scripts/install/configure_wifi_permissions.sh writes those rules (and a + PolicyKit rule for NetworkManager); first_time_install.sh runs it. Use + that script rather than granting commands by hand. It deliberately + grants neither ``iptables`` nor ``ip``: their rules take a live interface + name, so they would need a wildcard, and ``iptables --modprobe=`` + and ``ip netns exec`` both run an arbitrary program as root. The code + paths that call them with sudo therefore only work where the user has + broader sudo rights (a stock Raspberry Pi image grants the default user + blanket NOPASSWD). """ import subprocess @@ -39,6 +33,8 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from dataclasses import dataclass +from src.config_manager_atomic import atomic_write_json + logger = logging.getLogger(__name__) # Path for storing WiFi configuration (will be set dynamically) @@ -78,6 +74,22 @@ DNSMASQ_SERVICE = "dnsmasq" DEFAULT_AP_SSID = "LEDMatrix-Setup" DEFAULT_AP_CHANNEL = 7 +#: The access point's own address. Clients get 192.168.4.2-20 from dnsmasq +#: (hostapd mode) and every DNS name resolves here, which is what makes phones +#: show the captive-portal page. +AP_IP = "192.168.4.1" + +#: The web interface's port. The captive portal redirects port 80 to it. +PORTAL_PORT = 5000 + +#: The NetworkManager profile this module creates for the access point. +AP_PROFILE_NAME = "LEDMatrix-Setup-AP" + +#: AP profiles taken down and deleted before a new one is created and when AP +#: mode ends: ours, NetworkManager's default hotspot name, and an older name. +#: Deleted by name only, never by SSID, so a saved home network is never hit. +AP_PROFILE_NAMES = (AP_PROFILE_NAME, "Hotspot", "TickerSetup-AP") + # LED status message file (for display_controller integration) LED_STATUS_FILE = None # Will be set dynamically @@ -198,30 +210,8 @@ class WiFiManager: logger.debug(f"Could not clear LED status message: {e}") def _check_command(self, command: str) -> bool: - """Check if a command is available""" - try: - # First try 'which' command - result = subprocess.run( - ["which", command], - capture_output=True, - timeout=2 - ) - if result.returncode == 0: - return True - - # Check common sbin paths (not in standard user PATH) - sbin_paths = [ - f"/usr/sbin/{command}", - f"/sbin/{command}", - f"/usr/local/sbin/{command}" - ] - for path in sbin_paths: - if os.path.isfile(path) and os.access(path, os.X_OK): - return True - - return False - except (subprocess.TimeoutExpired, subprocess.SubprocessError, OSError): - return False + """Whether ``command`` is installed (see _find_command_path).""" + return self._find_command_path(command) is not None def _find_command_path(self, command: str) -> Optional[str]: """ @@ -321,14 +311,22 @@ class WiFiManager: del self.config["saved_networks"] self._save_config() - def _save_config(self): - """Save WiFi configuration to file""" + def _save_config(self) -> bool: + """Write ``self.config`` to ``self.config_path``. + + The write is atomic and keeps the file's owner and shared group (see + atomic_write_json), so a save by the root display service does not + lock the web user out of the file. Returns False when the file could + not be written, for example when an older root-run save left it + owned by root; the in-memory config is kept either way. + """ try: - with open(self.config_path, 'w') as f: - json.dump(self.config, f, indent=2) - logger.info(f"Saved WiFi config to {self.config_path}") - except Exception as e: - logger.error(f"Failed to save WiFi config: {e}") + atomic_write_json(self.config_path, self.config) + except (OSError, TypeError, ValueError) as e: + logger.error(f"Failed to save WiFi config to {self.config_path}: {e}") + return False + logger.info(f"Saved WiFi config to {self.config_path}") + return True def get_wifi_status(self) -> WiFiStatus: """ @@ -402,8 +400,6 @@ class WiFiManager: for line in result.stdout.strip().split('\n'): if '802-11-wireless.ssid:' in line: ssid = line.split(':', 1)[1].strip() - if ssid: - continue elif 'WIFI.SIGNAL:' in line: try: signal = int(line.split(':', 1)[1].strip()) @@ -425,24 +421,7 @@ class WiFiManager: ssid = parts[1].strip() if ssid: break - - # Fallback: Get signal strength if not already retrieved - if signal == 0 and wlan_device: - result = subprocess.run( - ["nmcli", "-t", "-f", "WIFI.SIGNAL", "device", "show", wlan_device], - capture_output=True, - text=True, - timeout=5 - ) - if result.returncode == 0: - for line in result.stdout.strip().split('\n'): - if 'WIFI.SIGNAL:' in line: - try: - signal = int(line.split(':', 1)[1].strip()) - break - except (ValueError, IndexError): - pass - + # Get IP address if connected if wifi_connected and wlan_device: result = subprocess.run( @@ -536,7 +515,7 @@ class WiFiManager: if result.returncode == 0: ips = result.stdout.strip().split() for ip in ips: - if not ip.startswith('192.168.4.1'): # Exclude AP IP + if ip != AP_IP: ip_address = ip break @@ -778,13 +757,13 @@ class WiFiManager: if subprocess.run( ["sudo", iptables, "-t", "nat", "-C", "PREROUTING", "-i", self._wifi_interface, "-p", "tcp", "--dport", "80", - "-j", "REDIRECT", "--to-port", "5000"], + "-j", "REDIRECT", "--to-port", str(PORTAL_PORT)], capture_output=True, timeout=5 ).returncode != 0: r = subprocess.run( ["sudo", iptables, "-t", "nat", "-A", "PREROUTING", "-i", self._wifi_interface, "-p", "tcp", "--dport", "80", - "-j", "REDIRECT", "--to-port", "5000"], + "-j", "REDIRECT", "--to-port", str(PORTAL_PORT)], capture_output=True, text=True, timeout=5 ) if r.returncode != 0: @@ -794,12 +773,12 @@ class WiFiManager: if subprocess.run( ["sudo", iptables, "-C", "INPUT", - "-i", self._wifi_interface, "-p", "tcp", "--dport", "5000", "-j", "ACCEPT"], + "-i", self._wifi_interface, "-p", "tcp", "--dport", str(PORTAL_PORT), "-j", "ACCEPT"], capture_output=True, timeout=5 ).returncode != 0: r = subprocess.run( ["sudo", iptables, "-A", "INPUT", - "-i", self._wifi_interface, "-p", "tcp", "--dport", "5000", "-j", "ACCEPT"], + "-i", self._wifi_interface, "-p", "tcp", "--dport", str(PORTAL_PORT), "-j", "ACCEPT"], capture_output=True, text=True, timeout=5 ) if r.returncode != 0: @@ -808,7 +787,7 @@ class WiFiManager: return False self._redirect_backend = "iptables" - logger.info("iptables: port 80→5000 redirect rules added") + logger.info(f"iptables: port 80→{PORTAL_PORT} redirect rules added") return True def _setup_iptables_redirect_nftables(self, nft: str) -> bool: @@ -819,7 +798,7 @@ class WiFiManager: ["sudo", nft, "add", "chain", "ip", "ledmatrix", "prerouting", "{", "type", "nat", "hook", "prerouting", "priority", "-100", ";", "}"], ["sudo", nft, "add", "rule", "ip", "ledmatrix", "prerouting", - "iif", self._wifi_interface, "tcp", "dport", "80", "redirect", "to", ":5000"], + "iif", self._wifi_interface, "tcp", "dport", "80", "redirect", "to", f":{PORTAL_PORT}"], ] for cmd in cmds: r = subprocess.run(cmd, capture_output=True, text=True, timeout=5) @@ -832,7 +811,7 @@ class WiFiManager: logger.debug(f"nft cmd non-zero (may already exist): {r.stderr.strip()}") self._redirect_backend = "nftables" - logger.info("nftables: port 80→5000 redirect rule added") + logger.info(f"nftables: port 80→{PORTAL_PORT} redirect rule added") return True def _teardown_iptables_redirect(self) -> None: @@ -847,12 +826,12 @@ class WiFiManager: subprocess.run( ["sudo", iptables, "-t", "nat", "-D", "PREROUTING", "-i", self._wifi_interface, "-p", "tcp", "--dport", "80", - "-j", "REDIRECT", "--to-port", "5000"], + "-j", "REDIRECT", "--to-port", str(PORTAL_PORT)], capture_output=True, timeout=5 ) subprocess.run( ["sudo", iptables, "-D", "INPUT", - "-i", self._wifi_interface, "-p", "tcp", "--dport", "5000", + "-i", self._wifi_interface, "-p", "tcp", "--dport", str(PORTAL_PORT), "-j", "ACCEPT"], capture_output=True, timeout=5 ) @@ -887,7 +866,7 @@ class WiFiManager: except Exception as e: logger.warning(f"Could not tear down port redirect: {e}") - def _write_nm_dnsmasq_captive_conf(self, ap_ip: str = "192.168.4.1") -> None: + def _write_nm_dnsmasq_captive_conf(self, ap_ip: str = AP_IP) -> None: """ Write the NM dnsmasq-shared.d drop-in that makes NM's built-in dnsmasq resolve every hostname to the AP IP. This triggers the OS captive-portal @@ -1026,39 +1005,53 @@ class WiFiManager: ) if result.returncode != 0: return [] - - seen_ssids = set() - for line in result.stdout.strip().split('\n'): - if not line or ':' not in line: - continue - parts = line.split(':') - if len(parts) >= 3: - ssid = parts[0].strip() - if not ssid or ssid in seen_ssids: - continue - seen_ssids.add(ssid) - try: - signal = int(parts[1].strip()) - security = parts[2].strip() if len(parts) > 2 else "open" - frequency_str = parts[3].strip() if len(parts) > 3 else "0" - frequency_str = frequency_str.replace(" MHz", "").replace("MHz", "").strip() - frequency = float(frequency_str) if frequency_str else 0.0 - if "WPA3" in security: - sec_type = "wpa3" - elif "WPA2" in security: - sec_type = "wpa2" - elif "WPA" in security: - sec_type = "wpa" - else: - sec_type = "open" - networks.append(WiFiNetwork(ssid=ssid, signal=signal, security=sec_type, frequency=frequency)) - except (ValueError, IndexError): - continue - networks.sort(key=lambda x: x.signal, reverse=True) + networks = self._parse_nmcli_wifi_list(result.stdout) except Exception as e: logger.debug(f"nmcli cached list failed: {e}") return networks + @staticmethod + def _parse_nmcli_wifi_list(stdout: str) -> List[WiFiNetwork]: + """Parse ``nmcli -t -f SSID,SIGNAL,SECURITY,FREQ device wifi list``. + + One entry per SSID (the first line seen for it; hidden networks with + an empty SSID are skipped), security reduced to wpa3/wpa2/wpa/open, + sorted strongest first. Unparseable lines are skipped. + """ + networks = [] + seen_ssids = set() + for line in stdout.strip().split('\n'): + if not line or ':' not in line: + continue + parts = line.split(':') + if len(parts) < 3: + continue + ssid = parts[0].strip() + if not ssid or ssid in seen_ssids: + continue + seen_ssids.add(ssid) + try: + signal = int(parts[1].strip()) + security = parts[2].strip() + frequency_str = parts[3].strip() if len(parts) > 3 else "0" + frequency_str = frequency_str.replace(" MHz", "").replace("MHz", "").strip() + frequency = float(frequency_str) if frequency_str else 0.0 + except (ValueError, IndexError) as e: + logger.debug(f"Skipping network line due to parsing error: {line[:50]}... Error: {e}") + continue + if "WPA3" in security: + sec_type = "wpa3" + elif "WPA2" in security: + sec_type = "wpa2" + elif "WPA" in security: + sec_type = "wpa" + else: + sec_type = "open" + networks.append(WiFiNetwork(ssid=ssid, signal=signal, security=sec_type, + frequency=frequency)) + networks.sort(key=lambda x: x.signal, reverse=True) + return networks + def _save_cached_scan(self, networks: List[WiFiNetwork]) -> None: """Save scan results to a cache file for use during AP mode.""" try: @@ -1088,7 +1081,6 @@ class WiFiManager: def _scan_nmcli(self) -> List[WiFiNetwork]: """Scan networks using nmcli""" - networks = [] try: # Trigger scan subprocess.run( @@ -1108,52 +1100,7 @@ class WiFiManager: if result.returncode != 0: return [] - - seen_ssids = set() - for line in result.stdout.strip().split('\n'): - if not line or ':' not in line: - continue - - parts = line.split(':') - if len(parts) >= 3: - ssid = parts[0].strip() - if not ssid or ssid in seen_ssids: - continue - - seen_ssids.add(ssid) - - try: - signal = int(parts[1].strip()) - security = parts[2].strip() if len(parts) > 2 else "open" - - # Parse frequency - strip " MHz" if present - frequency_str = parts[3].strip() if len(parts) > 3 else "0" - frequency_str = frequency_str.replace(" MHz", "").replace("MHz", "").strip() - frequency = float(frequency_str) if frequency_str else 0.0 - - # Normalize security type - if "WPA3" in security: - sec_type = "wpa3" - elif "WPA2" in security: - sec_type = "wpa2" - elif "WPA" in security: - sec_type = "wpa" - else: - sec_type = "open" - - networks.append(WiFiNetwork( - ssid=ssid, - signal=signal, - security=sec_type, - frequency=frequency - )) - except (ValueError, IndexError) as e: - logger.debug(f"Skipping network line due to parsing error: {line[:50]}... Error: {e}") - continue - - # Sort by signal strength - networks.sort(key=lambda x: x.signal, reverse=True) - return networks + return self._parse_nmcli_wifi_list(result.stdout) except Exception as e: logger.error(f"Error scanning with nmcli: {e}") return [] @@ -1357,27 +1304,7 @@ class WiFiManager: disconnect_success, disconnect_msg = self.disconnect_from_network(skip_ap_check=True) if disconnect_success: logger.info(f"Disconnected from {original_ssid}: {disconnect_msg}") - # Wait for device to be ready for new connection - # Check device state before proceeding - max_wait = 5 - wait_count = 0 - while wait_count < max_wait: - time.sleep(1) - result = subprocess.run( - ["nmcli", "-t", "-f", "STATE", "device", "status", self._wifi_interface], - capture_output=True, - text=True, - timeout=5 - ) - if result.returncode == 0: - state = result.stdout.strip().split(':')[-1] if ':' in result.stdout else result.stdout.strip() - # Device is ready if it's disconnected or unavailable (not connecting/connected) - if state in ["disconnected", "unavailable", "unmanaged"]: - logger.info(f"Device ready for new connection (state: {state})") - break - wait_count += 1 - - if wait_count >= max_wait: + if not self._wait_for_device_idle(5): logger.warning("Device may not be ready, but proceeding with connection attempt") else: logger.warning(f"Failed to disconnect from {original_ssid}: {disconnect_msg}") @@ -1403,26 +1330,15 @@ class WiFiManager: return False, f"Failed to connect to {ssid}, restored {original_ssid}" else: logger.error(f"Failed to restore original connection: {original_ssid}") - # Trigger AP mode as last resort - self._show_led_message("Enabling AP mode...", duration=5) - ap_success, ap_msg = self.enable_ap_mode(force=True) - if ap_success: - logger.info("AP mode enabled as failsafe") - return False, "Connection failed and restoration failed. AP mode enabled." - else: - logger.error(f"Failed to enable AP mode: {ap_msg}") - return False, f"Connection failed, restoration failed, and AP mode failed: {ap_msg}" + return self._failsafe_ap( + "Connection failed and restoration failed. AP mode enabled.", + "Connection failed, restoration failed, and AP mode failed") # If connection failed and no original connection to restore, enable AP mode elif not success: logger.warning(f"Connection to {ssid} failed and no original connection to restore") - self._show_led_message("Enabling AP mode...", duration=5) - ap_success, ap_msg = self.enable_ap_mode(force=True) - if ap_success: - logger.info("AP mode enabled as failsafe") - return False, "Connection failed. AP mode enabled." - else: - return False, f"Connection failed and AP mode failed: {ap_msg}" + return self._failsafe_ap("Connection failed. AP mode enabled.", + "Connection failed and AP mode failed") return success, message else: @@ -1443,6 +1359,22 @@ class WiFiManager: logger.error("Last-resort AP mode enable failed in recovery path: %s", ap_error, exc_info=True) return False, str(e) + def _failsafe_ap(self, enabled_msg: str, failed_msg: str) -> Tuple[bool, str]: + """Force the setup AP up after a connect that left no working network, + so the user can still reach the device. + + Returns the (False, message) result for connect_to_network: + ``enabled_msg`` when the AP came up, else ``failed_msg`` plus the + reason it did not. + """ + self._show_led_message("Enabling AP mode...", duration=5) + ap_success, ap_msg = self.enable_ap_mode(force=True) + if ap_success: + logger.info("AP mode enabled as failsafe") + return False, enabled_msg + logger.error(f"Failed to enable AP mode: {ap_msg}") + return False, f"{failed_msg}: {ap_msg}" + def _restore_original_connection(self, connection_name: str, ssid: str) -> bool: """ Restore a previously active WiFi connection. @@ -1496,68 +1428,96 @@ class WiFiManager: logger.error(f"Error restoring connection: {e}") return False + def _find_profile_for_ssid(self, ssid: str) -> Optional[str]: + """Name of the saved NetworkManager profile for ``ssid``, or None. + + ``802-11-wireless.ssid`` is not a column ``nmcli connection show`` + can list, so this lists the Wi-Fi profiles and asks each one for its + SSID. A profile named after the SSID is the fallback, for when the + listing fails. + """ + list_result = subprocess.run( # nosec B603 B607 - fixed args, no user input + ["nmcli", "-t", "-f", "NAME,TYPE", "connection", "show"], + capture_output=True, text=True, timeout=5 + ) + if list_result.returncode == 0: + for line in list_result.stdout.strip().split('\n'): + # Terse output escapes a colon inside a field as "\:". TYPE + # never contains one, so the last colon ends the name. + conn_name, sep, conn_type = line.rpartition(':') + if not sep or conn_type.strip() != '802-11-wireless': + continue + conn_name = conn_name.replace('\\:', ':').replace('\\\\', '\\') + ssid_r = subprocess.run( # nosec B603 B607 - conn_name from nmcli output, not user input + ["nmcli", "-g", "802-11-wireless.ssid", "connection", "show", conn_name], + capture_output=True, text=True, timeout=5 + ) + if ssid_r.returncode == 0 and ssid_r.stdout.strip() == ssid: + return conn_name + + direct_check = subprocess.run( # nosec B603 B607 - list args, no shell + ["nmcli", "connection", "show", ssid], + capture_output=True, text=True, timeout=5 + ) + if direct_check.returncode == 0: + return ssid + return None + + def _wait_for_device_idle(self, attempts: int) -> bool: + """Poll the Wi-Fi device, once a second for up to ``attempts`` checks, + until it is disconnected, unavailable or unmanaged: a profile + activated while the device is still connecting or tearing down an + old link can fail. True if it went idle, False on timeout.""" + for attempt in range(attempts): + result = subprocess.run( + ["nmcli", "-t", "-f", "STATE", "device", "status", self._wifi_interface], + capture_output=True, + text=True, + timeout=5 + ) + if result.returncode == 0: + state = result.stdout.strip().split(':')[-1] + if state in ("disconnected", "unavailable", "unmanaged"): + logger.debug(f"Wi-Fi device idle (state: {state})") + return True + if attempt < attempts - 1: + time.sleep(1) + return False + + def _verify_connected(self, ssid: str, attempts: int = 5, delay: float = 2.0, + stop_on_other_network: bool = False) -> Optional[WiFiStatus]: + """Wait for the device to report a connection to ``ssid``. + + nmcli returns before DHCP finishes, so the status is polled every + ``delay`` seconds, up to ``attempts`` times. Returns that status, or + None if it never showed ``ssid``. With ``stop_on_other_network`` a + connection to a different SSID ends the wait at once as a failure. + """ + for _ in range(attempts): + time.sleep(delay) + status = self.get_wifi_status() + if not status.connected: + continue + if status.ssid == ssid: + return status + if stop_on_other_network and status.ssid: + logger.warning(f"Connected to wrong network: {status.ssid} instead of {ssid}") + return None + return None + def _connect_nmcli(self, ssid: str, password: str) -> Tuple[bool, str]: """Connect using nmcli""" try: # Show LED message self._show_led_message(f"Connecting to {ssid}...", duration=10) - # Find existing NM connection for this SSID. - # 802-11-wireless.ssid is not a valid column in 'nmcli connection show', - # so list all wifi connections then query each one's SSID individually. - list_result = subprocess.run( # nosec B603 B607 - fixed args, no user input - ["nmcli", "-t", "-f", "NAME,TYPE", "connection", "show"], - capture_output=True, text=True, timeout=5 - ) - existing_conn_name = None - if list_result.returncode == 0: - for line in list_result.stdout.strip().split('\n'): - if ':' not in line: - continue - parts = line.split(':') - if len(parts) < 2 or parts[1].strip() != '802-11-wireless': - continue - conn_name = parts[0].strip() - ssid_r = subprocess.run( # nosec B603 B607 - conn_name from nmcli output, not user input - ["nmcli", "-g", "802-11-wireless.ssid", "connection", "show", conn_name], - capture_output=True, text=True, timeout=5 - ) - if ssid_r.returncode == 0 and ssid_r.stdout.strip() == ssid: - existing_conn_name = conn_name - break - - # Also try direct lookup by SSID (in case connection name matches SSID) - if not existing_conn_name: - direct_check = subprocess.run( - ["nmcli", "connection", "show", ssid], - capture_output=True, - text=True, - timeout=5 - ) - if direct_check.returncode == 0: - existing_conn_name = ssid - + existing_conn_name = self._find_profile_for_ssid(ssid) if existing_conn_name: # Connection exists, try to activate it first (faster and more reliable) logger.info(f"Found existing connection for {ssid}, activating...") - # Ensure device is ready before activating - # Wait for device to be in disconnected/unavailable state - max_wait = 3 - for wait_attempt in range(max_wait): - device_result = subprocess.run( - ["nmcli", "-t", "-f", "STATE", "device", "status", self._wifi_interface], - capture_output=True, - text=True, - timeout=5 - ) - if device_result.returncode == 0: - state = device_result.stdout.strip().split(':')[-1] if ':' in device_result.stdout else device_result.stdout.strip() - if state in ["disconnected", "unavailable", "unmanaged"]: - break - if wait_attempt < max_wait - 1: - time.sleep(1) - + self._wait_for_device_idle(3) + result = subprocess.run( ["nmcli", "connection", "up", existing_conn_name], capture_output=True, @@ -1566,19 +1526,8 @@ class WiFiManager: ) if result.returncode == 0: - # Wait longer for connection to stabilize and verify multiple times - max_verification_attempts = 5 - verification_delay = 2 - connected = False - - for attempt in range(max_verification_attempts): - time.sleep(verification_delay) - status = self.get_wifi_status() - if status.connected and status.ssid == ssid: - connected = True - break - - if connected: + status = self._verify_connected(ssid) + if status is not None: ip = status.ip_address or "Unknown" self._show_led_message(f"Connected! {ip}", duration=5) logger.info(f"Successfully connected to {ssid} with IP {ip}") @@ -1605,25 +1554,8 @@ class WiFiManager: ) if result.returncode == 0: - # Wait longer for connection to stabilize and verify multiple times - max_verification_attempts = 5 - verification_delay = 2 - connected = False - - for attempt in range(max_verification_attempts): - time.sleep(verification_delay) - status = self.get_wifi_status() - if status.connected: - # Verify we're connected to the correct SSID - if status.ssid == ssid: - connected = True - break - elif status.ssid: - # Connected to different network - this is a failure - logger.warning(f"Connected to wrong network: {status.ssid} instead of {ssid}") - break - - if connected: + status = self._verify_connected(ssid, stop_on_other_network=True) + if status is not None: ip = status.ip_address or "Unknown" self._show_led_message(f"Connected! {ip}", duration=5) logger.info(f"Successfully connected to {ssid} with IP {ip}") @@ -1717,14 +1649,10 @@ class WiFiManager: return any(ind in lower for ind in indicators) def _connect_wpa_supplicant(self, ssid: str, password: str) -> Tuple[bool, str]: - """Connect using wpa_supplicant (fallback)""" - try: - # This would require modifying /etc/wpa_supplicant/wpa_supplicant.conf - # For now, return not implemented - return False, "wpa_supplicant connection not yet implemented. Please use NetworkManager (nmcli)." - except Exception as e: - logger.error(f"Error connecting with wpa_supplicant: {e}") - return False, str(e) + """Without NetworkManager there is no supported way to connect: doing it + through wpa_supplicant would mean editing its config file, which is + not implemented. Always returns (False, reason).""" + return False, "wpa_supplicant connection not yet implemented. Please use NetworkManager (nmcli)." def disconnect_from_network(self, skip_ap_check: bool = False) -> Tuple[bool, str]: """ @@ -1745,33 +1673,18 @@ class WiFiManager: # Disconnect using nmcli if self.has_nmcli: - # Try to disconnect the specific connection first (more reliable) + # Take the profile down first, then the device, so the + # device ends up disconnected even when no profile is found. if status.ssid: - # Find the connection name for this SSID - conn_result = subprocess.run( - ["nmcli", "-t", "-f", "NAME,802-11-wireless.ssid", "connection", "show"], - capture_output=True, - text=True, - timeout=5 - ) - if conn_result.returncode == 0: - for line in conn_result.stdout.strip().split('\n'): - if ':' in line: - parts = line.split(':') - if len(parts) >= 2: - conn_name = parts[0].strip() - conn_ssid = parts[1].strip() if len(parts) > 1 else "" - if conn_ssid == status.ssid: - # Disconnect this specific connection - subprocess.run( - ["nmcli", "connection", "down", conn_name], - capture_output=True, - timeout=10 - ) - logger.info(f"Disconnected connection {conn_name} for {status.ssid}") - break - - # Also disconnect the device to ensure clean state + conn_name = self._find_profile_for_ssid(status.ssid) + if conn_name: + subprocess.run( # nosec B603 B607 - list args, no shell + ["nmcli", "connection", "down", conn_name], + capture_output=True, + timeout=10 + ) + logger.info(f"Disconnected connection {conn_name} for {status.ssid}") + result = subprocess.run( ["nmcli", "device", "disconnect", self._wifi_interface], capture_output=True, @@ -1814,7 +1727,11 @@ class WiFiManager: max_retries: Maximum number of retry attempts to enable WiFi radio Returns: - True if WiFi is enabled or was successfully enabled, False otherwise + True if the radio is enabled or was enabled here. Also True when + the state could not be checked at all (nmcli or rfkill raised on + every attempt): callers go ahead rather than refusing to act on a + radio that is probably fine. False only when the radio was seen + disabled or blocked and could not be turned on. """ for attempt in range(max_retries): try: @@ -2062,11 +1979,7 @@ class WiFiManager: if result[0]: self._ap_enabled_at = time.time() if force: - try: - self._FORCE_AP_FLAG_PATH.touch() - logger.debug(f"Force-AP flag created: {self._FORCE_AP_FLAG_PATH}") - except OSError as exc: - logger.warning(f"Failed to create force-AP flag {self._FORCE_AP_FLAG_PATH}: {exc}") + self._mark_forced() return result # Fallback to nmcli hotspot (simpler, no captive portal) @@ -2077,11 +1990,7 @@ class WiFiManager: if result[0]: self._ap_enabled_at = time.time() if force: - try: - self._FORCE_AP_FLAG_PATH.touch() - logger.debug(f"Force-AP flag created: {self._FORCE_AP_FLAG_PATH}") - except OSError as exc: - logger.warning(f"Failed to create force-AP flag {self._FORCE_AP_FLAG_PATH}: {exc}") + self._mark_forced() return result return False, "No WiFi tools available (nmcli, hostapd, or dnsmasq required)" @@ -2089,6 +1998,15 @@ class WiFiManager: logger.error(f"Error in enable_ap_mode: {e}") return False, str(e) + def _mark_forced(self) -> None: + """Record that AP mode was forced on, so the periodic check leaves it + up even when Ethernet is connected (see _manage_ap_mode).""" + try: + self._FORCE_AP_FLAG_PATH.touch() + logger.debug(f"Force-AP flag created: {self._FORCE_AP_FLAG_PATH}") + except OSError as exc: + logger.warning(f"Failed to create force-AP flag {self._FORCE_AP_FLAG_PATH}: {exc}") + def _enable_ap_mode_hostapd(self) -> Tuple[bool, str]: """Enable AP mode using hostapd and dnsmasq (captive portal)""" try: @@ -2115,7 +2033,7 @@ class WiFiManager: timeout=10 ) subprocess.run( - ["sudo", "ip", "addr", "add", "192.168.4.1/24", "dev", self._wifi_interface], + ["sudo", "ip", "addr", "add", f"{AP_IP}/24", "dev", self._wifi_interface], capture_output=True, timeout=10 ) @@ -2124,7 +2042,7 @@ class WiFiManager: capture_output=True, timeout=10 ) - logger.info(f"Configured {self._wifi_interface} with IP 192.168.4.1 for AP mode") + logger.info(f"Configured {self._wifi_interface} with IP {AP_IP} for AP mode") except (subprocess.TimeoutExpired, subprocess.SubprocessError, OSError) as e: logger.warning(f"Error setting up {self._wifi_interface} IP: {e}") @@ -2168,7 +2086,7 @@ class WiFiManager: # Use the validated SSID so the displayed name matches what hostapd broadcast ap_ssid, _ = self._validate_ap_config() self._show_led_message( - f"WiFi Setup\n{ap_ssid}\nNo password\n192.168.4.1:5000", duration=10 + f"WiFi Setup\n{ap_ssid}\nNo password\n{AP_IP}:{PORTAL_PORT}", duration=10 ) return True, "AP mode enabled" except Exception as e: @@ -2198,7 +2116,7 @@ class WiFiManager: # Delete only the specific application-managed AP profiles by name. # Never delete by SSID — that would destroy a user's saved home network. - for conn_name in ["Hotspot", "LEDMatrix-Setup-AP", "TickerSetup-AP"]: + for conn_name in AP_PROFILE_NAMES: subprocess.run(["nmcli", "connection", "down", conn_name], capture_output=True, timeout=5) subprocess.run(["nmcli", "connection", "delete", conn_name], @@ -2215,14 +2133,14 @@ class WiFiManager: cmd = [ "nmcli", "connection", "add", "type", "wifi", - "con-name", "LEDMatrix-Setup-AP", + "con-name", AP_PROFILE_NAME, "ifname", self._wifi_interface, "ssid", ap_ssid, "802-11-wireless.mode", "ap", "802-11-wireless.band", "bg", # 2.4 GHz for maximum compatibility "802-11-wireless.channel", str(ap_channel), "ipv4.method", "shared", - "ipv4.addresses", "192.168.4.1/24", + "ipv4.addresses", f"{AP_IP}/24", # No 802-11-wireless-security section → open network ] @@ -2247,14 +2165,14 @@ class WiFiManager: logger.info("AP connection profile created, bringing it up...") up_result = subprocess.run( - ["nmcli", "connection", "up", "LEDMatrix-Setup-AP"], + ["nmcli", "connection", "up", AP_PROFILE_NAME], capture_output=True, text=True, timeout=20 ) if up_result.returncode != 0: error_msg = up_result.stderr.strip() or up_result.stdout.strip() logger.error(f"Failed to bring up AP connection: {error_msg}") self._remove_nm_dnsmasq_captive_conf() - subprocess.run(["nmcli", "connection", "delete", "LEDMatrix-Setup-AP"], + subprocess.run(["nmcli", "connection", "delete", AP_PROFILE_NAME], capture_output=True, timeout=10) self._show_led_message("AP mode failed", duration=5) return False, f"Failed to start AP: {error_msg}" @@ -2266,9 +2184,9 @@ class WiFiManager: if not self._setup_iptables_redirect(): logger.error("Captive-portal redirect setup failed; rolling back AP profile") self._remove_nm_dnsmasq_captive_conf() - subprocess.run(["nmcli", "connection", "down", "LEDMatrix-Setup-AP"], + subprocess.run(["nmcli", "connection", "down", AP_PROFILE_NAME], capture_output=True, timeout=10) - subprocess.run(["nmcli", "connection", "delete", "LEDMatrix-Setup-AP"], + subprocess.run(["nmcli", "connection", "delete", AP_PROFILE_NAME], capture_output=True, timeout=10) self._clear_led_message() return False, "AP started but captive-portal redirect setup failed" @@ -2282,17 +2200,17 @@ class WiFiManager: logger.debug(f"AP verification attempt {_attempt + 1}/5 not yet active, waiting 2s") time.sleep(2) if status.get('active'): - ip = status.get('ip', '192.168.4.1') + ip = status.get('ip', AP_IP) logger.info(f"AP mode confirmed active at {ip} (open network, no password)") - self._show_led_message(f"WiFi Setup\n{ap_ssid}\nNo password\n{ip}:5000", duration=10) - return True, f"AP mode enabled (open network) - Access at {ip}:5000" + self._show_led_message(f"WiFi Setup\n{ap_ssid}\nNo password\n{ip}:{PORTAL_PORT}", duration=10) + return True, f"AP mode enabled (open network) - Access at {ip}:{PORTAL_PORT}" else: logger.error("AP mode started but not verified by status check — rolling back") self._teardown_iptables_redirect() self._remove_nm_dnsmasq_captive_conf() - subprocess.run(["nmcli", "connection", "down", "LEDMatrix-Setup-AP"], + subprocess.run(["nmcli", "connection", "down", AP_PROFILE_NAME], capture_output=True, timeout=10) - subprocess.run(["nmcli", "connection", "delete", "LEDMatrix-Setup-AP"], + subprocess.run(["nmcli", "connection", "delete", AP_PROFILE_NAME], capture_output=True, timeout=10) self._clear_led_message() return False, "AP mode started but verification failed" @@ -2326,9 +2244,9 @@ class WiFiManager: conn_name = parts[0].strip() conn_type = parts[1].strip().lower() # Match our known AP profile name OR the legacy nmcli hotspot type - if conn_name == "LEDMatrix-Setup-AP" or 'hotspot' in conn_type: + if conn_name == AP_PROFILE_NAME or 'hotspot' in conn_type: # Get actual IP address (may be 192.168.4.1 or 10.42.0.1 depending on config) - ip = '192.168.4.1' + ip = AP_IP interface = parts[2] if len(parts) > 2 else self._wifi_interface try: ip_result = subprocess.run( @@ -2399,7 +2317,7 @@ class WiFiManager: ) else: # Disable nmcli hotspot mode (fallback) - for conn_name in ["LEDMatrix-Setup-AP", "Hotspot", "TickerSetup-AP"]: + for conn_name in AP_PROFILE_NAMES: subprocess.run( ["nmcli", "connection", "down", conn_name], capture_output=True, @@ -2428,7 +2346,7 @@ class WiFiManager: # Clean up WiFi interface IP configuration subprocess.run( - ["sudo", "ip", "addr", "del", "192.168.4.1/24", "dev", self._wifi_interface], + ["sudo", "ip", "addr", "del", f"{AP_IP}/24", "dev", self._wifi_interface], capture_output=True, timeout=10 ) @@ -2541,13 +2459,13 @@ ignore_broadcast_ssid=0 dhcp-range=192.168.4.2,192.168.4.20,255.255.255.0,24h # Captive portal: Redirect all DNS queries to Pi -address=/#/192.168.4.1 +address=/#/{AP_IP} # Captive portal detection endpoints -address=/captive.apple.com/192.168.4.1 -address=/connectivitycheck.gstatic.com/192.168.4.1 -address=/www.msftconnecttest.com/192.168.4.1 -address=/detectportal.firefox.com/192.168.4.1 +address=/captive.apple.com/{AP_IP} +address=/connectivitycheck.gstatic.com/{AP_IP} +address=/www.msftconnecttest.com/{AP_IP} +address=/detectportal.firefox.com/{AP_IP} """ # Write config (requires sudo) @@ -2651,9 +2569,10 @@ address=/detectportal.firefox.com/192.168.4.1 # Pre-cache a WiFi scan so the captive portal can show networks try: logger.info("Running pre-AP WiFi scan for captive portal cache...") + # AP mode is not up yet, so this is a live scan, and + # scan_networks saves its result for the portal. networks, _cached = self.scan_networks(allow_cached=False) if networks: - self._save_cached_scan(networks) logger.info(f"Cached {len(networks)} networks for captive portal") except Exception as scan_err: logger.debug(f"Pre-AP scan failed (non-critical): {scan_err}") diff --git a/start_display.sh b/start_display.sh index df9fcb92..cbe4ca4a 100755 --- a/start_display.sh +++ b/start_display.sh @@ -1,9 +1,6 @@ #!/bin/bash -# Get the current user -CURRENT_USER=$(whoami) - -echo "Starting LED Matrix Display Service for user: $CURRENT_USER..." +echo "Starting LED Matrix Display Service..." # Start the service sudo systemctl start ledmatrix.service diff --git a/stop_display.sh b/stop_display.sh index da37b27c..533463c6 100755 --- a/stop_display.sh +++ b/stop_display.sh @@ -1,9 +1,6 @@ #!/bin/bash -# Get the current user -CURRENT_USER=$(whoami) - -echo "Stopping LED Matrix Display Service for user: $CURRENT_USER..." +echo "Stopping LED Matrix Display Service..." # Stop the service sudo systemctl stop ledmatrix.service diff --git a/test/test_api_helper.py b/test/test_api_helper.py index 55449766..6df0960d 100644 --- a/test/test_api_helper.py +++ b/test/test_api_helper.py @@ -92,7 +92,7 @@ class TestGet: helper.session.get.assert_not_called() rate_spy.assert_not_called() - def test_cache_miss_fetches_and_caches_without_ttl(self, helper, cache): + def test_cache_miss_fetches_and_caches_with_ttl(self, helper, cache): cache.get.return_value = None helper.session.get = Mock(return_value=_make_response({'a': 1})) @@ -100,9 +100,47 @@ class TestGet: cache_ttl=999) assert result == {'a': 1} - # Pin the ttl-dropped contract: CacheManager.set is called with - # (key, data) only — the cache_ttl argument is discarded. - cache.set.assert_called_once_with('k', {'a': 1}) + cache.set.assert_called_once_with('k', {'a': 1}, ttl=999) + + def test_set_cache_passes_ttl(self, helper, cache): + helper.set_cache('k', {'a': 1}, ttl=42) + cache.set.assert_called_once_with('k', {'a': 1}, ttl=42) + + +class TestCacheLifetimeWithRealCacheManager: + """cache_ttl decides how long a response is reused, in both directions: + past CacheManager's 300-second default read age, and not beyond it.""" + + @pytest.fixture + def real_cache(self, tmp_path): + from unittest.mock import patch + from src.cache_manager import CacheManager + with patch('src.cache_manager.CacheManager._get_writable_cache_dir', + return_value=str(tmp_path)): + cache = CacheManager() + yield cache + # Releases the class-wide cleanup-thread claim on this directory, + # which would otherwise leak into test_cache_cleanup_thread_ownership. + cache.stop_cleanup_thread() + + def _fetch_twice(self, real_cache, monkeypatch, ttl, elapsed): + helper = APIHelper(cache_manager=real_cache) + helper.set_rate_limit(0) + helper.session.get = Mock(side_effect=[_make_response({'n': 1}), + _make_response({'n': 2})]) + now = [1_000_000.0] + monkeypatch.setattr('src.cache.memory_cache.time.time', lambda: now[0]) + monkeypatch.setattr('src.cache.disk_cache.time.time', lambda: now[0]) + monkeypatch.setattr('src.cache_manager.time.time', lambda: now[0]) + helper.get('https://example.com/api', cache_key='lifetime_test', cache_ttl=ttl) + now[0] += elapsed + return helper.get('https://example.com/api', cache_key='lifetime_test', cache_ttl=ttl) + + def test_long_ttl_outlives_the_default_read_age(self, real_cache, monkeypatch): + assert self._fetch_twice(real_cache, monkeypatch, ttl=3600, elapsed=1000) == {'n': 1} + + def test_short_ttl_expires(self, real_cache, monkeypatch): + assert self._fetch_twice(real_cache, monkeypatch, ttl=60, elapsed=120) == {'n': 2} def test_request_exception_returns_none_and_caches_nothing( self, helper, cache): @@ -223,15 +261,6 @@ class TestClearCache: manager.clear_cache.assert_called_once_with() - def test_no_pattern_falls_back_to_clear(self): - manager = types.SimpleNamespace(clear=Mock()) - helper = APIHelper(cache_manager=manager) - helper.set_rate_limit(0) - - helper.clear_cache() - - manager.clear.assert_called_once_with() - def test_no_pattern_manager_without_any_clear_is_noop(self): helper = APIHelper(cache_manager=object()) helper.set_rate_limit(0) diff --git a/test/test_api_v3_health.py b/test/test_api_v3_health.py new file mode 100644 index 00000000..d4921c67 --- /dev/null +++ b/test/test_api_v3_health.py @@ -0,0 +1,90 @@ +"""GET /api/v3/health: the plugin count is real, and a failed check is logged. + +The plugin check counted ``plugin_manager.get_available_plugins()``, which +PluginManager does not have; a hasattr guard turned that into a permanent 0. +Each check that fails answers "see logs for details", so it has to log. +""" + +import logging +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402 + +URL = "/api/v3/health" + + +@pytest.fixture(autouse=True) +def _no_systemctl(monkeypatch): + monkeypatch.setattr("web_interface.blueprints.api_v3.misc._get_display_service_status", + lambda: {"active": True}) + + +def _checks(client): + response = client.get(URL) + assert response.status_code == 200, response.get_json() + return response.get_json()["data"]["checks"] + + +def test_plugin_count_is_the_number_of_discovered_plugins(api_v3_client, api_v3_module): + api_v3_module.api_v3.plugin_manager.plugin_manifests = { + "clock": {"id": "clock"}, "weather": {"id": "weather"}, "stocks": {"id": "stocks"}, + } + + check = _checks(api_v3_client)["plugin_system"] + + assert check == {"status": "operational", "plugin_count": 3} + + +def test_plugin_count_discovers_when_nothing_is_discovered_yet(api_v3_client, api_v3_module): + pm = api_v3_module.api_v3.plugin_manager + pm.plugin_manifests = {} + + def discover(): + pm.plugin_manifests = {"clock": {"id": "clock"}} + pm.discover_plugins.side_effect = discover + + assert _checks(api_v3_client)["plugin_system"]["plugin_count"] == 1 + + +def test_a_failed_config_check_is_logged(api_v3_client, api_v3_module, caplog): + api_v3_module.api_v3.config_manager.load_config.side_effect = OSError("disk gone") + + with caplog.at_level(logging.WARNING): + check = _checks(api_v3_client)["config_file"] + + assert check["error"] == "see logs for details" + logged = [r for r in caplog.records if "config file" in r.getMessage()] + assert logged and logged[0].exc_info and "disk gone" in str(logged[0].exc_info[1]) + + +def test_a_failed_plugin_check_is_logged(api_v3_client, api_v3_module, caplog, monkeypatch): + def boom(): + raise RuntimeError("manifests unreadable") + monkeypatch.setattr("web_interface.blueprints.api_v3.misc._discovered_plugin_manifests", boom) + + with caplog.at_level(logging.WARNING): + check = _checks(api_v3_client)["plugin_system"] + + assert check["status"] == "error" + logged = [r for r in caplog.records if "count plugins" in r.getMessage()] + assert logged and logged[0].exc_info + + +def test_a_failed_hardware_check_is_logged(api_v3_client, api_v3_module, caplog, monkeypatch): + def getmtime(_path): + raise PermissionError("denied") + fake_os = SimpleNamespace(path=SimpleNamespace(exists=lambda _p: True, getmtime=getmtime)) + monkeypatch.setattr("web_interface.blueprints.api_v3.misc.os", fake_os) + + with caplog.at_level(logging.WARNING): + check = _checks(api_v3_client)["hardware"] + + assert check["status"] == "unknown" + logged = [r for r in caplog.records if "snapshot" in r.getMessage()] + assert logged and logged[0].exc_info diff --git a/test/test_api_v3_plugin_health_single.py b/test/test_api_v3_plugin_health_single.py new file mode 100644 index 00000000..455914ea --- /dev/null +++ b/test/test_api_v3_plugin_health_single.py @@ -0,0 +1,68 @@ +"""GET /plugins/health/ and /plugins/metrics/ read the display +service's latest state, not the web process's first snapshot. + +The display service writes health and metrics to the shared cache; the web +process only reads them. Its tracker and monitor keep what they read first in +memory, so without ``force_reload`` the per-plugin routes kept answering with +that first read while the list routes (which pass it) moved on. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.plugin_system.plugin_health import PluginHealthTracker # noqa: E402 +from src.plugin_system.resource_monitor import PluginResourceMonitor # noqa: E402 +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402 + + +class SharedCache: + """The on-disk cache both processes see, reduced to a dict.""" + + def __init__(self): + self.entries = {} + + def get(self, key, max_age=None, memory_ttl=None): + return self.entries.get(key) + + def set(self, key, value, *args, **kwargs): + self.entries[key] = value + + +@pytest.fixture +def shared_cache(api_v3_module): + cache = SharedCache() + pm = api_v3_module.api_v3.plugin_manager + pm.health_tracker = PluginHealthTracker(cache) + pm.resource_monitor = PluginResourceMonitor(cache) + return cache + + +def test_health_reflects_failures_recorded_after_the_first_read(api_v3_client, shared_cache): + first = api_v3_client.get("/api/v3/plugins/health/weather").get_json()["data"] + assert first["total_failures"] == 0 + + display_side = PluginHealthTracker(shared_cache) + display_side.record_failure("weather", RuntimeError("api down")) + display_side.record_failure("weather", RuntimeError("api down")) + + later = api_v3_client.get("/api/v3/plugins/health/weather").get_json()["data"] + assert later["total_failures"] == 2 + + +def test_metrics_reflect_calls_recorded_after_the_first_read(api_v3_client, shared_cache): + first = api_v3_client.get("/api/v3/plugins/metrics/weather").get_json()["data"] + assert first["call_count"] == 0 + + shared_cache.set("plugin_metrics:weather", { + "memory_mb": 12.5, "cpu_percent": 3.0, "execution_time": 0.2, + "call_count": 40, "total_execution_time": 8.0, + "max_execution_time": 0.5, "min_execution_time": 0.1, + "last_update_time": 1000.0, + }) + + later = api_v3_client.get("/api/v3/plugins/metrics/weather").get_json()["data"] + assert later["call_count"] == 40 diff --git a/test/test_api_v3_registry_endpoints.py b/test/test_api_v3_registry_endpoints.py index 85f4b377..b4e6dd1f 100644 --- a/test/test_api_v3_registry_endpoints.py +++ b/test/test_api_v3_registry_endpoints.py @@ -66,12 +66,16 @@ class TestRefreshPluginStore: assert response.status_code == 200 @pytest.mark.parametrize("key", ["fetch_commit_info", "fetch_latest_versions"]) - def test_either_commit_info_key_extends_the_message( + def test_commit_info_flag_claims_no_refresh_it_does_not_do( self, api_v3_client, api_v3_module, key): - # fetch_latest_versions is the older spelling; both must work. - api_v3_module.api_v3.plugin_store_manager.fetch_registry.return_value = {"plugins": []} + # The route only re-downloads the registry. It used to append "(with + # refreshed commit metadata from GitHub)" for either flag without + # fetching any. + store = api_v3_module.api_v3.plugin_store_manager + store.fetch_registry.return_value = {"plugins": [{"id": "a"}]} response = api_v3_client.post(self.URL, json={key: True}) - assert "commit metadata" in response.get_json()["message"] + assert response.get_json()["message"] == "Plugin store refreshed" + store.fetch_registry.assert_called_once_with(force_refresh=True) def test_message_stays_plain_without_the_flag(self, api_v3_client, api_v3_module): api_v3_module.api_v3.plugin_store_manager.fetch_registry.return_value = {"plugins": []} diff --git a/test/test_api_v3_vegas_plugin_lists.py b/test/test_api_v3_vegas_plugin_lists.py new file mode 100644 index 00000000..e5dd709c --- /dev/null +++ b/test/test_api_v3_vegas_plugin_lists.py @@ -0,0 +1,55 @@ +"""POST /config/main refuses a malformed Vegas plugin order or exclusion list. + +Both were parsed with ``except JSONDecodeError: ... = []``, so a bad value +cleared the saved list and answered 200. They now fail the save with a 400, +as plugin_rotation_order already did. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402 + +URL = "/api/v3/config/main" + + +@pytest.fixture +def saved(api_v3_module): + config = {"display": {"vegas_scroll": {"plugin_order": ["clock", "weather"], + "excluded_plugins": ["stocks"]}}} + cm = api_v3_module.api_v3.config_manager + cm.load_config.return_value = config + cm.save_config_atomic.return_value.status.value = 'success' + return cm + + +@pytest.mark.parametrize("field", ["vegas_plugin_order", "vegas_excluded_plugins"]) +@pytest.mark.parametrize("value", ["[not json", '{"a": 1}', "[1, 2]", 7]) +def test_malformed_list_is_refused_and_nothing_is_saved(api_v3_client, saved, field, value): + response = api_v3_client.post(URL, json={field: value}) + + assert response.status_code == 400, response.get_json() + assert field in response.get_json()["message"] + saved.save_config_atomic.assert_not_called() + saved.save_config.assert_not_called() + + +@pytest.mark.parametrize("value", ['["weather", "clock"]', ["weather", "clock"]]) +def test_json_text_or_array_is_stored(api_v3_client, saved, value): + response = api_v3_client.post(URL, json={"vegas_plugin_order": value}) + + assert response.status_code == 200, response.get_json() + stored = saved.save_config_atomic.call_args.args[0] + assert stored["display"]["vegas_scroll"]["plugin_order"] == ["weather", "clock"] + assert stored["display"]["vegas_scroll"]["excluded_plugins"] == ["stocks"] + + +def test_rotation_order_keeps_its_messages(api_v3_client, saved): + response = api_v3_client.post(URL, json={"plugin_rotation_order": "[oops"}) + + assert response.status_code == 400 + assert response.get_json()["message"] == "plugin_rotation_order must be valid JSON" diff --git a/test/test_api_v3_wifi_endpoints.py b/test/test_api_v3_wifi_endpoints.py index 2f1df946..38814fb4 100644 --- a/test/test_api_v3_wifi_endpoints.py +++ b/test/test_api_v3_wifi_endpoints.py @@ -408,6 +408,14 @@ class TestAutoEnableApMode: assert response.status_code == 400 assert "auto_enable_ap_mode" not in wifi_manager.config + def test_a_failed_save_is_reported(self, api_v3_client, wifi_manager): + # wifi_config.json left owned by root is the usual cause. + wifi_manager.config = {} + wifi_manager._save_config.return_value = False + response = api_v3_client.post(self.URL, json={"auto_enable_ap_mode": False}) + assert response.status_code == 500 + assert response.get_json()["status"] == "error" + class TestRadioEnabledAndForceAcceptIntegers: """`{"enabled": 1}` / `{"enabled": 0}` used to be mishandled: the old diff --git a/test/test_background_data_service.py b/test/test_background_data_service.py index ef066d7c..ed16bb1a 100644 --- a/test/test_background_data_service.py +++ b/test/test_background_data_service.py @@ -362,6 +362,3 @@ class TestPriorityIsAcceptedAndIgnored: rid = service.submit_fetch_request( "nfl", 2026, "http://example.invalid/x", cache_key="k", priority=5) assert service.get_result(rid).cached is True - - def test_statistics_still_report_an_empty_queue(self, service): - assert service.get_statistics()["queue_size"] == 0 diff --git a/test/test_backup_manager.py b/test/test_backup_manager.py index fb603d43..0097c213 100644 --- a/test/test_backup_manager.py +++ b/test/test_backup_manager.py @@ -180,6 +180,31 @@ def test_create_backup_manifest(project: Path, tmp_path: Path) -> None: assert set(manifest["contents"]) >= {"config", "secrets", "wifi", "fonts", "plugin_uploads", "plugins"} +def test_manifest_version_is_the_core_release(project: Path, tmp_path: Path) -> None: + """Not a git sha or a truncated "ref: refs/he..." read from .git/HEAD.""" + from src import __version__ + git = project / ".git" + git.mkdir() + (git / "HEAD").write_text("ref: refs/heads/some-branch-that-is-not-there\n", encoding="utf-8") + zip_path = create_backup(project, output_dir=tmp_path / "exports") + with zipfile.ZipFile(zip_path) as zf: + manifest = json.loads(zf.read("manifest.json")) + assert manifest["ledmatrix_version"] == __version__ + + +def test_installed_plugins_come_from_the_configured_directory(tmp_path: Path) -> None: + root = tmp_path / "proj" + (root / "config").mkdir(parents=True) + (root / "config" / "config.json").write_text( + json.dumps({"plugin_system": {"plugins_directory": "plugins"}}), encoding="utf-8") + plugin_dir = root / "plugins" / "dev-plugin" + plugin_dir.mkdir(parents=True) + (plugin_dir / "manifest.json").write_text( + json.dumps({"id": "dev-plugin", "version": "0.3.0"}), encoding="utf-8") + + assert [p["plugin_id"] for p in list_installed_plugins(root)] == ["dev-plugin"] + + # --------------------------------------------------------------------------- # Validate # --------------------------------------------------------------------------- diff --git a/test/test_base_odds_manager.py b/test/test_base_odds_manager.py index e53a5036..ad7cec4b 100644 --- a/test/test_base_odds_manager.py +++ b/test/test_base_odds_manager.py @@ -342,7 +342,6 @@ class TestLoadConfiguration: 'base_odds_manager': { 'update_interval': 100, 'timeout': 5, - 'cache_ttl': 42, } } @@ -352,7 +351,6 @@ class TestLoadConfiguration: # Key/attr mismatch pin: the config key is 'timeout' but the # attribute is request_timeout. assert manager.request_timeout == 5 - assert manager.cache_ttl == 42 def test_get_config_raising_keeps_defaults(self, cache_manager): config_manager = MagicMock() @@ -362,4 +360,3 @@ class TestLoadConfiguration: assert manager.update_interval == 3600 assert manager.request_timeout == 5 - assert manager.cache_ttl == 1800 diff --git a/test/test_config_validation_edge_cases.py b/test/test_config_validation_edge_cases.py index 040902cd..fa1fce13 100644 --- a/test/test_config_validation_edge_cases.py +++ b/test/test_config_validation_edge_cases.py @@ -299,3 +299,48 @@ class TestDefaultMerging: assert merged["enabled"] is False assert merged["display_duration"] == 60 + + +class TestMissingRequiredFields: + """One message per missing field, naming that field. + + A manual ``required`` loop used to run after Draft7Validator, which already + reports ``required``, so every missing top-level field was listed twice -- + and the validator's copy printed the schema's whole ``required`` list as + if it were the field name. + """ + + SCHEMA = { + "type": "object", + "properties": { + "api_key": {"type": "string"}, + "city": {"type": "string"}, + "units": {"type": "string"}, + }, + "required": ["api_key", "city", "units"], + } + + def test_each_missing_field_is_reported_once_by_name(self): + ok, errors = SchemaManager().validate_config_against_schema( + {"units": "metric"}, self.SCHEMA, "test-plugin") + + assert not ok + assert errors == [ + "Field root: Missing required property 'api_key'", + "Field root: Missing required property 'city'", + ] + + def test_nested_missing_field_names_the_field_and_its_parent(self): + schema = { + "type": "object", + "properties": {"nfl": { + "type": "object", + "properties": {"api_key": {"type": "string"}}, + "required": ["api_key"], + }}, + } + ok, errors = SchemaManager().validate_config_against_schema( + {"nfl": {}}, schema, "test-plugin") + + assert not ok + assert errors == ["Field 'nfl': Missing required property 'api_key'"] diff --git a/test/test_display_preview.py b/test/test_display_preview.py new file mode 100644 index 00000000..aaaebfe0 --- /dev/null +++ b/test/test_display_preview.py @@ -0,0 +1,79 @@ +"""GET /api/v3/display/current passes the snapshot PNG through untouched. + +It used to PIL-decode the snapshot and re-encode it, which cost CPU on the Pi +for no change in the picture, and it swallowed any read failure with +``except Exception: pass``. It now sends the file's own bytes, the payload the +/stream/display SSE stream sends, and logs a failed read. +""" + +import base64 +import io +import logging +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from flask import Flask +from PIL import Image, PngImagePlugin + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from web_interface import display_preview # noqa: E402 + + +@pytest.fixture +def client(monkeypatch): + from web_interface.blueprints.api_v3 import api_v3 + monkeypatch.setattr(api_v3, 'config_manager', MagicMock(), raising=False) + api_v3.config_manager.load_config.return_value = {} + app = Flask(__name__) + app.config['TESTING'] = True + app.register_blueprint(api_v3, url_prefix='/api/v3') + with app.test_client() as test_client: + yield test_client + + +@pytest.fixture +def snapshot(tmp_path, monkeypatch): + """A snapshot PNG carrying a text chunk, which a PIL re-encode drops.""" + info = PngImagePlugin.PngInfo() + info.add_text('written-by', 'display_manager') + buffer = io.BytesIO() + Image.new('RGB', (4, 2), (255, 0, 0)).save(buffer, format='PNG', pnginfo=info) + path = tmp_path / 'led_matrix_preview.png' + path.write_bytes(buffer.getvalue()) + monkeypatch.setattr(display_preview, 'SNAPSHOT_PATH', str(path)) + return path + + +def _image(client): + response = client.get('/api/v3/display/current') + assert response.status_code == 200 + return response.get_json()['data'] + + +def test_the_snapshot_bytes_are_sent_as_they_are(client, snapshot): + data = _image(client) + assert base64.b64decode(data['image']) == snapshot.read_bytes() + + +def test_route_and_stream_send_the_same_payload_keys(client, snapshot): + data = _image(client) + assert set(data) == set(display_preview.preview_payload(1, 1, None)) + + +def test_no_snapshot_is_a_null_image_without_a_warning(client, tmp_path, monkeypatch, caplog): + monkeypatch.setattr(display_preview, 'SNAPSHOT_PATH', str(tmp_path / 'missing.png')) + with caplog.at_level(logging.WARNING): + assert _image(client)['image'] is None + assert not [r for r in caplog.records if 'snapshot' in r.getMessage()] + + +def test_an_unreadable_snapshot_is_logged(client, snapshot, monkeypatch, caplog): + def denied(_path=None): + raise PermissionError('denied') + monkeypatch.setattr(display_preview, 'read_snapshot_base64', denied) + with caplog.at_level(logging.WARNING): + assert _image(client)['image'] is None + assert [r for r in caplog.records if 'snapshot' in r.getMessage() and r.exc_info] diff --git a/test/test_element_visibility_align_scale.py b/test/test_element_visibility_align_scale.py index 171f9f4e..3209bad4 100644 --- a/test/test_element_visibility_align_scale.py +++ b/test/test_element_visibility_align_scale.py @@ -185,3 +185,35 @@ class TestLogoScale: def test_an_unusable_scale_is_ignored(self, logo, bad): helper = LogoHelper(display_width=64, display_height=32) assert helper.load_logo("AAA", logo, 32, 32, scale=bad).size == (32, 32) + + def test_a_scale_the_schema_allows_is_applied(self, logo): + """The Scale field's maximum is honoured, not reset to 1.0.""" + from src.element_style import MAX_ELEMENT_SCALE + helper = LogoHelper(display_width=64, display_height=32) + big = helper.load_logo("AAA", logo, 4, 4, scale=MAX_ELEMENT_SCALE) + assert big.size == (40, 40) + + def test_a_scale_beyond_the_range_is_clamped(self, logo): + from src.element_style import MAX_ELEMENT_SCALE, MIN_ELEMENT_SCALE + helper = LogoHelper(display_width=64, display_height=32) + assert helper.load_logo("AAA", logo, 4, 4, scale=MAX_ELEMENT_SCALE * 3).size == (40, 40) + assert helper.load_logo("AAA", logo, 40, 40, scale=MIN_ELEMENT_SCALE / 2).size == (4, 4) + + +class TestScaleCoercion: + """One range for the schema, element_scale and LogoHelper.""" + + def test_schema_bounds_are_the_clamp_bounds(self): + from src.element_style import (MAX_ELEMENT_SCALE, MIN_ELEMENT_SCALE, + _offset_block_from_spec) + prop = _offset_block_from_spec("home_logo", {"scale": True})["properties"]["scale"] + assert (prop["minimum"], prop["maximum"]) == (MIN_ELEMENT_SCALE, MAX_ELEMENT_SCALE) + + @pytest.mark.parametrize("raw,expected", [ + (0.5, 0.5), (25, 10.0), (0.01, 0.1), + (0, 1.0), (-2, 1.0), ("x", 1.0), (True, 1.0), + (float("nan"), 1.0), (float("inf"), 1.0), + ]) + def test_element_scale_clamps_and_rejects(self, raw, expected): + cfg = {"customization": {"layout": {"home_logo": {"scale": raw}}}} + assert element_scale(cfg, "home_logo") == expected diff --git a/test/test_error_aggregator.py b/test/test_error_aggregator.py index 84bb89e3..2d8ca40e 100644 --- a/test/test_error_aggregator.py +++ b/test/test_error_aggregator.py @@ -124,6 +124,45 @@ class TestErrorRecording: assert aggregator._plugin_error_counts["plugin-a"]["ValueError"] == 2 assert aggregator._plugin_error_counts["plugin-b"]["ValueError"] == 1 + def test_stack_trace_recorded_outside_except_block(self): + """The trace comes from the exception, not from the handler in progress. + + plugin_executor records exceptions caught on a worker thread after + its except block has ended, where format_exc() only says + "NoneType: None". + """ + def failing_plugin_update(): + raise ValueError("boom") + + caught = [] + + def worker(): + try: + failing_plugin_update() + except ValueError as e: + caught.append(e) + + thread = threading.Thread(target=worker) + thread.start() + thread.join() + + record = ErrorAggregator().record_error(caught[0], plugin_id="p") + + assert "NoneType: None" not in record.stack_trace + assert "failing_plugin_update" in record.stack_trace + assert "ValueError: boom" in record.stack_trace + + def test_record_error_leaves_caller_context_unchanged(self): + """LEDMatrixError context is merged into a copy of the caller's dict.""" + context = {"caller": "value"} + error = PluginError("failed", plugin_id="p", context={"extra": 1}) + + record = ErrorAggregator().record_error(error, context=context) + + assert context == {"caller": "value"} + assert record.context["caller"] == "value" + assert record.context["extra"] == 1 + class TestPatternDetection: """Test error pattern detection.""" diff --git a/test/test_fix_web_permissions_rehardens.py b/test/test_fix_web_permissions_rehardens.py new file mode 100644 index 00000000..fc0855c6 --- /dev/null +++ b/test/test_fix_web_permissions_rehardens.py @@ -0,0 +1,109 @@ +"""scripts/fix_perms/fix_web_permissions.sh must not undo the installer's hardening. + +The script chowns the whole project to the web user. That used to include the +two helpers /etc/sudoers.d/ledmatrix_web lets the web user run as root +(safe_plugin_rm.sh, safe_pip_install.sh) -- a helper the web user owns is a +root shell for anyone who can edit it -- and config_secrets.json, which lost +the ledmatrix group first_time_install.sh gives it. After the chown the script +now puts both back the way the installer's Steps 11 and 11.1 leave them. + +The behavioural test runs the real script against a scratch copy of the +project with `sudo`, `getent` and `journalctl` stubbed, and checks the order +of what it asked sudo to do. +""" +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "fix_perms" / "fix_web_permissions.sh" +LIB = ROOT / "scripts" / "install" / "lib_sudoers.sh" + + +def _text(path): + return path.read_text(encoding="utf-8", errors="replace").replace("\r\n", "\n") + + +def _granted_helpers(): + helpers = set(re.findall(r"scripts/fix_perms/([\w.-]+\.sh) \*", _text(LIB))) + assert helpers, "no fix_perms helper grant found in lib_sudoers.sh" + return helpers + + +def test_every_granted_helper_is_rehardened_after_the_chown(): + text = _text(SCRIPT) + chown = text.index('sudo chown -R "$WEB_USER:$WEB_USER" "$PROJECT_DIR"') + loop = re.search(r"for helper in ([^;]+); do\n(.*?)\ndone", text, re.S) + assert loop, "no helper-hardening loop in fix_web_permissions.sh" + assert "sudo chown root:root" in loop.group(2) and "sudo chmod 755" in loop.group(2) + assert loop.start() > chown, "helpers are hardened before the chown that undoes it" + assert _granted_helpers() <= set(loop.group(1).split()) + + +def test_no_longer_claims_to_configure_sudoers(): + text = _text(SCRIPT) + assert "Configure sudoers for passwordless access" not in text + assert "./configure_web_sudo.sh" not in text.replace("scripts/install/configure_web_sudo.sh", "") + + +_STUB_SUDO = """#!/bin/bash +printf '%s\\n' "$*" >> "$SUDO_LOG" +# `sudo -n ...` probes and `sudo -u ...` tests: report failure, run nothing. +case "$1" in -n|-u) exit 1 ;; esac +exit 0 +""" + + +@pytest.mark.skipif(sys.platform == "win32" or shutil.which("bash") is None, + reason="needs a POSIX bash") +def test_script_rehardens_helpers_and_secrets(tmp_path): + project = tmp_path / "LED Matrix" + (project / "scripts" / "fix_perms").mkdir(parents=True) + (project / "config").mkdir() + script = project / "scripts" / "fix_perms" / "fix_web_permissions.sh" + script.write_text(_text(SCRIPT), encoding="utf-8") + for helper in ("safe_plugin_rm.sh", "safe_pip_install.sh"): + (project / "scripts" / "fix_perms" / helper).write_text("#!/bin/bash\n") + (project / "config" / "config_secrets.json").write_text("{}\n") + + stubs = tmp_path / "stubs" + stubs.mkdir() + for name, body in (("sudo", _STUB_SUDO), + ("getent", "#!/bin/sh\nexit 0\n"), + ("journalctl", "#!/bin/sh\nexit 1\n")): + (stubs / name).write_text(body) + (stubs / name).chmod(0o755) + log = tmp_path / "sudo.log" + env = dict(os.environ, SUDO_LOG=str(log), + PATH=os.pathsep.join([str(stubs), os.environ.get("PATH", "")])) + + result = subprocess.run(["bash", str(script)], input="y", env=env, + capture_output=True, text=True) + if os.geteuid() == 0: + # The script refuses to run as root; that refusal is the whole test. + assert result.returncode == 1 and "should not be run as root" in result.stdout + return + assert result.returncode == 0, result.stdout + result.stderr + + calls = log.read_text().splitlines() + user = subprocess.run(["whoami"], capture_output=True, text=True).stdout.strip() + chown_all = calls.index(f"chown -R {user}:{user} {project}") + for helper in ("safe_plugin_rm.sh", "safe_pip_install.sh"): + path = project / "scripts" / "fix_perms" / helper + assert calls.index(f"chown root:root {path}") > chown_all, calls + assert calls.index(f"chmod 755 {path}") > chown_all, calls + secrets = project / "config" / "config_secrets.json" + # The owner is the installed web unit's User= when there is one. + owner = user + unit = Path("/etc/systemd/system/ledmatrix-web.service") + if unit.is_file(): + m = re.search(r"^User=(.*)$", unit.read_text(), re.M) + if m and m.group(1): + owner = m.group(1) + assert calls.index(f"chown {owner}:ledmatrix {secrets}") > chown_all, calls + assert calls.index(f"chmod 640 {secrets}") > chown_all, calls diff --git a/test/test_font_manager.py b/test/test_font_manager.py index 232361ce..e9e80c7b 100644 --- a/test/test_font_manager.py +++ b/test/test_font_manager.py @@ -8,10 +8,14 @@ test here asserts observable behavior: returned font types, cache identity, fallback selection, and BDF native-size reading. """ +import json +import shutil + import freetype import pytest from PIL import ImageFont +from src.common.font_layout import resolve_asset_path from src.font_manager import FontManager @@ -132,3 +136,37 @@ class TestCacheLifecycle: fm.reload_config({}) assert fm.cache_generation == gen_before + 1 assert not fm.font_cache + + +class TestPluginFonts: + """plugin:// sources resolve against the plugin's own directory, which + by default lives under plugin-repos/, not a cwd-relative plugins/.""" + + MANIFEST = {"fonts": [{"family": "bundled", "source": "plugin://fonts/Bundled.ttf"}]} + + @staticmethod + def _plugin_with_font(root, name="my-plugin"): + plugin_dir = root / name + (plugin_dir / "fonts").mkdir(parents=True) + (plugin_dir / "manifest.json").write_text(json.dumps({"id": "my-plugin"})) + shutil.copy(resolve_asset_path("assets/fonts/PressStart2P-Regular.ttf"), + plugin_dir / "fonts" / "Bundled.ttf") + return plugin_dir + + def test_font_resolves_under_the_given_plugin_dir(self, fm, tmp_path): + plugin_dir = self._plugin_with_font(tmp_path / "plugin-repos") + + assert fm.register_plugin_fonts("my-plugin", self.MANIFEST, plugin_dir=plugin_dir) + + assert fm.font_catalog["my-plugin::bundled"] == str(plugin_dir / "fonts" / "Bundled.ttf") + font = fm.resolve_font("x.y", "bundled", 8, plugin_id="my-plugin") + assert isinstance(font, ImageFont.FreeTypeFont) + + def test_without_a_plugin_dir_the_configured_directory_is_searched(self, tmp_path): + plugins_root = tmp_path / "installed" + plugin_dir = self._plugin_with_font(plugins_root, name="ledmatrix-my-plugin") + fm = FontManager({"plugin_system": {"plugins_directory": str(plugins_root)}}) + + assert fm.register_plugin_fonts("my-plugin", self.MANIFEST) + + assert fm.font_catalog["my-plugin::bundled"] == str(plugin_dir / "fonts" / "Bundled.ttf") diff --git a/test/test_font_pixel_grid.py b/test/test_font_pixel_grid.py index 3889d91a..2ff72c0f 100644 --- a/test/test_font_pixel_grid.py +++ b/test/test_font_pixel_grid.py @@ -133,6 +133,26 @@ class TestAssetPathsIgnoreTheWorkingDirectory: rel = f"assets/fonts/{FOUR_BY_SIX}" assert FontManager._resolve_asset_path(rel) == resolve_asset_path(rel) + def test_font_overrides_file_lives_in_the_install_config(self, tmp_path, monkeypatch): + from src.font_manager import FontManager + monkeypatch.chdir(tmp_path) + fm = FontManager({}) + assert fm.font_overrides_file == str(PROJECT_ROOT / "config" / "font_overrides.json") + + def test_logo_placeholder_draws_with_the_bundled_font(self, tmp_path, monkeypatch): + import src.logo_downloader as logo_downloader + from src.logo_downloader import LogoDownloader + loaded = [] + + def spy(font, size, **kwargs): + loaded.append(font) + return load_truetype(font, size, **kwargs) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(logo_downloader, "load_truetype", spy) + assert LogoDownloader().create_placeholder_logo("AB", str(tmp_path)) + assert loaded == [str(PROJECT_ROOT / "assets" / "fonts" / PRESS_START)] + class TestTheHarnessForkAgreesWithTheCore: """The divergence that let the wrong rendering be blessed as golden. diff --git a/test/test_http_headers.py b/test/test_http_headers.py index bc5836a1..bc4f55c6 100644 --- a/test/test_http_headers.py +++ b/test/test_http_headers.py @@ -78,3 +78,25 @@ class TestBackgroundDataServiceHeaders: assert 'yourusername' not in str(headers) finally: service.shutdown(wait=False) + + +class TestResolverHeaders: + def test_dynamic_team_resolver_sends_the_user_agent(self): + from unittest.mock import patch + from src.dynamic_team_resolver import DynamicTeamResolver + DynamicTeamResolver._rankings_cache = {} + DynamicTeamResolver._cache_timestamp = 0 + try: + with patch('src.dynamic_team_resolver.requests.get', + side_effect=RuntimeError("stop")) as get: + DynamicTeamResolver().resolve_teams(["AP_TOP_5"]) + assert get.call_args.kwargs['headers']['User-Agent'] == USER_AGENT + finally: + DynamicTeamResolver._rankings_cache = {} + DynamicTeamResolver._cache_timestamp = 0 + + def test_odds_manager_uses_the_shared_headers(self): + from src.base_odds_manager import BaseOddsManager + headers = BaseOddsManager(MagicMock()).session.headers + for name, value in DEFAULT_HTTP_HEADERS.items(): + assert headers[name] == value diff --git a/test/test_install_reboot_is_last.py b/test/test_install_reboot_is_last.py new file mode 100644 index 00000000..3ae62832 --- /dev/null +++ b/test/test_install_reboot_is_last.py @@ -0,0 +1,134 @@ +"""first_time_install.sh prints its completion summary before it reboots. + +With -y (and so with the one-shot `curl | bash` installer, which always +passes -y) the reboot used to be issued ~180 lines before the "Installation +Complete / Web UI Access" summary. `reboot` returns at once and the script +carried on printing while the system went down, so the SSH session usually +dropped before the user saw the web UI address. + +first_time_install.sh exits on anything but Raspberry Pi OS Trixie before it +parses its arguments, so the behavioural test runs only the tail of the +script -- from the summary to the end -- with systemctl, nmcli, hostname, ip +and reboot stubbed. +""" +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +FIRST_TIME = ROOT / "first_time_install.sh" +SUMMARY_START = 'echo "Installation Complete!"' + + +def _text(): + return FIRST_TIME.read_text(encoding="utf-8").replace("\r\n", "\n") + + +def test_every_reboot_comes_after_the_summary(): + text = _text() + summary = text.index(SUMMARY_START) + lines = text.splitlines() + reboots = [i for i, line in enumerate(lines) if line.strip() == "reboot"] + assert reboots, "no reboot call found" + summary_line = text[:summary].count("\n") + enjoy_line = text[:text.index('echo "Enjoy your LED Matrix display!"')].count("\n") + assert all(i > enjoy_line > summary_line for i in reboots), ( + f"reboot at line(s) {[i + 1 for i in reboots]} runs before the summary " + f"(line {summary_line + 1}) has finished printing") + + +def _tail(): + """The script from the summary header to the end, header rule included.""" + text = _text() + start = text.rindex('echo "=========================================="', 0, + text.index(SUMMARY_START)) + return text[start:] + + +_POSIX = pytest.mark.skipif(sys.platform == "win32" or shutil.which("bash") is None, + reason="needs a POSIX bash") + + +def _run(tmp_path, env_extra, nmcli_active_line=True, stdin="", hostapd_active=False): + stubs = tmp_path / "stubs" + stubs.mkdir() + log = tmp_path / "calls.log" + active = 'echo "yes:HomeNet"' if nmcli_active_line else ":" + hostapd = 'case "$*" in *"is-active --quiet hostapd"*) exit 0 ;; esac\n' if hostapd_active else "" + bodies = { + "reboot": f'#!/bin/sh\necho REBOOT-CALLED\necho reboot >> "{log}"\n', + "systemctl": f"#!/bin/sh\n{hostapd}exit 3\n", + "hostname": '#!/bin/sh\necho "192.168.1.50 fe80::1"\n', + "ip": "#!/bin/sh\nexit 1\n", + # device status -> one connected wifi device; device wifi -> active line + "nmcli": ('#!/bin/sh\ncase "$*" in\n' + ' *"device status"*) echo "wlan0:wifi:connected" ;;\n' + f' *"device wifi"*) {active} ;;\n' + "esac\n"), + } + for name, body in bodies.items(): + (stubs / name).write_text(body) + (stubs / name).chmod(0o755) + script = "\n".join([ + "set -Eeuo pipefail", + "on_error() { echo \"ERR-TRAP line $1\" >&2; exit 1; }", + "trap 'on_error $LINENO' ERR", + "PROJECT_ROOT_DIR=/home/pi/LEDMatrix", + "ASSUME_YES=${ASSUME_YES:-0}", + "SKIP_REBOOT_PROMPT=${SKIP_REBOOT_PROMPT:-0}", + _tail(), + ]) + env = dict(os.environ, PATH=os.pathsep.join([str(stubs), "/usr/bin", "/bin"]), **env_extra) + result = subprocess.run(["bash", "-c", script], env=env, input=stdin, + capture_output=True, text=True) + calls = log.read_text().splitlines() if log.exists() else [] + return result, calls + + +@_POSIX +@pytest.mark.parametrize("nmcli_active_line", [True, False], ids=["ssid", "no-ssid"]) +def test_assume_yes_prints_the_summary_then_reboots(tmp_path, nmcli_active_line): + result, calls = _run(tmp_path, {"ASSUME_YES": "1"}, nmcli_active_line) + out = result.stdout + assert result.returncode == 0, out + result.stderr + assert calls == ["reboot"] + for text in ("Installation Complete!", "Web UI Access:", "http://192.168.1.50:5000", + "Enjoy your LED Matrix display!"): + assert out.index(text) < out.index("REBOOT-CALLED"), text + assert "Password: ledmatrix123" not in out + + +@_POSIX +def test_setup_access_point_is_described_as_open(tmp_path): + """wifi_manager creates the setup AP with no security ("No password" on + the panel); the summary used to print a password it does not have.""" + result, _ = _run(tmp_path, {"ASSUME_YES": "1"}, hostapd_active=True) + assert result.returncode == 0, result.stdout + result.stderr + assert "AP Mode is ACTIVE" in result.stdout + assert "Open network, no password" in result.stdout + assert "Password:" not in result.stdout + + +@_POSIX +def test_no_reboot_prompt_prints_the_summary_and_does_not_reboot(tmp_path): + result, calls = _run(tmp_path, {"ASSUME_YES": "1", "SKIP_REBOOT_PROMPT": "1"}) + assert result.returncode == 0, result.stdout + result.stderr + assert calls == [] + assert "Enjoy your LED Matrix display!" in result.stdout + assert "Skipping reboot prompt" in result.stdout + + +@_POSIX +@pytest.mark.parametrize("answer,expected", [("y", ["reboot"]), ("n", [])]) +def test_interactive_prompt_comes_after_the_summary(tmp_path, answer, expected): + result, calls = _run(tmp_path, {}, stdin=answer) + assert result.returncode == 0, result.stdout + result.stderr + assert calls == expected + out = result.stdout + assert "Enjoy your LED Matrix display!" in out + if expected: + assert out.index("Enjoy your LED Matrix display!") < out.index("REBOOT-CALLED") diff --git a/test/test_live_status_fields.py b/test/test_live_status_fields.py index 571683de..8f94f20f 100644 --- a/test/test_live_status_fields.py +++ b/test/test_live_status_fields.py @@ -31,8 +31,12 @@ import pytest PROJECT_ROOT = Path(__file__).parent.parent sys.path.insert(0, str(PROJECT_ROOT)) +from web_interface import system_metrics # noqa: E402 from web_interface.system_metrics import collect_system_metrics # noqa: E402 +MB = 1024 * 1024 +GB = 1024 * MB + @pytest.fixture def metrics(monkeypatch): @@ -45,10 +49,11 @@ def metrics(monkeypatch): lambda _p: (_ for _ in ()).throw(OSError("no such mount"))) else: monkeypatch.setattr(psutil, "disk_usage", - lambda _p: SimpleNamespace(percent=disk_percent)) + lambda _p: SimpleNamespace(percent=disk_percent, + total=32 * GB, used=4 * GB)) monkeypatch.setattr(psutil, "virtual_memory", - lambda: SimpleNamespace(percent=used_percent, - available=available_bytes)) + lambda: SimpleNamespace(percent=used_percent, total=1024 * MB, + used=600 * MB, available=available_bytes)) return collect_system_metrics() return _collect @@ -101,3 +106,18 @@ class TestEveryPredictiveFieldIsPresent: "memory_available_mb", "disk_used_percent"): assert field in m assert m["disk_used_percent"] is None + assert all(m[key] is None for key in m if key != "cpu_temp") + + +class TestUnavailableIsNull: + """One answer for "could not be read": None, never 0.""" + + def test_unreadable_temperature_is_null_not_zero_degrees(self, metrics, monkeypatch): + monkeypatch.setattr(system_metrics, "_THERMAL_ZONE", "/nonexistent/thermal/temp") + assert metrics()["cpu_temp"] is None + + def test_readable_temperature_is_degrees_c(self, metrics, monkeypatch, tmp_path): + zone = tmp_path / "temp" + zone.write_text("48312\n") + monkeypatch.setattr(system_metrics, "_THERMAL_ZONE", str(zone)) + assert metrics()["cpu_temp"] == 48.3 diff --git a/test/test_logo_downloader.py b/test/test_logo_downloader.py index a303b087..581997bd 100644 --- a/test/test_logo_downloader.py +++ b/test/test_logo_downloader.py @@ -357,6 +357,33 @@ class TestRefreshPlaceholderTimestamp: assert refresh_placeholder_timestamp(tmp_path / "nope.png") is False +class TestFailurePaths: + def test_a_team_without_logos_is_a_failed_download(self, tmp_path): + downloader = LogoDownloader() + with patch.object(downloader, "fetch_single_team", + return_value={"team": {"logos": []}}): + assert downloader.download_missing_logo_for_team( + "nfl", "1", "XYZ", tmp_path / "XYZ.png") is False + + def test_placeholder_is_written_where_the_caller_looks(self, tmp_path): + """A path that is not .png (the plugin's own + file naming, or an abbreviation normalize_abbreviation rewrites) + still ends up holding the placeholder, so True means it exists.""" + logo_path = tmp_path / "TA&M.png" + with patch.object(LogoDownloader, "download_logo", return_value=False): + assert download_missing_logo( + "ncaa_fb", "245", "TA&M", logo_path, + logo_url="http://example/tamu.png") is True + assert is_placeholder_logo(logo_path) + assert not (tmp_path / "TAANDM.png").exists() + + def test_placeholder_uses_the_placeholder_geometry(self, tmp_path): + assert LogoDownloader().create_placeholder_logo("AB", str(tmp_path)) + with Image.open(tmp_path / "AB.png") as img: + assert img.size == PLACEHOLDER_SIZE + assert img.convert("RGBA").getpixel((0, 0)) == PLACEHOLDER_BG + + # --------------------------------------------------------------------------- # download_logo: the download the scoreboard plugins actually use # diff --git a/test/test_metrics_cache_unknown_fields.py b/test/test_metrics_cache_unknown_fields.py index 729893a3..3dd4e1d3 100644 --- a/test/test_metrics_cache_unknown_fields.py +++ b/test/test_metrics_cache_unknown_fields.py @@ -116,10 +116,14 @@ def test_values_of_the_wrong_type_fall_back_to_usable_defaults(bad): assert isinstance(getattr(metrics, field_name), (int, float)), \ f"{field_name} came back as {getattr(metrics, field_name)!r}" - # The real proof: arithmetic on the loaded metrics must not explode. + # The real proof: the arithmetic monitor_call and get_metrics_summary do + # on the loaded metrics must not explode. metrics.call_count += 1 metrics.total_execution_time += 0.5 - metrics.update_average_execution_time() + metrics.max_execution_time = max(metrics.max_execution_time, 0.5) + metrics.min_execution_time = min(metrics.min_execution_time, 0.5) + metrics.memory_mb = max(metrics.memory_mb, 1.0) + assert metrics.total_execution_time / metrics.call_count >= 0 def test_a_numeric_string_is_accepted_rather_than_discarded(): diff --git a/test/test_pages_v3_partials.py b/test/test_pages_v3_partials.py new file mode 100644 index 00000000..83fcf2ab --- /dev/null +++ b/test/test_pages_v3_partials.py @@ -0,0 +1,65 @@ +"""/partials/ dispatch and the plugin web_ui page's directory lookup.""" + +import logging +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from flask import Flask + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from web_interface.blueprints import pages_v3 as module # noqa: E402 + + +@pytest.fixture +def client(tmp_path, monkeypatch): + plugin_manager = MagicMock() + plugin_manager.plugins_dir = tmp_path + monkeypatch.setattr(module.pages_v3, "plugin_manager", plugin_manager, raising=False) + monkeypatch.setattr(module.pages_v3, "config_manager", + MagicMock(load_config=lambda: {}), raising=False) + app = Flask(__name__, template_folder=str( + Path(module.__file__).resolve().parents[1] / "templates")) + app.register_blueprint(module.pages_v3) + return app.test_client() + + +def test_every_tab_the_old_if_chain_served_is_still_served(): + assert set(module._PARTIAL_LOADERS) == { + "overview", "general", "display", "durations", "schedule", "plugins", + "fonts", "logs", "raw-json", "backup-restore", "wifi", "cache", + "operation-history", "tools", + } + + +def test_an_unknown_partial_is_a_404(client): + response = client.get("/partials/nope") + assert response.status_code == 404 + + +def test_a_failing_loader_is_one_500_that_names_the_partial(client, monkeypatch, caplog): + def boom(): + raise RuntimeError("template exploded") + monkeypatch.setitem(module._PARTIAL_LOADERS, "logs", boom) + + with caplog.at_level(logging.ERROR): + response = client.get("/partials/logs") + + assert response.status_code == 500 + assert response.get_data(as_text=True) == "Error loading partial" + errors = [r for r in caplog.records if r.levelno >= logging.ERROR] + assert len(errors) == 1 + assert errors[0].getMessage() == "Error loading partial logs" + + +def test_web_ui_page_uses_the_ledmatrix_prefix_fallback(client, tmp_path): + web_ui = tmp_path / "ledmatrix-radar" / "web_ui" + web_ui.mkdir(parents=True) + (web_ui / "panel.html").write_text("

radar panel

", encoding="utf-8") + + response = client.get("/plugin-ui/radar/web-ui/panel.html") + + assert response.status_code == 200 + assert "radar panel" in response.get_data(as_text=True) diff --git a/test/test_plugin_loading_failures.py b/test/test_plugin_loading_failures.py index 0bb3e0d7..8035ac3a 100644 --- a/test/test_plugin_loading_failures.py +++ b/test/test_plugin_loading_failures.py @@ -294,6 +294,31 @@ class TestValidateConfigFailure: assert result is False +class TestPluginFontRegistration: + """A manifest's fonts block is registered against the directory the + plugin was loaded from, which plugin:// font sources are relative to.""" + + def test_plugin_dir_is_passed_to_the_font_manager(self, temp_plugin_dir, mock_managers): + plugin_dir = temp_plugin_dir / "test-plugin" + plugin_dir.mkdir() + fonts = {"fonts": [{"family": "f", "source": "plugin://f.ttf"}]} + manifest = {"id": "test-plugin", "name": "Test Plugin", + "entry_point": "manager.py", "class_name": "TestPlugin", + "fonts": fonts} + + with patch('src.common.permission_utils.ensure_directory_permissions'): + manager = PluginManager(plugins_dir=str(temp_plugin_dir), **mock_managers) + manager.plugin_manifests["test-plugin"] = manifest + with patch.object(manager.plugin_loader, 'load_plugin', + return_value=(MagicMock(), MagicMock())): + with patch.object(manager.plugin_loader, 'find_plugin_directory', + return_value=plugin_dir): + manager.load_plugin("test-plugin") + + mock_managers["font_manager"].register_plugin_fonts.assert_called_once_with( + "test-plugin", fonts, plugin_dir=plugin_dir) + + class TestPluginStateOnFailure: """Test that plugin state is correctly set on various failures.""" diff --git a/test/test_plugin_manager_reload.py b/test/test_plugin_manager_reload.py new file mode 100644 index 00000000..be8f25db --- /dev/null +++ b/test/test_plugin_manager_reload.py @@ -0,0 +1,36 @@ +"""reload_plugin re-reads the manifest from the plugin's actual directory. + +It read ``plugins_dir / plugin_id / manifest.json``, but a plugin directory's +name need not be the id its manifest declares -- discovery maps ids to +directories for exactly that reason. For such a plugin the path did not exist, +the re-read was skipped silently, and the reload kept the stale manifest. +""" + +import json + +import pytest + +from src.plugin_system.plugin_manager import PluginManager + + +@pytest.fixture +def pm_with_renamed_dir(tmp_path): + plugins_dir = tmp_path / "plugins" + plugin_dir = plugins_dir / "stock-ticker-v2" + plugin_dir.mkdir(parents=True) + manifest_path = plugin_dir / "manifest.json" + manifest_path.write_text(json.dumps({"id": "stocks", "version": "1.0.0"})) + pm = PluginManager(plugins_dir=str(plugins_dir)) + assert pm.discover_plugins() == ["stocks"] + return pm, manifest_path + + +def test_reload_picks_up_an_edited_manifest(pm_with_renamed_dir, monkeypatch): + pm, manifest_path = pm_with_renamed_dir + manifest_path.write_text(json.dumps({"id": "stocks", "version": "2.0.0"})) + loaded = [] + monkeypatch.setattr(pm, "load_plugin", lambda pid: loaded.append(pid) or True) + + assert pm.reload_plugin("stocks") is True + assert pm.plugin_manifests["stocks"]["version"] == "2.0.0" + assert loaded == ["stocks"] diff --git a/test/test_plugin_state_transition_count.py b/test/test_plugin_state_transition_count.py index ae2ba05f..90ea40dd 100644 --- a/test/test_plugin_state_transition_count.py +++ b/test/test_plugin_state_transition_count.py @@ -115,5 +115,22 @@ def test_get_state_info_is_a_consistent_snapshot(): assert not inconsistent, f"observed a torn snapshot: {inconsistent[:1]}" +def test_state_info_reports_only_what_something_records(): + """No field that is always null. + + ``last_display`` was reported here, but nothing ever recorded a display() + call, so it was null for every plugin. Its only reader is the web process, + whose PluginManager never calls display(), so recording it in the display + process could not have filled it either. + """ + manager = PluginStateManager() + manager.set_state("clock", PluginState.ENABLED) + manager.record_update("clock") + + info = manager.get_state_info("clock") + assert "last_display" not in info + assert info["last_update"] is not None + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/test_repo_urls.py b/test/test_repo_urls.py new file mode 100644 index 00000000..3e960bf7 --- /dev/null +++ b/test/test_repo_urls.py @@ -0,0 +1,95 @@ +"""Repository URL handling shared by the plugin store and saved repositories. + +The store cleaned URLs with ``url.rstrip('/').replace('.git', '')`` in two +places, which removes ``.git`` anywhere in the URL: +``https://github.com/user/my.github.io`` became ``.../myhub.io``, so installing +or browsing such a repository asked GitHub for one that does not exist. +""" + +from unittest.mock import MagicMock + +import pytest + +from src.plugin_system.repo_urls import ( + github_api_headers, github_owner_repo, normalize_repo_url, same_repo, +) +from src.plugin_system.store_manager import PluginStoreManager + +PAGES_REPO = "https://github.com/user/my.github.io" + + +class TestNormalizeRepoUrl: + @pytest.mark.parametrize("raw, expected", [ + (PAGES_REPO, PAGES_REPO), + (PAGES_REPO + ".git", PAGES_REPO), + ("https://github.com/user/repo.git/", "https://github.com/user/repo"), + (" https://github.com/user/repo/ ", "https://github.com/user/repo"), + ]) + def test_only_a_trailing_dot_git_is_removed(self, raw, expected): + assert normalize_repo_url(raw) == expected + + def test_same_repo_ignores_case_and_suffix(self): + assert same_repo("https://github.com/Owner/Repo.git", + "https://github.com/owner/repo/") + assert not same_repo("https://github.com/owner/repo", + "https://github.com/owner/other") + + +class TestGithubOwnerRepo: + @pytest.mark.parametrize("url, expected", [ + (PAGES_REPO + ".git", ("user", "my.github.io")), + ("https://www.github.com/owner/repo", ("owner", "repo")), + ("https://github.com/owner/repo/tree/main/plugins/x", ("owner", "repo")), + ]) + def test_github_urls(self, url, expected): + assert github_owner_repo(url) == expected + + @pytest.mark.parametrize("url", [ + "https://github.com.example.org/owner/repo", + "https://gitlab.com/owner/repo", + "https://github.com/owner", + "github.com/owner/repo", + ]) + def test_anything_else_is_not_a_github_repo(self, url): + assert github_owner_repo(url) is None + + def test_headers_carry_the_token_only_when_given(self): + assert "Authorization" not in github_api_headers(None) + assert github_api_headers("abc")["Authorization"] == "token abc" + + +@pytest.fixture +def store(tmp_path): + return PluginStoreManager( + plugins_dir=str(tmp_path / "plugins"), + uninstalled_registry_path=str(tmp_path / "uninstalled.json")) + + +def test_install_from_url_keeps_an_interior_dot_git(store, monkeypatch): + cloned_from = [] + monkeypatch.setattr(store, "_install_via_git", + lambda url, *a, **k: cloned_from.append(url)) + downloaded = [] + monkeypatch.setattr(store, "_install_via_download", + lambda url, *a, **k: downloaded.append(url) or False) + + result = store.install_from_url(PAGES_REPO + ".git") + + assert result["success"] is False + assert cloned_from == [PAGES_REPO] + assert all(url.startswith(PAGES_REPO + "/archive/") for url in downloaded) + + +def test_fetch_registry_from_url_asks_for_the_named_repository(store, monkeypatch): + requested = [] + + def fake_get(url, **kwargs): + requested.append(url) + return MagicMock(status_code=404) + + monkeypatch.setattr(store, "_http_get_with_retries", fake_get) + + assert store.fetch_registry_from_url(PAGES_REPO) is None + assert requested + assert all(url.startswith("https://raw.githubusercontent.com/user/my.github.io/") + for url in requested) diff --git a/test/test_resource_monitor.py b/test/test_resource_monitor.py index 2f651d1d..5e913210 100644 --- a/test/test_resource_monitor.py +++ b/test/test_resource_monitor.py @@ -93,6 +93,27 @@ class TestResourceLimits: with pytest.raises(ResourceLimitExceeded): mon.monitor_call("p", lambda: time.sleep(0.02)) + def test_memory_limit_judges_each_call_on_its_own_growth(self): + """One expensive call must not fail every call after it. + + The check used to compare the stored high-water mark, which never + decreases, so after one call grew memory past the limit every later + call raised too and the plugin never updated again. + """ + mon = PluginResourceMonitor(_cache(), enable_monitoring=False) + mon.enable_monitoring = True # measure without needing psutil + readings = iter([100.0, 200.0, # first call grows RSS by 100 MB + 200.0, 201.0]) # second call grows it by 1 MB + mon._get_process_memory_mb = lambda: next(readings) + mon._get_process_cpu_percent = lambda: 0.0 + mon.set_limits("p", ResourceLimits(max_memory_mb=50)) + + with pytest.raises(ResourceLimitExceeded): + mon.monitor_call("p", lambda: None) + assert mon.monitor_call("p", lambda: "ok") == "ok" + # The high-water mark is still reported. + assert mon.get_metrics("p").memory_mb == 100.0 + def test_reset_metrics_clears_counts(self): cache = _cache() mon = PluginResourceMonitor(cache, enable_monitoring=False) diff --git a/test/test_saved_repositories.py b/test/test_saved_repositories.py index 39bed1e4..0736c44c 100644 --- a/test/test_saved_repositories.py +++ b/test/test_saved_repositories.py @@ -5,7 +5,7 @@ SavedRepositoriesManager contract. Covers: the three accepted on-disk load shapes (bare list, wrapped {"repositories": [...]}, anything else -> []) and that saves always write the bare-list form; add/remove/has round trips through a fresh manager; -URL normalization post-fix (_clean_url strips only a TRAILING '.git' after +URL normalization post-fix (normalize_repo_url strips only a TRAILING '.git' after trailing slashes — the old unanchored .replace('.git', '') mangled URLs like my.github.io); name derivation and registry-vs-single type classification (the ledmatrix-plugins check is lowercased, the diff --git a/test/test_startup_validator.py b/test/test_startup_validator.py index 43cc2f38..2bda887f 100644 --- a/test/test_startup_validator.py +++ b/test/test_startup_validator.py @@ -63,6 +63,15 @@ class TestValidateConfig: assert "Missing required configuration key: display" in errors assert "Missing required configuration key: timezone" in errors + @pytest.mark.parametrize("config,expected", [ + ({'timezone': 'UTC'}, "Missing required configuration key: display"), + ({'display': {}, 'timezone': 'UTC'}, "Display configuration is empty"), + ]) + def test_a_missing_display_section_is_reported_once(self, good_cache, config, expected): + validator = StartupValidator(make_config_manager(config)) + _, errors, _ = validator.validate_all() + assert errors == [expected] + def test_config_error_does_not_propagate(self, good_cache): mgr = make_config_manager(GOOD_CONFIG) mgr.load_config.side_effect = ConfigError("bad json") diff --git a/test/test_store_install_default_branch.py b/test/test_store_install_default_branch.py new file mode 100644 index 00000000..8997e444 --- /dev/null +++ b/test/test_store_install_default_branch.py @@ -0,0 +1,77 @@ +"""A repository whose only branch is neither main nor master still installs. + +_install_via_git tries the candidate branches, then the repository's default +branch -- but it returned None both for "every clone failed" and for "the +default-branch clone succeeded". install_from_url took the None as failure, +fell through to the archive download of main/master (which does not exist), +and reported "Failed to clone or download repository" for a repository it had +just cloned. +""" + +import json +import shutil +import subprocess + +import pytest + +from src.plugin_system.store_manager import PluginStoreManager + +pytestmark = pytest.mark.skipif(shutil.which("git") is None, reason="git not installed") + +MANIFEST = { + "id": "develop-only", "name": "Develop Only", "class_name": "P", + "display_modes": ["develop_only"], "version": "1.0.0", +} + + +def _git(*args, cwd): + subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@example.invalid", *args], + cwd=cwd, check=True, capture_output=True) + + +@pytest.fixture +def develop_only_repo(tmp_path): + repo = tmp_path / "upstream" + repo.mkdir() + _git("init", "-q", "-b", "develop", cwd=repo) + (repo / "manifest.json").write_text(json.dumps(MANIFEST)) + (repo / "manager.py").write_text("class P: pass\n") + _git("add", ".", cwd=repo) + _git("commit", "-q", "-m", "init", cwd=repo) + return repo.as_uri() + + +@pytest.fixture +def store(tmp_path, monkeypatch): + mgr = PluginStoreManager( + plugins_dir=str(tmp_path / "plugins"), + uninstalled_registry_path=str(tmp_path / "uninstalled.json")) + monkeypatch.setattr(mgr, "_install_dependencies", lambda *a, **k: True) + downloads = [] + monkeypatch.setattr(mgr, "_install_via_download", + lambda url, *a, **k: downloads.append(url) or False) + mgr.downloads = downloads + return mgr + + +def test_a_default_branch_clone_reports_its_branch(store, develop_only_repo, tmp_path): + target = tmp_path / "clone" + assert store._install_via_git(develop_only_repo, target, ["main", "master"]) == "develop" + assert (target / "manifest.json").exists() + + +def test_a_failed_clone_reports_none(store, tmp_path): + missing = (tmp_path / "no-such-repo").as_uri() + target = tmp_path / "clone" + assert store._install_via_git(missing, target, ["main"]) is None + assert not target.exists() + + +def test_install_from_url_installs_a_develop_only_repository(store, develop_only_repo): + result = store.install_from_url(develop_only_repo) + + assert result == {"success": True, "plugin_id": "develop-only", + "name": "Develop Only", "branch": "develop"} + assert (store.plugins_dir / "develop-only" / "manifest.json").exists() + assert store.downloads == [] diff --git a/test/test_store_update_non_git_remote.py b/test/test_store_update_non_git_remote.py new file mode 100644 index 00000000..21be96b7 --- /dev/null +++ b/test/test_store_update_non_git_remote.py @@ -0,0 +1,62 @@ +"""update_plugin must not borrow the enclosing LEDMatrix checkout's remote. + +Plugins live in ``plugin-repos/`` inside the LEDMatrix git checkout. For a +plugin installed from a ZIP (no ``.git`` of its own), ``git -C `` +walks up to the LEDMatrix repository, and ``git config --local --get +remote.origin.url`` answers with LEDMatrix's own URL. update_plugin then tried +to "reinstall" the plugin from the LEDMatrix repository. +""" + +import json +import shutil +import subprocess + +import pytest + +from src.plugin_system.store_manager import PluginStoreManager + +pytestmark = pytest.mark.skipif(shutil.which("git") is None, reason="git not installed") + +PLUGIN_ID = "zip-installed" +PARENT_REMOTE = "https://github.com/example/LEDMatrix" + + +def _git(*args, cwd): + subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True) + + +@pytest.fixture +def store_inside_checkout(tmp_path): + checkout = tmp_path / "LEDMatrix" + checkout.mkdir() + _git("init", "-q", cwd=checkout) + _git("remote", "add", "origin", PARENT_REMOTE, cwd=checkout) + + plugins_dir = checkout / "plugin-repos" + plugin_dir = plugins_dir / PLUGIN_ID + plugin_dir.mkdir(parents=True) + (plugin_dir / "manifest.json").write_text(json.dumps( + {"id": PLUGIN_ID, "name": "Zip", "version": "1.0.0"})) + + store = PluginStoreManager( + plugins_dir=str(plugins_dir), + uninstalled_registry_path=str(tmp_path / "uninstalled.json")) + return store, plugin_dir + + +def test_a_plugin_without_its_own_git_has_no_remote(store_inside_checkout, monkeypatch): + store, plugin_dir = store_inside_checkout + # The premise: git itself does report the parent's remote here. + parent_view = subprocess.run( + ["git", "-C", str(plugin_dir), "config", "--local", "--get", "remote.origin.url"], + capture_output=True, text=True) + assert parent_view.stdout.strip() == PARENT_REMOTE + + monkeypatch.setattr(store, "fetch_registry", lambda *a, **k: {"plugins": []}) + monkeypatch.setattr(store, "get_plugin_info", lambda *a, **k: None) + install_calls = [] + monkeypatch.setattr(store, "install_from_url", + lambda *a, **k: install_calls.append((a, k)) or {"success": True}) + + assert store.update_plugin(PLUGIN_ID) is False + assert install_calls == [] diff --git a/test/test_sudo_allowlist_covers_calls.py b/test/test_sudo_allowlist_covers_calls.py index 8f3b88f7..eb1004fd 100644 --- a/test/test_sudo_allowlist_covers_calls.py +++ b/test/test_sudo_allowlist_covers_calls.py @@ -11,12 +11,16 @@ Four such calls were ungranted, all of them captive-portal teardown/setup: rfkill unblock wifi wifi_manager.py:1811 mkdir -p .../dnsmasq-shared.d wifi_manager.py:922 +The drop-in written into that directory was missing too: the literal +`cp /tmp/ledmatrix-nm-dnsmasq.conf .../dnsmasq-shared.d/ledmatrix-captive.conf` +and `rm -f` of the same file, so the directory was granted but not the file. + It goes unnoticed because a stock Raspberry Pi image ships /etc/sudoers.d/010_pi-nopasswd granting the default user `ALL=(ALL) NOPASSWD: ALL`, which satisfies every gap in both files. It only bites once that blanket rule is removed or the service runs as another user. -Scope, deliberately narrow: this pins the four commands above, each of which +Scope, deliberately narrow: this pins the commands above, each of which can be written out literally. The portal makes further sudo calls whose arguments are built at runtime -- iptables and nft rules carrying an interface name and a port, `ip addr`, `ip link` -- and those cannot be granted safely @@ -50,6 +54,11 @@ REQUIRED = ( ("nft", "delete", "table", "ip", "ledmatrix"), ("rfkill", "unblock", "wifi"), ("mkdir", "-p", "/etc/NetworkManager/dnsmasq-shared.d"), + # The drop-in that directory exists for, written and removed by + # _write_nm_dnsmasq_captive_conf / _remove_nm_dnsmasq_captive_conf. + ("cp", "/tmp/ledmatrix-nm-dnsmasq.conf", + "/etc/NetworkManager/dnsmasq-shared.d/ledmatrix-captive.conf"), + ("rm", "-f", "/etc/NetworkManager/dnsmasq-shared.d/ledmatrix-captive.conf"), ) #: Tools with an option that executes a program of the caller's choosing. diff --git a/test/test_sudoers_is_validated.py b/test/test_sudoers_is_validated.py index a65bbd0e..5e6aeafb 100644 --- a/test/test_sudoers_is_validated.py +++ b/test/test_sudoers_is_validated.py @@ -19,6 +19,7 @@ import pytest REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) FIRST_TIME = os.path.join(REPO_ROOT, "first_time_install.sh") CONFIGURE = os.path.join(REPO_ROOT, "scripts", "install", "configure_web_sudo.sh") +WIFI = os.path.join(REPO_ROOT, "scripts", "install", "configure_wifi_permissions.sh") VISUDO = shutil.which("visudo") or ( "/usr/sbin/visudo" if os.path.exists("/usr/sbin/visudo") else None @@ -62,6 +63,54 @@ def test_configure_web_sudo_validates_before_installing(): assert validate < install, "the rules must be checked before they are installed" +def test_configure_web_sudo_does_not_use_a_predictable_temp_file(): + body = _read(CONFIGURE) + assert 'TEMP_SUDOERS=$(mktemp' in body + assert "/tmp/ledmatrix_web_sudoers_$$" not in body + assert "trap 'rm -f \"$TEMP_SUDOERS\"' EXIT" in body + + +def test_configure_web_sudo_installs_mode_440(): + body = _read(CONFIGURE) + install = body.index('cp "$TEMP_SUDOERS" /etc/sudoers.d/ledmatrix_web') + assert body.index("chmod 440 /etc/sudoers.d/ledmatrix_web") > install + + +def test_configure_wifi_permissions_validates_before_installing(): + """The third sudoers writer. It installed its rules unchecked.""" + body = _read(WIFI) + # The check itself, as a condition -- not merely the command appearing in + # the error report that follows it. + validate = body.index('if ! visudo -c -f "$TEMP_SUDOERS"') + install = body.index('sudo cp "$TEMP_SUDOERS" "$SUDOERS_FILE"') + assert validate < install, "the rules must be checked before they are installed" + # ...and a failed check stops the script before the copy. + assert "exit 1" in body[validate:install] + assert "TEMP_SUDOERS=$(mktemp" in body + + +@pytest.mark.skipif(sys.platform == "win32", reason="visudo is POSIX only") +@pytest.mark.skipif(VISUDO is None, reason="visudo not installed") +def test_the_wifi_rules_actually_parse(tmp_path): + """Render configure_wifi_permissions.sh's heredoc with realistic paths.""" + body = _read(WIFI) + opener = 'cat > "$TEMP_SUDOERS" << EOF\n' + start = body.index(opener) + len(opener) + end = body.index("\nEOF\n", start) + out = tmp_path / "wifi" + script = "\n".join([ + "WEB_USER=ledmatrix", "NMCLI_PATH=/usr/bin/nmcli", + "SYSTEMCTL_PATH=/usr/bin/systemctl", "SYSCTL_PATH=/usr/sbin/sysctl", + "NFT_PATH=/usr/sbin/nft", "RFKILL_PATH=/usr/sbin/rfkill", + "MKDIR_PATH=/usr/bin/mkdir", + f"cat > '{out}' << EOF", body[start:end], "EOF", + ]) + subprocess.run(["bash", "-c", script], check=True) + os.chmod(out, 0o440) + result = subprocess.run([VISUDO, "-c", "-f", str(out)], capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + + def test_a_missing_rules_library_installs_nothing(): """If lib_sudoers.sh is missing, nothing is generated -- and an empty file would pass `visudo -c` -- so that branch must set the flag the install is diff --git a/test/test_sync_manager.py b/test/test_sync_manager.py index ced77d4d..da9b52d2 100644 --- a/test/test_sync_manager.py +++ b/test/test_sync_manager.py @@ -38,6 +38,7 @@ import numpy as np import pytest from PIL import Image +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401 from src.common import sync_manager from src.common.sync_manager import ( DisplaySyncManager, @@ -878,6 +879,14 @@ class TestWriteStatusFile: mgr.write_status_file() # must not raise assert mgr.logger.debug.called + def test_web_status_endpoint_reads_the_file_that_was_written(self, api_v3_client): + """GET /sync/status reads STATUS_FILE, which lives under + tempfile.gettempdir() -- not always /tmp.""" + mgr = make_manager(role=SyncRole.LEADER) + mgr.write_status_file() + response = api_v3_client.get("/api/v3/sync/status") + assert response.get_json()["data"]["role"] == "leader" + class TestStop: def _stub_with_sockets(self): diff --git a/test/test_system_status_available_memory.py b/test/test_system_status_available_memory.py index c08bde22..0113ea67 100644 --- a/test/test_system_status_available_memory.py +++ b/test/test_system_status_available_memory.py @@ -51,7 +51,7 @@ def _memory(total_mb, used_mb, available_mb): def _get_status(client, memory): # The endpoint caches for 10s; bypass so each case is measured fresh. - with patch("web_interface.cache.get_cached", return_value=None), \ + with patch("web_interface.blueprints.api_v3.system.get_cached", return_value=None), \ patch("psutil.virtual_memory", return_value=memory), \ patch("psutil.cpu_percent", return_value=5.0), \ patch("psutil.boot_time", return_value=0.0): @@ -90,3 +90,24 @@ def test_a_nearly_exhausted_board_reports_a_small_number(client): # to fork. The readout has to surface that rather than round it away. data = _get_status(client, _memory(total_mb=905, used_mb=800, available_mb=73)) assert data["memory_available_mb"] == pytest.approx(73, abs=0.5) + + +def test_status_and_live_stream_give_the_same_answer(client, monkeypatch): + """/system/status is built on collect_system_metrics(), so a metric has one + value, and "could not be read" is null in both.""" + from web_interface import system_metrics + monkeypatch.setattr(system_metrics, "_THERMAL_ZONE", "/nonexistent/thermal/temp") + memory = _memory(total_mb=905, used_mb=620, available_mb=284) + with patch("psutil.virtual_memory", return_value=memory), \ + patch("psutil.cpu_percent", return_value=5.0), \ + patch("psutil.boot_time", return_value=0.0), \ + patch("psutil.disk_usage", side_effect=OSError("no such mount")): + streamed = system_metrics.collect_system_metrics() + with patch("web_interface.blueprints.api_v3.system.get_cached", return_value=None): + status = json.loads(client.get("/api/v3/system/status").data)["data"] + + assert status["cpu_temp"] is None + assert status["disk_used_percent"] is None + for key in system_metrics.METRIC_KEYS: + if key != "uptime_seconds": # the two reads are a moment apart + assert status[key] == streamed[key], key diff --git a/test/test_systemd_unit_drift.py b/test/test_systemd_unit_drift.py index 8afaadb0..9673beff 100644 --- a/test/test_systemd_unit_drift.py +++ b/test/test_systemd_unit_drift.py @@ -17,6 +17,7 @@ editing files under /etc and restarting services is the installer's job, not something a display process should do to a machine while it boots. """ import logging +import re import shlex import subprocess from pathlib import Path @@ -253,6 +254,24 @@ def test_sed_escape_replacement_preserves_special_characters(): "a sed-special character in the replacement was not preserved literally") +def test_every_unit_renderer_escapes_its_replacement(): + """Each `sed s|__PLACEHOLDER__|$VALUE|` in an install script uses an escaped value. + + install_dns_fix.sh and install_mqtt_bridge.sh interpolated the raw project + path while the other three renderers went through sed_escape_replacement, + so a checkout under a path containing `&` rendered a broken unit from + those two only. + """ + project_root = Path("src/startup_validator.py").resolve().parent.parent + offenders = [] + for script in sorted((project_root / "scripts" / "install").glob("*.sh")): + text = script.read_text(encoding="utf-8") + for m in re.finditer(r"s\|__[A-Z_]+__\|\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?\|", text): + if not m.group(1).startswith("ESCAPED_") and m.group(1) != "root": + offenders.append(f"{script.name}: ${m.group(1)}") + assert not offenders, "unescaped sed replacement(s): " + ", ".join(offenders) + + def test_no_installer_carries_its_own_copy_of_a_unit(): """The regression guard. diff --git a/test/test_text_helper.py b/test/test_text_helper.py index 1e143ddf..0a585e42 100644 --- a/test/test_text_helper.py +++ b/test/test_text_helper.py @@ -24,10 +24,13 @@ class TestTextHelper: assert th.font_dir == tmp_path assert th._font_cache == {} - def test_init_default_font_dir(self): - """Test TextHelper initialization with default font directory.""" + def test_init_default_font_dir(self, tmp_path, monkeypatch): + """The default is the install's assets/fonts, not a cwd-relative path.""" + from pathlib import Path + monkeypatch.chdir(tmp_path) th = TextHelper() - assert th.font_dir == pytest.importorskip("pathlib").Path("assets/fonts") + assert th.font_dir == Path(__file__).resolve().parents[1] / "assets" / "fonts" + assert isinstance(th.load_fonts()["score"], ImageFont.FreeTypeFont) @patch('PIL.ImageFont.truetype') @patch('PIL.ImageFont.load_default') @@ -123,6 +126,17 @@ class TestTextHelper: def test_get_default_font_config(self, text_helper): """Test getting default font configuration.""" config = text_helper._get_default_font_config() - + assert isinstance(config, dict) assert len(config) > 0 + + def test_each_font_file_and_size_is_loaded_once(self): + th = TextHelper() + first = th.load_fonts() + second = th.load_fonts() + # Six names, three (file, size) pairs: PressStart2P at 10 and 8, 4x6 at 6. + assert first["score"] is second["score"] is first["rank"] + assert th.get_font_cache_stats()["cached_fonts"] == 3 + th.clear_font_cache() + assert th.get_font_cache_stats()["cached_fonts"] == 0 + assert th.load_fonts()["score"] is not first["score"] diff --git a/test/test_wifi_manager_ap.py b/test/test_wifi_manager_ap.py index 82e75625..fbcee733 100644 --- a/test/test_wifi_manager_ap.py +++ b/test/test_wifi_manager_ap.py @@ -369,6 +369,26 @@ def test_default_config_has_no_saved_networks(tmp_path: Path) -> None: assert "saved_networks" not in json.loads(config_path.read_text()) +@pytest.mark.unit +def test_save_config_reports_a_failed_write(manager: WiFiManager, tmp_path: Path) -> None: + # A directory where the file should be: every write to it fails, as one + # to a root-owned wifi_config.json does for the web user. + blocked = tmp_path / "blocked.json" + blocked.mkdir() + manager.config_path = blocked + + assert manager._save_config() is False + assert list(tmp_path.glob(".blocked.json.tmp.*")) == [] + + +@pytest.mark.unit +def test_save_config_round_trips(manager: WiFiManager) -> None: + manager.config["auto_enable_ap_mode"] = False + + assert manager._save_config() is True + assert json.loads(manager.config_path.read_text())["auto_enable_ap_mode"] is False + + @pytest.mark.unit def test_connecting_does_not_store_the_password(manager: WiFiManager) -> None: commands = [] @@ -389,3 +409,77 @@ def test_connecting_does_not_store_the_password(manager: WiFiManager) -> None: "the new-connection path was not reached" assert "hunter22" not in json.dumps(manager.config) assert "hunter22" not in manager.config_path.read_text() + + +# --------------------------------------------------------------------------- +# 7. Disconnect takes the saved profile down +# --------------------------------------------------------------------------- + +def _profile_nmcli(profiles: dict): + """A fake subprocess.run for `nmcli connection show` over ``profiles`` + (profile name -> SSID), recording every command it is given.""" + commands = [] + + def fake_run(cmd, *args, **kwargs): + commands.append(cmd) + if cmd == ["nmcli", "-t", "-f", "NAME,TYPE", "connection", "show"]: + lines = [name.replace(":", "\\:") + ":802-11-wireless" for name in profiles] + lines.append("Wired connection 1:802-3-ethernet") + return _ok(stdout="\n".join(lines) + "\n") + if cmd[:4] == ["nmcli", "-g", "802-11-wireless.ssid", "connection"]: + return _ok(stdout=profiles.get(cmd[-1], "") + "\n") + if cmd[:3] == ["nmcli", "connection", "show"]: + return _ok() if cmd[3] in profiles else _fail() + # nmcli rejects 802-11-wireless.ssid as a `connection show -f` column. + if "802-11-wireless.ssid" in cmd: + return _fail(stderr="Error: invalid field '802-11-wireless.ssid'") + return _ok() + + return fake_run, commands + + +@pytest.mark.unit +def test_find_profile_for_ssid_matches_by_ssid_not_name(manager: WiFiManager) -> None: + fake_run, _ = _profile_nmcli({"home: upstairs": "HomeNet", "Office": "OfficeNet"}) + with patch("src.wifi_manager.subprocess.run", side_effect=fake_run): + assert manager._find_profile_for_ssid("HomeNet") == "home: upstairs" + assert manager._find_profile_for_ssid("OfficeNet") == "Office" + assert manager._find_profile_for_ssid("Elsewhere") is None + + +@pytest.mark.unit +def test_disconnect_takes_the_profile_down(manager: WiFiManager) -> None: + from src.wifi_manager import WiFiStatus + + fake_run, commands = _profile_nmcli({"Home profile": "HomeNet"}) + with patch("src.wifi_manager.subprocess.run", side_effect=fake_run), \ + patch("src.wifi_manager.time.sleep"), \ + patch.object(manager, "get_wifi_status", + return_value=WiFiStatus(connected=True, ssid="HomeNet")): + ok, _ = manager.disconnect_from_network(skip_ap_check=True) + + assert ok + assert ["nmcli", "connection", "down", "Home profile"] in commands + assert ["nmcli", "device", "disconnect", "wlan0"] in commands + + +# --------------------------------------------------------------------------- +# 8. The nmcli Wi-Fi list parser both scan paths share +# --------------------------------------------------------------------------- + +@pytest.mark.unit +def test_nmcli_wifi_list_parsing() -> None: + out = ( + "HomeNet:40:WPA2:2437 MHz\n" + "Cafe:80::5180 MHz\n" + "HomeNet:90:WPA2:5180 MHz\n" # duplicate SSID: first line wins + ":70:WPA2:2412 MHz\n" # hidden network + "Broken:notanumber:WPA2:2412 MHz\n" + "Modern:60:WPA3 SAE:5745 MHz\n" + ) + networks = WiFiManager._parse_nmcli_wifi_list(out) + assert [(n.ssid, n.signal, n.security, n.frequency) for n in networks] == [ + ("Cafe", 80, "open", 5180.0), + ("Modern", 60, "wpa3", 5745.0), + ("HomeNet", 40, "wpa2", 2437.0), + ] diff --git a/test/web_interface/integration/test_plugin_operations.py b/test/web_interface/integration/test_plugin_operations.py index 35de83cf..128f14ed 100644 --- a/test/web_interface/integration/test_plugin_operations.py +++ b/test/web_interface/integration/test_plugin_operations.py @@ -21,10 +21,7 @@ class TestPluginOperationsIntegration(unittest.TestCase): self.temp_dir = Path(tempfile.mkdtemp()) # Initialize components - self.operation_queue = PluginOperationQueue( - history_file=str(self.temp_dir / "operations.json"), - max_history=100 - ) + self.operation_queue = PluginOperationQueue(max_history=100) self.state_manager = PluginStateManager( state_file=str(self.temp_dir / "state.json"), diff --git a/test/web_interface/test_api_v3_plugin_config_save.py b/test/web_interface/test_api_v3_plugin_config_save.py new file mode 100644 index 00000000..c0de008c --- /dev/null +++ b/test/web_interface/test_api_v3_plugin_config_save.py @@ -0,0 +1,210 @@ +"""POST /plugins/config and POST /plugins/config/reset over a real +ConfigManager and SchemaManager. + +The save path reshapes what the browser posts before validating it, and these +tests pin the reshaping that validation depends on: repeats in a uniqueItems +list are dropped, and a list the form posted as numbered fields becomes a list +again. +""" + +import json +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from src.config_manager import ConfigManager # noqa: E402 +from src.plugin_system.schema_manager import SchemaManager # noqa: E402 +from test._api_v3_test_helpers import api_v3_module, build_app # noqa: E402,F401 + +PLUGIN_ID = "stocks" + +SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "enabled": {"type": "boolean", "default": True}, + "stock_symbols": { + "type": "array", + "uniqueItems": True, + "items": {"type": "string"}, + "default": ["AAPL"], + }, + "refresh_seconds": {"type": "integer", "default": 60}, + "api_key": {"type": "string", "x-secret": True, "default": ""}, + }, +} + +# The shape of the news plugin's schema: a list of objects nested one level +# down. A form posts it as feeds.custom_feeds.0.name, feeds.custom_feeds.0.url, +# which lands as a dict keyed "0", "1", ... +NEWS_ID = "news" +NEWS_SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "enabled": {"type": "boolean", "default": True}, + "feeds": { + "type": "object", + "properties": { + "enabled_feeds": { + "type": "array", "items": {"type": "string"}, "default": [], + }, + "custom_feeds": { + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "url": {"type": "string"}, + "enabled": {"type": "boolean", "default": True}, + }, + }, + }, + }, + }, + }, +} + + +@pytest.fixture +def env(tmp_path, api_v3_module): + """Real config and schema managers under tmp_path, on the blueprint.""" + plugins_dir = tmp_path / "plugins" + for plugin_id, schema in ((PLUGIN_ID, SCHEMA), (NEWS_ID, NEWS_SCHEMA)): + plugin_dir = plugins_dir / plugin_id + plugin_dir.mkdir(parents=True) + (plugin_dir / "config_schema.json").write_text(json.dumps(schema)) + (plugin_dir / "manifest.json").write_text(json.dumps( + {"id": plugin_id, "name": plugin_id, "version": "1.0.0"})) + + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps( + {PLUGIN_ID: {"enabled": True, "stock_symbols": ["AAPL", "FNMA"]}})) + config_manager = ConfigManager( + config_path=str(config_file), + secrets_path=str(tmp_path / "config_secrets.json")) + config_manager.template_path = str(tmp_path / "no-template.json") + + api = api_v3_module.api_v3 + api.config_manager = config_manager + api.schema_manager = SchemaManager(plugins_dir=plugins_dir, project_root=tmp_path) + api.plugin_manager.plugin_manifests = {PLUGIN_ID: {"id": PLUGIN_ID}, + NEWS_ID: {"id": NEWS_ID}} + api.plugin_manager.get_plugin.return_value = None + + class Env: + client = build_app(api).test_client() + plugin_manager = api.plugin_manager + + @staticmethod + def stored(plugin_id=PLUGIN_ID): + return json.loads(config_file.read_text())[plugin_id] + + Env.config_manager = config_manager + return Env + + +class TestUniqueItemsRepeats: + """A repeat in a uniqueItems list is dropped, not a failed save.""" + + def test_form_post_repeating_a_saved_symbol_saves(self, env): + response = env.client.post( + f"/api/v3/plugins/config?plugin_id={PLUGIN_ID}", + data={"stock_symbols": "AAPL, FNMA, FNMA"}) + + assert response.status_code == 200, response.get_json() + assert env.stored()["stock_symbols"] == ["AAPL", "FNMA"] + + def test_json_post_keeps_first_occurrence_order(self, env): + response = env.client.post("/api/v3/plugins/config", json={ + "plugin_id": PLUGIN_ID, + "config": {"stock_symbols": ["TSLA", "AAPL", "TSLA", "FNMA"]}, + }) + + assert response.status_code == 200, response.get_json() + assert env.stored()["stock_symbols"] == ["TSLA", "AAPL", "FNMA"] + + +class TestNumberedFieldsBecomeLists: + """The news plugin's custom feeds, posted one field per row.""" + + FEEDS = [{"name": "Local", "url": "https://example.com/local.xml", "enabled": True}, + {"name": "Tech", "url": "https://example.com/tech.xml", "enabled": False}] + + def test_form_post(self, env): + response = env.client.post(f"/api/v3/plugins/config?plugin_id={NEWS_ID}", data={ + "feeds.custom_feeds.0.name": "Local", + "feeds.custom_feeds.0.url": "https://example.com/local.xml", + "feeds.custom_feeds.0.enabled": "on", + "feeds.custom_feeds.1.name": "Tech", + "feeds.custom_feeds.1.url": "https://example.com/tech.xml", + }) + + assert response.status_code == 200, response.get_json() + assert env.stored(NEWS_ID)["feeds"]["custom_feeds"] == self.FEEDS + + def test_json_post(self, env): + response = env.client.post("/api/v3/plugins/config", json={ + "plugin_id": NEWS_ID, + "config": {"feeds": {"custom_feeds": {"1": self.FEEDS[1], "0": self.FEEDS[0]}}}, + }) + + assert response.status_code == 200, response.get_json() + assert env.stored(NEWS_ID)["feeds"]["custom_feeds"] == self.FEEDS + + +class TestReset: + """POST /plugins/config/reset saves the way every other plugin save does.""" + + def test_reset_saves_atomically_with_a_backup(self, env, monkeypatch): + cm = env.config_manager + calls = [] + real_atomic = cm.save_config_atomic + + def spy(config, create_backup=True, **kwargs): + calls.append(create_backup) + return real_atomic(config, create_backup=create_backup, **kwargs) + + def no_plain_save(_config): + raise AssertionError("reset bypassed the atomic save") + + monkeypatch.setattr(cm, "save_config_atomic", spy) + monkeypatch.setattr(cm, "save_config", no_plain_save) + + response = env.client.post("/api/v3/plugins/config/reset", + json={"plugin_id": PLUGIN_ID}) + + assert response.status_code == 200, response.get_json() + assert calls == [True] + assert env.stored()["stock_symbols"] == ["AAPL"] + + def test_reset_notifies_the_plugin_with_its_prepared_config(self, env): + plugin = MagicMock() + env.plugin_manager.get_plugin.return_value = plugin + env.plugin_manager.prepare_plugin_config.side_effect = ( + lambda _pid, raw: {**raw, "prepared": True}) + + response = env.client.post("/api/v3/plugins/config/reset", + json={"plugin_id": PLUGIN_ID}) + + assert response.status_code == 200, response.get_json() + handed_over = plugin.on_config_change.call_args.args[0] + assert handed_over["prepared"] is True + assert handed_over["stock_symbols"] == ["AAPL"] + + def test_a_failed_save_is_reported(self, env, monkeypatch): + failed = MagicMock(message="disk full") + failed.status.value = "failed" + monkeypatch.setattr(env.config_manager, "save_config_atomic", + MagicMock(return_value=failed)) + + response = env.client.post("/api/v3/plugins/config/reset", + json={"plugin_id": PLUGIN_ID}) + + assert response.status_code == 500 + assert "disk full" in response.get_json()["message"] diff --git a/test/web_interface/test_config_arrays.py b/test/web_interface/test_config_arrays.py new file mode 100644 index 00000000..dd617695 --- /dev/null +++ b/test/web_interface/test_config_arrays.py @@ -0,0 +1,58 @@ +"""coerce_array_shapes: position-keyed dicts back into lists before validation.""" + +from src.web_interface.config_arrays import coerce_array_shapes + +COLOR = {"type": "array", "items": {"type": "integer"}, + "minItems": 3, "maxItems": 3, "default": [255, 255, 255]} + + +def test_position_keys_become_a_list_in_numeric_order(): + config = {"tags": {"10": "k", "2": "c", "0": "a"}} + coerce_array_shapes(config, {"tags": {"type": "array"}}) + assert config["tags"] == ["a", "c", "k"] + + +def test_an_empty_dict_becomes_an_empty_list(): + config = {"tags": {}} + coerce_array_shapes(config, {"tags": {"type": "array"}}) + assert config["tags"] == [] + + +def test_a_dict_with_other_keys_is_left_for_validation_to_report(): + config = {"tags": {"0": "a", "name": "b"}} + coerce_array_shapes(config, {"tags": {"type": "array"}}) + assert config["tags"] == {"0": "a", "name": "b"} + + +def test_element_types_are_left_to_normalization(): + config = {"color": {"0": "1", "1": "2", "2": "3"}} + coerce_array_shapes(config, {"color": COLOR}) + assert config["color"] == ["1", "2", "3"] + + +def test_nested_objects_and_array_items_are_walked(): + schema = {"feeds": {"type": "object", "properties": { + "custom_feeds": {"type": "array", "items": {"type": "object", "properties": { + "tags": {"type": "array"}, + }}}, + }}} + config = {"feeds": {"custom_feeds": {"0": {"tags": {"0": "news"}}}}} + coerce_array_shapes(config, schema) + assert config == {"feeds": {"custom_feeds": [{"tags": ["news"]}]}} + + +def test_a_short_form_list_takes_the_default_only_when_asked(): + config = {"color": ["10", "20"]} + coerce_array_shapes(config, {"color": COLOR}) + assert config["color"] == ["10", "20"] + + coerce_array_shapes(config, {"color": COLOR}, short_lists_take_default=True) + assert config["color"] == [255, 255, 255] + assert config["color"] is not COLOR["default"] + + +def test_a_default_too_short_itself_is_not_used(): + schema = {"color": dict(COLOR, default=[0])} + config = {"color": ["10"]} + coerce_array_shapes(config, schema, short_lists_take_default=True) + assert config["color"] == ["10"] diff --git a/test/web_interface/test_dedup_unique_arrays.py b/test/web_interface/test_dedup_unique_arrays.py index 170f776c..787b4f2f 100644 --- a/test/web_interface/test_dedup_unique_arrays.py +++ b/test/web_interface/test_dedup_unique_arrays.py @@ -1,11 +1,9 @@ -"""Tests for dedup_unique_arrays used by save_plugin_config. +"""Tests for dedup_unique_arrays, which the plugin-config save path +(_prepare_plugin_config_for_save in api_v3/plugins.py) runs before validation. -Validates that arrays with uniqueItems: true in the JSON schema have -duplicates removed before validation, preventing spurious validation -failures when form merging introduces duplicate entries. - -Tests import the production function from src.web_interface.validators -to ensure they exercise the real code path. +Arrays with uniqueItems: true in the JSON schema lose their duplicates, so a +repeat introduced by form merging does not fail validation. +test_api_v3_plugin_config_save.py checks the same through the endpoint. """ diff --git a/web_interface/app.py b/web_interface/app.py index 9c75a62c..535b055d 100644 --- a/web_interface/app.py +++ b/web_interface/app.py @@ -46,6 +46,7 @@ _JOURNALCTL = shutil.which('journalctl') _SYSTEMCTL = shutil.which('systemctl') _VCGENCMD = shutil.which('vcgencmd') +from web_interface import display_preview from web_interface.system_metrics import collect_system_metrics # Create Flask app @@ -53,14 +54,10 @@ app = Flask(__name__) app.secret_key = os.urandom(24) config_manager = ConfigManager() -# CSRF protection disabled for local-only application -# CSRF is designed for internet-facing web apps to prevent cross-site request forgery. -# For a local-only Raspberry Pi application, the threat model is different: -# - If an attacker has network access to perform CSRF, they have other attack vectors -# - All API endpoints are programmatic (HTMX/fetch) and don't include CSRF tokens -# - Forms use HTMX which doesn't automatically include CSRF tokens -# If you need CSRF protection (e.g., exposing to internet), properly implement CSRF tokens in HTMX forms -csrf = None +# No CSRF protection: the UI is meant for the local network, where anyone who +# can forge a request can also send it directly, and neither the HTMX forms +# nor the fetch() calls carry a token. Exposing the UI beyond the LAN needs +# CSRF tokens added to both first. # Initialize rate limiting (prevent accidental abuse, not security) try: @@ -93,8 +90,6 @@ except ImportError: "'pip install flask-compress'." ) -# Import cache functions from separate module to avoid circular imports - # Initialize plugin managers - read plugins directory from config config = config_manager.load_config() plugin_system_config = config.get('plugin_system', {}) @@ -144,12 +139,7 @@ schema_manager = SchemaManager( ) # Initialize operation queue for plugin operations -# Use lazy_load=True to defer file loading until first use (improves startup time) -operation_queue = PluginOperationQueue( - history_file=str(project_root / "data" / "plugin_operations.json"), - max_history=500, - lazy_load=True -) +operation_queue = PluginOperationQueue(max_history=500) # Initialize plugin state manager # Use lazy_load=True to defer file loading until first use (improves startup time) @@ -242,7 +232,6 @@ def serve_plugin_asset(plugin_id, filename): if assets_dir is None: return jsonify({'status': 'error', 'message': 'Invalid asset path'}), 403 - # Security check: ensure the assets directory exists and is within project_root if not assets_dir.exists() or not assets_dir.is_dir(): return jsonify({'status': 'error', 'message': 'Asset directory not found'}), 404 @@ -699,7 +688,7 @@ def system_status_generator(): 'power': _get_power_status() } yield status - except Exception as e: + except Exception: app.logger.error("SSE generator error", exc_info=True) yield {'error': 'An error occurred; see server logs'} time.sleep(10) # Update every 10 seconds (reduced frequency for better performance) @@ -707,9 +696,7 @@ def system_status_generator(): # Display preview generator for SSE def display_preview_generator(): """Generate display preview updates from snapshot file""" - import base64 - - snapshot_path = "/tmp/led_matrix_preview.png" # nosec B108 - fixed path matches display_manager; only read here + snapshot_path = display_preview.SNAPSHOT_PATH # Viewer marker: this generator only runs while the broadcaster has # subscribers (it exits with no clients), so touching the marker each # loop tells the DISPLAY service a browser is actually watching — it @@ -744,20 +731,8 @@ def display_preview_generator(): # Only read if file is new or has been updated if last_modified is None or current_modified > last_modified: try: - # The snapshot is already a PNG, written atomically by - # the display service (tmp + os.replace in - # display_manager), so pass the raw bytes straight - # through instead of PIL-decoding and re-encoding — - # identical payload, much less CPU on the Pi. - with open(snapshot_path, 'rb') as f: - img_str = base64.b64encode(f.read()).decode('utf-8') - - preview_data = { - 'timestamp': time.time(), - 'width': width, - 'height': height, - 'image': img_str - } + preview_data = display_preview.preview_payload( + width, height, display_preview.read_snapshot_base64(snapshot_path)) last_modified = current_modified yield preview_data except OSError: @@ -765,27 +740,20 @@ def display_preview_generator(): # between mtime check and read); skip this update. app.logger.debug("Preview snapshot read failed; skipping frame", exc_info=True) else: - # No snapshot available - yield { - 'timestamp': time.time(), - 'width': width, - 'height': height, - 'image': None - } + yield display_preview.preview_payload(width, height, None) - except Exception as e: + except Exception: app.logger.error("SSE generator error", exc_info=True) yield {'error': 'An error occurred; see server logs'} - time.sleep(1.0) # Check once per second — halves PIL encode overhead vs 0.5s + time.sleep(1.0) # the snapshot is re-read only when its mtime changes # Logs generator for SSE def logs_generator(): """Generate log updates from journalctl""" while True: try: - # Get recent logs from journalctl (simplified version) - # Note: User should be in systemd-journal group to read logs without sudo + # Reading the journal without sudo needs the systemd-journal group. try: if not _JOURNALCTL: yield {'timestamp': time.time(), 'logs': 'journalctl not found; cannot read logs'} @@ -882,29 +850,19 @@ def stream_display(): def stream_logs(): return _sse_stream(_logs_broadcaster) -# Exempt SSE streams from CSRF and apply a generous rate limit. -# SSE connections are long-lived HTTP requests, not repeated API calls, so the -# tight "20 per minute" default would be exhausted quickly on reconnects. -if csrf: - csrf.exempt(stream_stats) - csrf.exempt(stream_display) - csrf.exempt(stream_logs) - # Note: api_v3 blueprint is exempted above after registration - +# Each SSE stream is one long-lived request, so only a (re)connect counts +# against a limit. The streams get their own 200 per minute, tighter than the +# 1000 per minute default, which bounds a client stuck reconnecting. if limiter: limiter.limit("200 per minute")(stream_stats) limiter.limit("200 per minute")(stream_display) limiter.limit("200 per minute")(stream_logs) -# The pages blueprint's index now serves '/' directly (see the un-prefixed -# blueprint registration above), so no redirect route is needed here. - @app.route('/favicon.ico') def favicon(): """Return 204 No Content for favicon to avoid 404 errors""" return '', 204 -_reconciliation_done = False _reconciliation_started = False import threading as _threading _reconciliation_lock = _threading.Lock() @@ -912,18 +870,13 @@ _reconciliation_lock = _threading.Lock() def _run_startup_reconciliation() -> None: """Run state reconciliation in background to auto-repair missing plugins. - Reconciliation runs exactly once per process lifetime, regardless of - whether every inconsistency could be auto-fixed. Previously, a failed - auto-repair (e.g. a config entry referencing a plugin that no longer - exists in the registry) would reset ``_reconciliation_started`` to False, - causing the ``@app.before_request`` hook to re-trigger reconciliation on - every single HTTP request — an infinite install-retry loop that pegged - the CPU and flooded the log. Unresolved issues are now left in place for - the user to address via the UI; the reconciler itself also caches - per-plugin unrecoverable failures internally so repeated reconcile calls - stay cheap. + Runs once per process, whether or not every inconsistency could be fixed: + ``_reconciliation_started`` is never reset. Resetting it after a failed + repair (a config entry naming a plugin the registry no longer has) made + the ``before_request`` hook rerun reconciliation on every request, an + install-retry loop that pegged the CPU and flooded the log. Unresolved + issues are left for the user to address in the UI. """ - global _reconciliation_done from src.logging_config import get_logger _logger = get_logger('reconciliation') @@ -985,11 +938,6 @@ def _run_startup_reconciliation() -> None: pass except Exception as e: _logger.error("[Reconciliation] Error: %s", e, exc_info=True) - finally: - # Always mark done — we do not want an unhandled exception (or an - # unresolved inconsistency) to cause the @before_request hook to - # retrigger reconciliation on every subsequent request. - _reconciliation_done = True # Run reconciliation in the background on first request @app.before_request diff --git a/web_interface/blueprints/api_v3/__init__.py b/web_interface/blueprints/api_v3/__init__.py index 4e5815df..46eade4f 100644 --- a/web_interface/blueprints/api_v3/__init__.py +++ b/web_interface/blueprints/api_v3/__init__.py @@ -49,7 +49,6 @@ from src.plugin_system.operation_types import OperationType from src.web_interface.validators import ( validate_file_upload ) -from src.error_aggregator import get_error_aggregator from src.common.permission_utils import install_requirements_file from src.common.path_safety import resolve_under from src.device_location import DeviceLocationResolver, apply_device_location @@ -95,13 +94,11 @@ def _scrub_git_remote_url(url: str) -> str: # `config_manager` used to resolve to a None that was never assigned, which # silently disabled the /health checks and made /display/current fall back # to a hardcoded 128x64. -# Get project root directory (web_interface/../..) -# web_interface/blueprints/api_v3/_common.py -> up four to the project root. -# This was three levels when everything lived in web_interface/blueprints/api_v3.py; -# the split moved the file one directory deeper and silently pointed PROJECT_ROOT -# at web_interface/ instead. Nothing failed at import -- it surfaced as routes -# 404ing and "installation script not found", because every path built from it -# was wrong. Asserted in test_api_v3_url_map.py so the next move cannot repeat it. +# The project root, three directories above this package. The split from a +# single api_v3.py moved this file one directory deeper, and a count left at +# the old depth pointed PROJECT_ROOT at web_interface/ without failing at +# import: routes 404ed and reported "installation script not found". +# test_api_v3_url_map.py asserts it so the next move cannot repeat that. PROJECT_ROOT = Path(__file__).resolve().parents[3] # System fonts that cannot be deleted (used by catalog API and delete endpoint) SYSTEM_FONTS = frozenset([ @@ -415,9 +412,9 @@ def resolve_pull_command(project_dir): is: first_time_install.sh chmods five scripts that git tracked as 644, so every machine that ran the installer carries five permanent mode changes and the update button reports "cannot pull with rebase: You have unstaged - changes". Those modes are corrected in this commit, but a user cannot pull - the correction while the pull is what is blocked, and any other local edit - would reproduce it anyway. Autostash reapplies the changes afterwards. + changes". The repository now tracks those modes, but a user cannot pull + that correction while the pull is what is blocked, and any other local + edit would reproduce it anyway. Autostash reapplies the changes afterwards. Returns ``(args, note, error)``. When ``origin/`` exists the pull is made explicit against it, so the update proceeds and the branch is @@ -594,12 +591,7 @@ def _installed_plugin_ids(): enumerate the installed plugins and read each one's persisted summary by ID instead of relying on the tracker's in-memory `get_all_*` view. """ - manifests = _discovered_plugin_manifests() - try: - return list(manifests.keys()) if manifests else [] - except Exception: - logger.debug('listing plugin_manifests failed while building plugin ids', exc_info=True) - return [] + return list(_discovered_plugin_manifests()) def _discovered_plugin_manifests(plugin_id=None, rescan=False): """The plugin manager's manifests, discovering plugins first if needed. @@ -741,8 +733,6 @@ def _parse_form_value(value): Parse a form value into the appropriate Python type. Handles booleans, numbers, JSON arrays/objects, and strings. """ - import json - if value is None: return None @@ -892,8 +882,6 @@ def _parse_form_value_with_schema(value, key_path, schema): Returns: Parsed value with correct type, or _SKIP_FIELD to indicate the field should not be set """ - import json - # Get the schema property for this field prop = _get_schema_property(schema, key_path) @@ -1394,16 +1382,24 @@ def _prune_credential_backups(plugin_dir: Path) -> None: # calendarList.list pages at 250 entries maximum. Ten pages is far past any # real account and exists only so a malformed nextPageToken cannot spin here. _CALENDAR_LIST_MAX_PAGES = 10 +def _plugin_directory(plugin_id: str) -> Optional[Path]: + """An installed plugin's directory, or None when it has none on disk. + + Only the plugin manager is asked, so no plugin manager means None. There + is no fallback to the legacy plugins/ directory: the loader never scans + it, so a plugin found only there is one that never runs. + """ + if not api_v3.plugin_manager: + return None + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + if not plugin_dir or not Path(plugin_dir).exists(): + return None + return Path(plugin_dir) + + def _calendar_plugin_dir() -> Optional[Path]: """Where the calendar plugin is installed, or None if it is not.""" - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory('calendar') - else: - plugin_dir = PROJECT_ROOT / 'plugins' / 'calendar' - if not plugin_dir: - return None - plugin_dir = Path(plugin_dir) - return plugin_dir if plugin_dir.exists() else None + return _plugin_directory('calendar') def _run_calendar_registration(plugin_dir: Path, stdin_payload: str): """Run the plugin's OAuth script and return the JSON object it prints. @@ -1901,8 +1897,6 @@ def _write_starlark_manifest(manifest: Dict[str, Any]) -> bool: return False def _install_star_file(app_id: str, star_file_path: str, metadata: Dict[str, Any], assets_dir: Optional[str] = None) -> bool: """Install a .star file and update the manifest (standalone, no plugin needed).""" - import shutil - import json app_dir, path_error = _validate_starlark_app_path(app_id) if path_error: logger.warning("Refusing to install %r: %s", app_id, path_error) diff --git a/web_interface/blueprints/api_v3/backup.py b/web_interface/blueprints/api_v3/backup.py index f15220ea..edc53865 100644 --- a/web_interface/blueprints/api_v3/backup.py +++ b/web_interface/blueprints/api_v3/backup.py @@ -1,7 +1,7 @@ """Backup creation, listing and restore. -Routes decorate the shared `api_v3` Blueprint from ._common, so their -endpoint names are unchanged by living here. +Routes decorate the shared `api_v3` Blueprint from the package `__init__`, +so their endpoint names are unchanged by living here. """ from web_interface.blueprints.api_v3 import ( PROJECT_ROOT, Path, _coerce_to_bool, _safe_backup_path, api_v3, diff --git a/web_interface/blueprints/api_v3/config.py b/web_interface/blueprints/api_v3/config.py index 4cff4e51..00355af1 100644 --- a/web_interface/blueprints/api_v3/config.py +++ b/web_interface/blueprints/api_v3/config.py @@ -1,20 +1,20 @@ """Reading and writing configuration, including schedules. -Routes decorate the shared `api_v3` Blueprint from ._common, so their -endpoint names are unchanged by living here. +Routes decorate the shared `api_v3` Blueprint from the package `__init__`, +so their endpoint names are unchanged by living here. """ from web_interface.blueprints.api_v3 import ( - ErrorCode, Optional, PROJECT_ROOT, Path, _coerce_to_bool, + ErrorCode, Optional, _coerce_to_bool, _redact_credentials, _validate_time_format, api_v3, deep_merge, - describe_exception, error_response, find_secret_fields, json, jsonify, - logger, logging, mask_all_secret_values, merge_secrets, os, - remove_empty_secrets, request, separate_secrets, strip_masked_values, - success_response, + describe_exception, error_response, json, jsonify, + logger, mask_all_secret_values, merge_secrets, + request, strip_masked_values, success_response, ) from src.common.path_safety import resolve_under from src.display_geometry import ORIENTATION_ROTATE_DEGREES from src.matrix_support import INT_SETTING_LIMITS, describe_range, library_refusals, refusal_message from src.pi5_matrix_support import is_raspberry_pi_5 +from web_interface.cache import invalidate_cache import web_interface.blueprints.api_v3 as _pkg # Read through the module rather than bound by value: tests patch these @@ -27,6 +27,33 @@ import web_interface.blueprints.api_v3 as _pkg #: that a missing checkbox was unchecked, not merely left out of an API call. FORM_SECTION_FIELD = '__form_section' +#: Fields of the General tab. Any one of them in a /config/main post means the +#: General form was submitted, so its unchecked checkboxes read as False. +GENERAL_FIELDS = ('timezone', 'city', 'state', 'country', 'web_display_autostart', + 'plugins_directory', 'auto_update_enabled') + +#: Top-level fields save_main_config stores somewhere of its own (location, +#: plugin_system, ...), never as a config key of the same name. +_MAPPED_TOP_LEVEL_FIELDS = GENERAL_FIELDS + ( + 'auto_discover', 'auto_load_enabled', 'development_mode', 'target_fps') + + +def _plugin_id_list(raw, field_name): + """``(ids, None)`` for a list of plugin ids, or ``(None, message)``. + + The settings forms post these lists as JSON text in a hidden input; a JSON + client may send the array itself. Anything else is refused rather than + coerced: storing ``[]`` for a malformed value clears the saved order or + exclusions without a word. + """ + try: + parsed = json.loads(raw) if isinstance(raw, str) else raw + except (json.JSONDecodeError, TypeError, ValueError): + return None, f'{field_name} must be valid JSON' + if not isinstance(parsed, list) or not all(isinstance(p, str) for p in parsed): + return None, f'{field_name} must be a list of plugin-id strings' + return parsed, None + def _day_setting(data, day, flat_key, nested_key): """(present, value) of one per-day schedule setting in a POST body. @@ -203,16 +230,10 @@ def save_schedule_config(): status_code=500 ) - # Invalidate cache on config change - try: - from web_interface.cache import invalidate_cache - invalidate_cache() - except ImportError: - pass + invalidate_cache() return success_response(message='Schedule configuration saved successfully') except Exception as e: - import logging logger.error("Error saving schedule config", exc_info=True) return error_response( ErrorCode.CONFIG_SAVE_FAILED, @@ -223,11 +244,8 @@ def save_schedule_config(): @api_v3.route('/config/dim-schedule', methods=['GET']) def get_dim_schedule_config(): """Get current dim schedule configuration""" - import logging - import json - if not api_v3.config_manager: - logging.error("[DIM SCHEDULE] Config manager not initialized") + logger.error("[DIM SCHEDULE] Config manager not initialized") return error_response( ErrorCode.CONFIG_LOAD_FAILED, 'Config manager not initialized', @@ -247,28 +265,28 @@ def get_dim_schedule_config(): return success_response(data=dim_schedule_config) except FileNotFoundError as e: - logging.error(f"[DIM SCHEDULE] Config file not found: {e}", exc_info=True) + logger.error(f"[DIM SCHEDULE] Config file not found: {e}", exc_info=True) return error_response( ErrorCode.CONFIG_LOAD_FAILED, "Configuration file not found", status_code=500 ) except json.JSONDecodeError as e: - logging.error(f"[DIM SCHEDULE] Invalid JSON in config file: {e}", exc_info=True) + logger.error(f"[DIM SCHEDULE] Invalid JSON in config file: {e}", exc_info=True) return error_response( ErrorCode.CONFIG_LOAD_FAILED, "Configuration file contains invalid JSON", status_code=500 ) except (IOError, OSError) as e: - logging.error(f"[DIM SCHEDULE] Error reading config file: {e}", exc_info=True) + logger.error(f"[DIM SCHEDULE] Error reading config file: {e}", exc_info=True) return error_response( ErrorCode.CONFIG_LOAD_FAILED, "An error occurred; see logs for details", status_code=500, details=describe_exception(e) ) except Exception as e: - logging.error(f"[DIM SCHEDULE] Unexpected error loading config: {e}", exc_info=True) + logger.error(f"[DIM SCHEDULE] Unexpected error loading config: {e}", exc_info=True) return error_response( ErrorCode.CONFIG_LOAD_FAILED, "An error occurred; see logs for details", @@ -420,16 +438,10 @@ def save_dim_schedule_config(): status_code=500 ) - # Invalidate cache on config change - try: - from web_interface.cache import invalidate_cache - invalidate_cache() - except ImportError: - pass + invalidate_cache() return success_response(message='Dim schedule configuration saved successfully') except Exception as e: - import logging logger.error("Error saving dim schedule config", exc_info=True) return error_response( ErrorCode.CONFIG_SAVE_FAILED, @@ -490,11 +502,7 @@ def save_main_config(): current_config = api_v3.config_manager.load_config() was_auto_update_enabled = bool((current_config.get('auto_update') or {}).get('enabled')) - # Handle general settings - # Note: Checkboxes don't send data when unchecked, so we need to check if we're updating general settings - # If any general setting is present, we're updating the general tab - is_general_update = any(k in data for k in ['timezone', 'city', 'state', 'country', 'web_display_autostart', - 'plugins_directory', 'auto_update_enabled']) + is_general_update = any(k in data for k in GENERAL_FIELDS) if is_general_update: # For checkbox: if not present in data during a general *form* @@ -872,28 +880,13 @@ def save_main_config(): }), 400 vegas_config[config_key] = int_value - # Handle plugin order and exclusions (JSON arrays) - if 'vegas_plugin_order' in data: - try: - if isinstance(data['vegas_plugin_order'], str): - parsed = json.loads(data['vegas_plugin_order']) - else: - parsed = data['vegas_plugin_order'] - # Ensure result is a list - vegas_config['plugin_order'] = list(parsed) if isinstance(parsed, (list, tuple)) else [] - except (json.JSONDecodeError, TypeError, ValueError): - vegas_config['plugin_order'] = [] - - if 'vegas_excluded_plugins' in data: - try: - if isinstance(data['vegas_excluded_plugins'], str): - parsed = json.loads(data['vegas_excluded_plugins']) - else: - parsed = data['vegas_excluded_plugins'] - # Ensure result is a list - vegas_config['excluded_plugins'] = list(parsed) if isinstance(parsed, (list, tuple)) else [] - except (json.JSONDecodeError, TypeError, ValueError): - vegas_config['excluded_plugins'] = [] + for field_name, config_key in (('vegas_plugin_order', 'plugin_order'), + ('vegas_excluded_plugins', 'excluded_plugins')): + if field_name in data: + ids, id_error = _plugin_id_list(data[field_name], field_name) + if id_error: + return jsonify({'status': 'error', 'message': id_error}), 400 + vegas_config[config_key] = ids # Handle multi-display sync settings sync_fields = ["sync_role", "sync_port", "sync_follower_position"] @@ -921,19 +914,11 @@ def save_main_config(): return jsonify({"status": "error", "message": "sync_follower_position must be left or right"}), 400 current_config["sync"]["follower_position"] = pos_val - # Handle primary rotation order: must be a JSON array of plugin-id - # strings. Reject anything else with a 400 rather than silently - # coercing, so a buggy client can't clear or corrupt the saved order. if 'plugin_rotation_order' in data: - raw_order = data.pop('plugin_rotation_order') - try: - parsed = json.loads(raw_order) if isinstance(raw_order, str) else raw_order - except (json.JSONDecodeError, TypeError, ValueError): - return jsonify({'status': 'error', - 'message': 'plugin_rotation_order must be valid JSON'}), 400 - if not isinstance(parsed, list) or not all(isinstance(p, str) for p in parsed): - return jsonify({'status': 'error', - 'message': 'plugin_rotation_order must be a list of plugin-id strings'}), 400 + parsed, id_error = _plugin_id_list(data.pop('plugin_rotation_order'), + 'plugin_rotation_order') + if id_error: + return jsonify({'status': 'error', 'message': id_error}), 400 if 'display' not in current_config: current_config['display'] = {} current_config['display']['plugin_rotation_order'] = parsed @@ -1084,27 +1069,15 @@ def save_main_config(): for key in plugin_keys_to_remove: del data[key] - # Handle any remaining config keys - # System settings (timezone, city, etc.) are already handled above - # Plugin configs should use /api/v3/plugins/config endpoint, but we'll handle them here too for flexibility + # Whatever no section above claimed is stored as a top-level key, a + # dict merged onto the stored one. Plugin sections were handled and + # removed above. Form field names the sections above already stored + # elsewhere are skipped, or each would land as a top-level key too. + mapped_fields = set(_MAPPED_TOP_LEVEL_FIELDS).union( + display_fields, sync_fields, vegas_fields, double_sided_fields) for key in data: - # Skip system settings that are already handled above - if key in ['timezone', 'city', 'state', 'country', - 'web_display_autostart', 'auto_discover', - 'auto_load_enabled', 'development_mode', - 'plugins_directory', 'target_fps', 'auto_update_enabled']: + if key in mapped_fields: continue - # Skip fields that are already handled above in their own named sections. - # Without this, every form field name lands as a top-level config key too. - if key in display_fields: - continue - if key in sync_fields: - continue - if key in vegas_fields: - continue - if key in double_sided_fields: - continue - # For any remaining keys (including plugin keys), use deep merge to preserve existing settings if key in current_config and isinstance(current_config[key], dict) and isinstance(data[key], dict): # Deep merge to preserve existing settings current_config[key] = deep_merge(current_config[key], data[key]) @@ -1120,12 +1093,7 @@ def save_main_config(): status_code=500 ) - # Invalidate cache on config change - try: - from web_interface.cache import invalidate_cache - invalidate_cache() - except ImportError: - pass + invalidate_cache() # Notify saved plugins of their new config (with secrets merged), now # that it is on disk. diff --git a/web_interface/blueprints/api_v3/display.py b/web_interface/blueprints/api_v3/display.py index 68a0ab0d..0d99c0df 100644 --- a/web_interface/blueprints/api_v3/display.py +++ b/web_interface/blueprints/api_v3/display.py @@ -1,13 +1,14 @@ """Display control, on-demand playback and preview. -Routes decorate the shared `api_v3` Blueprint from ._common, so their -endpoint names are unchanged by living here. +Routes decorate the shared `api_v3` Blueprint from the package `__init__`, +so their endpoint names are unchanged by living here. """ from web_interface.blueprints.api_v3 import ( _ensure_display_service_running, _get_display_service_status, _stop_display_service, api_v3, - jsonify, logger, os, request, uuid, + jsonify, logger, request, uuid, ) +from web_interface import display_preview import web_interface.blueprints.api_v3 as _pkg # Read through the module rather than bound by value: tests patch these # as module attributes, and a value binding would not see the patch. @@ -30,13 +31,11 @@ def _cache_manager(): @api_v3.route('/display/current', methods=['GET']) def get_display_current(): - """Get current display state""" - import base64 - from PIL import Image - import io - - snapshot_path = "/tmp/led_matrix_preview.png" + """The latest display preview, as the /stream/display SSE stream sends it. + ``data`` is ``{timestamp, width, height, image}``; ``image`` is the + snapshot PNG base64-encoded, or null when there is none to show. + """ # Get display dimensions from config: the logical size DisplayManager # renders at, so double-sided setups preview one screen from src.display_geometry import logical_size @@ -46,26 +45,16 @@ def get_display_current(): except Exception: width, height = logical_size({}) - # Try to read snapshot file - image_data = None - if os.path.exists(snapshot_path): - try: - with Image.open(snapshot_path) as img: - # Convert to PNG and encode as base64 - buffer = io.BytesIO() - img.save(buffer, format='PNG') - image_data = base64.b64encode(buffer.getvalue()).decode('utf-8') - except Exception as img_err: - # File might be being written or corrupted, return None - pass + try: + image = display_preview.read_snapshot_base64() + except FileNotFoundError: + image = None # the display service has not written one yet + except OSError: + logger.warning("Could not read the display preview snapshot", exc_info=True) + image = None - display_data = { - 'timestamp': _pkg.time.time(), - 'width': width, - 'height': height, - 'image': image_data # Base64 encoded image data or None if unavailable - } - return jsonify({'status': 'success', 'data': display_data}) + return jsonify({'status': 'success', + 'data': display_preview.preview_payload(width, height, image)}) @api_v3.route('/display/modes', methods=['GET']) def get_display_modes(): """Every display mode that can be requested on-demand, with its plugin. diff --git a/web_interface/blueprints/api_v3/fonts.py b/web_interface/blueprints/api_v3/fonts.py index ff52c744..2eb28a8a 100644 --- a/web_interface/blueprints/api_v3/fonts.py +++ b/web_interface/blueprints/api_v3/fonts.py @@ -1,12 +1,13 @@ """Font catalogue, upload, preview and deletion. -Routes decorate the shared `api_v3` Blueprint from ._common, so their -endpoint names are unchanged by living here. +Routes decorate the shared `api_v3` Blueprint from the package `__init__`, +so their endpoint names are unchanged by living here. """ from web_interface.blueprints.api_v3 import ( PROJECT_ROOT, Path, Response, SYSTEM_FONTS, api_v3, jsonify, logger, os, re, request, validate_file_upload, ) +from web_interface.cache import delete_cached, get_cached, set_cached def _catalog_response(catalog): @@ -52,16 +53,9 @@ def _catalog_response(catalog): @api_v3.route('/fonts/catalog', methods=['GET']) def get_fonts_catalog(): """Get fonts catalog""" - # Check cache first (5 minute TTL) - try: - from web_interface.cache import get_cached, set_cached - cached_result = get_cached('fonts_catalog', ttl_seconds=300) - if cached_result is not None: - return _catalog_response(cached_result) - except ImportError: - # Cache not available, continue without caching - get_cached = None - set_cached = None + cached_result = get_cached('fonts_catalog', ttl_seconds=300) + if cached_result is not None: + return _catalog_response(cached_result) # Try to import freetype, but continue without it if unavailable try: @@ -148,12 +142,7 @@ def get_fonts_catalog(): 'metadata': metadata if metadata else None } - # Cache the result (5 minute TTL) if available - if set_cached: - try: - set_cached('fonts_catalog', catalog, ttl_seconds=300) - except Exception: - logger.error("[FontCatalog] Failed to cache fonts_catalog", exc_info=True) + set_cached('fonts_catalog', catalog, ttl_seconds=300) return _catalog_response(catalog) @api_v3.route('/fonts/tokens', methods=['GET']) @@ -229,14 +218,7 @@ def upload_font(): # Save the file font_file.save(str(filepath)) - # Clear font catalog cache - try: - from web_interface.cache import delete_cached - delete_cached('fonts_catalog') - except ImportError as e: - logger.warning("[FontUpload] Cache module not available: %s", e) - except Exception: - logger.error("[FontUpload] Failed to clear fonts_catalog cache", exc_info=True) + delete_cached('fonts_catalog') return jsonify({ 'status': 'success', @@ -453,14 +435,7 @@ def delete_font(font_family: str) -> tuple[Response, int] | Response: if not deleted: return jsonify({'status': 'error', 'message': f'Font not found: {font_family}'}), 404 - # Clear font catalog cache - try: - from web_interface.cache import delete_cached - delete_cached('fonts_catalog') - except ImportError as e: - logger.warning("[FontDelete] Cache module not available: %s", e) - except Exception: - logger.error("[FontDelete] Failed to clear fonts_catalog cache", exc_info=True) + delete_cached('fonts_catalog') return jsonify({ 'status': 'success', diff --git a/web_interface/blueprints/api_v3/misc.py b/web_interface/blueprints/api_v3/misc.py index 0f50f490..e1fd7af4 100644 --- a/web_interface/blueprints/api_v3/misc.py +++ b/web_interface/blueprints/api_v3/misc.py @@ -1,12 +1,12 @@ """Routes with no larger group of their own: errors, integrations, cache, sync, logs, health and hardware. -Routes decorate the shared `api_v3` Blueprint from ._common, so their -endpoint names are unchanged by living here. +Routes decorate the shared `api_v3` Blueprint from the package `__init__`, +so their endpoint names are unchanged by living here. """ from web_interface.blueprints.api_v3 import ( - _coerce_to_bool, - ErrorCode, Path, _JOURNALCTL, _MQTT_BRIDGE_CONFIG, _MQTT_BRIDGE_DEFAULTS, + _coerce_to_bool, _discovered_plugin_manifests, + ErrorCode, _JOURNALCTL, _MQTT_BRIDGE_CONFIG, _MQTT_BRIDGE_DEFAULTS, _MQTT_BRIDGE_DIR, _SUDO, _coerce_mqtt_bridge_value, _get_display_service_status, _mqtt_bridge_service_state, _read_mqtt_bridge_config, api_v3, contextlib, describe_exception, @@ -14,7 +14,9 @@ from web_interface.blueprints.api_v3 import ( subprocess, success_response, tempfile, ) from src.common.path_safety import safe_path_component +from src.common import sync_manager as _sync from src import error_aggregator as _errors +from web_interface import display_preview import web_interface.blueprints.api_v3 as _pkg # Read through the module rather than bound by value: tests patch these # as module attributes, and a value binding would not see the patch. @@ -33,11 +35,9 @@ def get_health(): 'checks': {} } - # Check web interface service - # Stamp the start _pkg.time before measuring against it -- reading it with a - # fallback of _pkg.time.time() and only assigning afterwards made the very - # first call subtract two separate clock reads, reporting a small - # negative uptime. + # Stamp the start time before measuring against it: reading it with a + # fallback of time.time() and assigning it afterwards made the first + # call subtract two separate clock reads, a small negative uptime. if not hasattr(get_health, '_start_time'): get_health._start_time = _pkg.time.time() health_status['services']['web_interface'] = { @@ -55,7 +55,7 @@ def get_health(): # Check config file accessibility try: if api_v3.config_manager: - test_config = api_v3.config_manager.load_config() + api_v3.config_manager.load_config() health_status['checks']['config_file'] = { 'status': 'accessible', 'readable': True @@ -65,7 +65,8 @@ def get_health(): 'status': 'unknown', 'readable': False } - except Exception as e: + except Exception: + logger.warning("Health check could not read the config file", exc_info=True) health_status['checks']['config_file'] = { 'status': 'error', 'readable': False, @@ -75,8 +76,7 @@ def get_health(): # Check plugin system try: if api_v3.plugin_manager: - # Try to discover plugins (lightweight check) - plugin_count = len(api_v3.plugin_manager.get_available_plugins()) if hasattr(api_v3.plugin_manager, 'get_available_plugins') else 0 + plugin_count = len(_discovered_plugin_manifests()) health_status['checks']['plugin_system'] = { 'status': 'operational', 'plugin_count': plugin_count @@ -85,7 +85,8 @@ def get_health(): health_status['checks']['plugin_system'] = { 'status': 'not_initialized' } - except Exception as e: + except Exception: + logger.warning("Health check could not count plugins", exc_info=True) health_status['checks']['plugin_system'] = { 'status': 'error', 'error': 'see logs for details' @@ -93,7 +94,7 @@ def get_health(): # Check hardware connectivity (if display manager available) try: - snapshot_path = "/tmp/led_matrix_preview.png" + snapshot_path = display_preview.SNAPSHOT_PATH if os.path.exists(snapshot_path): # Check if snapshot is recent (updated in last 60 seconds) mtime = os.path.getmtime(snapshot_path) @@ -107,7 +108,8 @@ def get_health(): 'status': 'no_snapshot', 'note': 'Display service may not be running' } - except Exception as e: + except Exception: + logger.warning("Health check could not read the preview snapshot", exc_info=True) health_status['checks']['hardware'] = { 'status': 'unknown', 'error': 'see logs for details' @@ -190,21 +192,21 @@ def get_logs(): @api_v3.route('/sync/status', methods=['GET']) def get_sync_status(): """Return live multi-display sync status written by the display process.""" - import os as _os - status_file = "/tmp/led_matrix_sync_status.json" + # The display process writes this file; read it where it is written. + status_file = _sync.STATUS_FILE # Also surface config so the UI can show the configured role even before # the display process has written a status file. cfg_role = "standalone" - cfg_port = 5765 + cfg_port = _sync.SYNC_PORT if api_v3.config_manager: try: cfg = api_v3.config_manager.load_config().get("sync", {}) cfg_role = cfg.get("role", "standalone") - cfg_port = int(cfg.get("port", 5765)) + cfg_port = int(cfg.get("port", _sync.SYNC_PORT)) except Exception: pass - if _os.path.exists(status_file): + if os.path.exists(status_file): try: with open(status_file) as f: live = json.load(f) diff --git a/web_interface/blueprints/api_v3/plugins.py b/web_interface/blueprints/api_v3/plugins.py index 24ecc362..a5ebefbf 100644 --- a/web_interface/blueprints/api_v3/plugins.py +++ b/web_interface/blueprints/api_v3/plugins.py @@ -1,7 +1,7 @@ """Plugin install, update, enable/disable, config and store routes. -Routes decorate the shared `api_v3` Blueprint from ._common, so their -endpoint names are unchanged by living here. +Routes decorate the shared `api_v3` Blueprint from the package `__init__`, +so their endpoint names are unchanged by living here. """ from web_interface.blueprints.api_v3 import ( ErrorCode, OperationType, PROJECT_ROOT, Path, Response, @@ -9,13 +9,14 @@ from web_interface.blueprints.api_v3 import ( _do_transactional_uninstall, _enhance_schema_with_core_properties, _filter_config_by_schema, _get_plugin_version, _get_schema_property, _hidden_array_item_property, _installed_plugin_ids, _is_plugin_update_available, + _plugin_directory, _parse_form_value_with_schema, _prune_credential_backups, _run_calendar_registration, _schema_allows_null, _schema_type_is, _set_missing_booleans_to_false, _set_nested_value, _starlark_virtual_plugins, _toggle_starlark_app, api_v3, datetime, deep_merge, describe_exception, error_response, exception_error_response, - find_secret_fields, hashlib, json, jsonify, logger, logging, + find_secret_fields, hashlib, json, jsonify, logger, merge_secrets, os, redact_text, remove_empty_secrets, request, separate_secrets, shutil, stat, subprocess, success_response, tempfile, uuid, validate_request_json, @@ -23,6 +24,8 @@ from web_interface.blueprints.api_v3 import ( from src.common.path_safety import ( resolve_under, safe_path_component, safe_relative_parts, ) +from src.web_interface.config_arrays import coerce_array_shapes +from src.web_interface.validators import dedup_unique_arrays import web_interface.blueprints.api_v3 as _pkg # Read through the module rather than bound by value: tests patch these # as module attributes, and a value binding would not see the patch. @@ -36,9 +39,6 @@ def get_installed_plugins(): if not api_v3.plugin_manager or not api_v3.plugin_store_manager: return jsonify({'status': 'error', 'message': 'Plugin managers not initialized'}), 500 - import json - from pathlib import Path - # Re-discover plugins to ensure we have the latest list # This handles cases where plugins are added/removed after app startup api_v3.plugin_manager.discover_plugins() @@ -180,8 +180,7 @@ def get_plugin_health(): if not api_v3.plugin_manager: return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - # Check if health tracker is available - if not hasattr(api_v3.plugin_manager, 'health_tracker') or not api_v3.plugin_manager.health_tracker: + if not api_v3.plugin_manager.health_tracker: return jsonify({ 'status': 'success', 'data': {}, @@ -216,15 +215,15 @@ def get_plugin_health_single(plugin_id): if not api_v3.plugin_manager: return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - # Check if health tracker is available - if not hasattr(api_v3.plugin_manager, 'health_tracker') or not api_v3.plugin_manager.health_tracker: + if not api_v3.plugin_manager.health_tracker: return jsonify({ 'status': 'error', 'message': 'Health tracking not available' }), 503 - # Get health summary for specific plugin - health_summary = api_v3.plugin_manager.health_tracker.get_health_summary(plugin_id) + # force_reload for the same reason as the list route above. + health_summary = api_v3.plugin_manager.health_tracker.get_health_summary( + plugin_id, force_reload=True) return jsonify({ 'status': 'success', @@ -236,8 +235,7 @@ def reset_plugin_health(plugin_id): if not api_v3.plugin_manager: return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - # Check if health tracker is available - if not hasattr(api_v3.plugin_manager, 'health_tracker') or not api_v3.plugin_manager.health_tracker: + if not api_v3.plugin_manager.health_tracker: return jsonify({ 'status': 'error', 'message': 'Health tracking not available' @@ -256,8 +254,7 @@ def get_plugin_metrics(): if not api_v3.plugin_manager: return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - # Check if resource monitor is available - if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: + if not api_v3.plugin_manager.resource_monitor: return jsonify({ 'status': 'success', 'data': {}, @@ -291,15 +288,15 @@ def get_plugin_metrics_single(plugin_id): if not api_v3.plugin_manager: return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - # Check if resource monitor is available - if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: + if not api_v3.plugin_manager.resource_monitor: return jsonify({ 'status': 'error', 'message': 'Resource monitoring not available' }), 503 - # Get metrics summary for specific plugin - metrics_summary = api_v3.plugin_manager.resource_monitor.get_metrics_summary(plugin_id) + # force_reload for the same reason as the list route above. + metrics_summary = api_v3.plugin_manager.resource_monitor.get_metrics_summary( + plugin_id, force_reload=True) return jsonify({ 'status': 'success', @@ -311,8 +308,7 @@ def reset_plugin_metrics(plugin_id): if not api_v3.plugin_manager: return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - # Check if resource monitor is available - if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: + if not api_v3.plugin_manager.resource_monitor: return jsonify({ 'status': 'error', 'message': 'Resource monitoring not available' @@ -331,8 +327,7 @@ def manage_plugin_limits(plugin_id): if not api_v3.plugin_manager: return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - # Check if resource monitor is available - if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: + if not api_v3.plugin_manager.resource_monitor: return jsonify({ 'status': 'error', 'message': 'Resource monitoring not available' @@ -433,17 +428,13 @@ def toggle_plugin(): config[plugin_id] = {} config[plugin_id]['enabled'] = enabled - # Use atomic save if available - if hasattr(api_v3.config_manager, 'save_config_atomic'): - result = api_v3.config_manager.save_config_atomic(config, create_backup=True) - if result.status.value != 'success': - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - f"Failed to save configuration: {result.message}", - status_code=500 - ) - else: - api_v3.config_manager.save_config(config) + success, error_msg = _pkg._save_config_atomic(api_v3.config_manager, config, create_backup=True) + if not success: + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + f"Failed to save configuration: {error_msg}", + status_code=500 + ) # Update state manager if available if api_v3.plugin_state_manager: @@ -470,8 +461,7 @@ def toggle_plugin(): plugin.on_disable() except Exception as lifecycle_error: # Log the error but don't fail the toggle - config is already saved - import logging - logging.warning(f"Lifecycle method error for {plugin_id}: {lifecycle_error}", exc_info=True) + logger.warning("Lifecycle method error for %s: %s", plugin_id, lifecycle_error, exc_info=True) return success_response( message=f"Plugin {plugin_id} {'enabled' if enabled else 'disabled'} successfully" @@ -748,25 +738,16 @@ def get_plugin_config(): plugin_config, schema_mgr.load_schema(plugin_id, use_cache=True), defaults) except Exception as e: # Log but don't fail - defaults merge is best effort - import logging - logging.warning(f"Could not merge defaults for {plugin_id}: {e}") + logger.warning("Could not merge defaults for %s: %s", plugin_id, e) # Special handling for of-the-day plugin: populate uploaded_files and categories from disk if plugin_id == 'of-the-day' or plugin_id == 'ledmatrix-of-the-day': - # Get plugin directory - plugin_id in manifest is 'of-the-day', but directory is 'ledmatrix-of-the-day' - plugin_dir_name = 'ledmatrix-of-the-day' - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_dir_name) - # If not found, try with the plugin_id - if not plugin_dir or not Path(plugin_dir).exists(): - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_dir_name - if not plugin_dir.exists(): - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if plugin_dir and Path(plugin_dir).exists(): - data_dir = Path(plugin_dir) / 'of_the_day' + # The manifest id is 'of-the-day'; the directory is usually + # 'ledmatrix-of-the-day'. + plugin_dir = (_plugin_directory('ledmatrix-of-the-day') + or _plugin_directory(plugin_id)) + if plugin_dir: + data_dir = plugin_dir / 'of_the_day' if data_dir.exists(): # Scan for JSON files uploaded_files = [] @@ -934,7 +915,6 @@ def update_plugin(): if manifest_path.exists(): try: - import json with open(manifest_path, 'r', encoding='utf-8') as f: manifest = json.load(f) current_last_updated = manifest.get('last_updated') @@ -983,7 +963,6 @@ def update_plugin(): updated_version = current_version try: if manifest_path.exists(): - import json with open(manifest_path, 'r', encoding='utf-8') as f: manifest = json.load(f) updated_last_updated = manifest.get('last_updated', current_last_updated) @@ -1224,10 +1203,7 @@ def install_plugin(): return jsonify({'status': 'error', 'message': f"{plugin_id} is a {registry_entry.get('type')!r} entry, not a plugin"}), 400 - # Install the plugin - # Log the plugins directory being used for debugging plugins_dir = api_v3.plugin_store_manager.plugins_dir - branch_info = f" (branch: {branch})" if branch else "" logger.info("Installing plugin to directory: %s", plugins_dir) # Use operation queue if available @@ -1590,24 +1566,21 @@ def get_github_auth_status(): }) @api_v3.route('/plugins/store/refresh', methods=['POST']) def refresh_plugin_store(): - """Refresh plugin store repository""" + """Re-download the plugin registry, bypassing its cache. + + Takes no body. Answers ``{status, message, plugin_count}``, the count + being the registry's entries. Commit metadata is not refreshed here: the + store list fetches it per plugin when it is shown. + """ if not api_v3.plugin_store_manager: return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - data = request.get_json(silent=True) or {} - fetch_commit_info = data.get('fetch_commit_info', data.get('fetch_latest_versions', False)) - - # Force refresh the registry registry = api_v3.plugin_store_manager.fetch_registry(force_refresh=True) plugin_count = len(registry.get('plugins', [])) - message = 'Plugin store refreshed' - if fetch_commit_info: - message += ' (with refreshed commit metadata from GitHub)' - return jsonify({ 'status': 'success', - 'message': message, + 'message': 'Plugin store refreshed', 'plugin_count': plugin_count }) @api_v3.route('/plugins/config', methods=['POST']) @@ -1699,7 +1672,6 @@ def save_plugin_config(): # Process bracket notation fields and set directly in plugin_config # Use JSON encoding instead of comma-join to handle values containing commas - import json for base_path, values in bracket_array_fields.items(): # Get schema property to verify it's an array base_prop = _get_schema_property(schema, base_path) @@ -1797,254 +1769,11 @@ def save_plugin_config(): if parsed_value is not _SKIP_FIELD: _set_nested_value(plugin_config, key, parsed_value) - # Post-process: Fix array fields that might have been incorrectly structured - # This handles cases where array fields are stored as dicts (e.g., from indexed form fields) - def fix_array_structures(config_dict, schema_props, prefix=''): - """Recursively fix array structures (convert dicts with numeric keys to arrays, fix length issues)""" - for prop_key, prop_schema in schema_props.items(): - prop_type = prop_schema.get('type') - - if prop_type == 'array': - # Navigate to the field location - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if isinstance(parent, dict) and part in parent: - parent = parent[part] - else: - parent = None - break - - if parent is not None and isinstance(parent, dict) and prop_key in parent: - current_value = parent[prop_key] - # If it's a dict with numeric string keys, convert to array - if isinstance(current_value, dict) and not isinstance(current_value, list): - try: - # Check if all keys are numeric strings (array indices) - keys = [k for k in current_value.keys()] - if all(k.isdigit() for k in keys): - # Convert to sorted array by index - sorted_keys = sorted(keys, key=int) - array_value = [current_value[k] for k in sorted_keys] - # Convert array elements to correct types based on schema - items_schema = prop_schema.get('items', {}) - item_type = items_schema.get('type') - if item_type in ('number', 'integer'): - converted_array = [] - for v in array_value: - if isinstance(v, str): - try: - if item_type == 'integer': - converted_array.append(int(v)) - else: - converted_array.append(float(v)) - except (ValueError, TypeError, OverflowError): - converted_array.append(v) - else: - converted_array.append(v) - array_value = converted_array - parent[prop_key] = array_value - current_value = array_value # Update for length check below - except (ValueError, KeyError, TypeError): - # Conversion failed, check if we should use default - pass - - # If it's an array, ensure correct types and check minItems - if isinstance(current_value, list): - # First, ensure array elements are correct types - items_schema = prop_schema.get('items', {}) - item_type = items_schema.get('type') - if item_type in ('number', 'integer'): - converted_array = [] - for v in current_value: - if isinstance(v, str): - try: - if item_type == 'integer': - converted_array.append(int(v)) - else: - converted_array.append(float(v)) - except (ValueError, TypeError, OverflowError): - converted_array.append(v) - else: - converted_array.append(v) - parent[prop_key] = converted_array - current_value = converted_array - - # Then check minItems - min_items = prop_schema.get('minItems') - if min_items is not None and len(current_value) < min_items: - # Use default if available, otherwise keep as-is (validation will catch it) - default = prop_schema.get('default') - if default and isinstance(default, list) and len(default) >= min_items: - parent[prop_key] = default - else: - # Top-level field - if prop_key in config_dict: - current_value = config_dict[prop_key] - # If it's a dict with numeric string keys, convert to array - if isinstance(current_value, dict) and not isinstance(current_value, list): - try: - keys = list(current_value.keys()) - if keys and all(str(k).isdigit() for k in keys): - sorted_keys = sorted(keys, key=lambda x: int(str(x))) - array_value = [current_value[k] for k in sorted_keys] - # Convert array elements to correct types based on schema - items_schema = prop_schema.get('items', {}) - item_type = items_schema.get('type') - if item_type in ('number', 'integer'): - converted_array = [] - for v in array_value: - if isinstance(v, str): - try: - if item_type == 'integer': - converted_array.append(int(v)) - else: - converted_array.append(float(v)) - except (ValueError, TypeError, OverflowError): - converted_array.append(v) - else: - converted_array.append(v) - array_value = converted_array - config_dict[prop_key] = array_value - current_value = array_value # Update for length check below - except (ValueError, KeyError, TypeError) as e: - logger.debug(f"Failed to convert {prop_key} to array: {e}") - - # If it's an array, ensure correct types and check minItems - if isinstance(current_value, list): - # First, ensure array elements are correct types - items_schema = prop_schema.get('items', {}) - item_type = items_schema.get('type') - if item_type in ('number', 'integer'): - converted_array = [] - for v in current_value: - if isinstance(v, str): - try: - if item_type == 'integer': - converted_array.append(int(v)) - else: - converted_array.append(float(v)) - except (ValueError, TypeError, OverflowError): - converted_array.append(v) - else: - converted_array.append(v) - config_dict[prop_key] = converted_array - current_value = converted_array - - # Then check minItems - min_items = prop_schema.get('minItems') - if min_items is not None and len(current_value) < min_items: - default = prop_schema.get('default') - if default and isinstance(default, list) and len(default) >= min_items: - config_dict[prop_key] = default - - # Recurse into nested objects - elif prop_type == 'object' and 'properties' in prop_schema: - nested_prefix = f"{prefix}.{prop_key}" if prefix else prop_key - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if isinstance(parent, dict) and part in parent: - parent = parent[part] - else: - parent = None - break - nested_dict = parent.get(prop_key) if parent is not None and isinstance(parent, dict) else None - else: - nested_dict = config_dict.get(prop_key) - - if isinstance(nested_dict, dict): - # Pass no prefix: config_dict is already the navigated sub-dict, - # so path segments from the parent would mis-navigate it. - fix_array_structures(nested_dict, prop_schema['properties']) - - # Also ensure array fields that are None get converted to empty arrays - def ensure_array_defaults(config_dict, schema_props, prefix=''): - """Recursively ensure array fields have defaults if None""" - for prop_key, prop_schema in schema_props.items(): - prop_type = prop_schema.get('type') - - if prop_type == 'array': - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if isinstance(parent, dict) and part in parent: - parent = parent[part] - else: - parent = None - break - - if parent is not None and isinstance(parent, dict): - if prop_key not in parent or parent[prop_key] is None: - default = prop_schema.get('default', []) - parent[prop_key] = default if default else [] - else: - if prop_key not in config_dict or config_dict[prop_key] is None: - default = prop_schema.get('default', []) - config_dict[prop_key] = default if default else [] - - elif prop_type == 'object' and 'properties' in prop_schema: - nested_prefix = f"{prefix}.{prop_key}" if prefix else prop_key - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if isinstance(parent, dict) and part in parent: - parent = parent[part] - else: - parent = None - break - nested_dict = parent.get(prop_key) if parent is not None and isinstance(parent, dict) else None - else: - nested_dict = config_dict.get(prop_key) - - if nested_dict is None: - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if part not in parent: - parent[part] = {} - parent = parent[part] - if prop_key not in parent: - parent[prop_key] = {} - nested_dict = parent[prop_key] - else: - if prop_key not in config_dict: - config_dict[prop_key] = {} - nested_dict = config_dict[prop_key] - - if isinstance(nested_dict, dict): - # Pass no prefix: config_dict is already navigated. - ensure_array_defaults(nested_dict, prop_schema['properties']) - + # Before the booleans below: that walk replaces anything it + # expects to be a list and finds is not one. if schema and 'properties' in schema: - # First, fix any dict structures that should be arrays - # This must be called BEFORE validation to convert dicts with numeric keys to arrays - fix_array_structures(plugin_config, schema['properties']) - # Then, ensure None arrays get defaults - ensure_array_defaults(plugin_config, schema['properties']) - - # Debug: Log the structure after fixing - if 'feeds' in plugin_config and 'custom_feeds' in plugin_config.get('feeds', {}): - custom_feeds = plugin_config['feeds']['custom_feeds'] - logger.debug(f"After fix_array_structures: custom_feeds type={type(custom_feeds)}, value={custom_feeds}") - - # Force fix for feeds.custom_feeds if it's still a dict (fallback) - if 'feeds' in plugin_config: - feeds_config = plugin_config.get('feeds') or {} - if feeds_config and 'custom_feeds' in feeds_config and isinstance(feeds_config['custom_feeds'], dict): - custom_feeds_dict = feeds_config['custom_feeds'] - # Check if all keys are numeric - keys = list(custom_feeds_dict.keys()) - if keys and all(str(k).isdigit() for k in keys): - # Convert to array - sorted_keys = sorted(keys, key=lambda x: int(str(x))) - feeds_config['custom_feeds'] = [custom_feeds_dict[k] for k in sorted_keys] - logger.info(f"Force-converted feeds.custom_feeds from dict to array: {len(feeds_config['custom_feeds'])} items") + coerce_array_shapes(plugin_config, schema['properties'], + short_lists_take_default=True) # Fix unchecked boolean checkboxes: HTML checkboxes don't submit values # when unchecked, so the existing config value (potentially True) persists. @@ -2104,8 +1833,6 @@ def save_plugin_config(): try: api_v3.config_manager.save_raw_file_content('secrets', current_secrets) except PermissionError as e: - # Log the error with more details - import os secrets_path = api_v3.config_manager.secrets_path secrets_dir = os.path.dirname(secrets_path) if secrets_path else None @@ -2126,12 +1853,9 @@ def save_plugin_config(): f"Failed to save secrets configuration: Permission denied. Check file permissions on {secrets_path}", status_code=500 ) - except Exception as e: - # Log the error but don't fail the entire config save - import os + except Exception: secrets_path = api_v3.config_manager.secrets_path logger.error("Error saving secrets config for %s (path=%s)", plugin_id, secrets_path, exc_info=True) - # Return error response with more context return error_response( ErrorCode.CONFIG_SAVE_FAILED, "Failed to save secrets configuration; see logs for details", @@ -2177,8 +1901,7 @@ def save_plugin_config(): plugin_instance.on_disable() except Exception as lifecycle_error: # Log the error but don't fail the save - config is already saved - import logging - logging.warning(f"Lifecycle method error for {plugin_id}: {lifecycle_error}", exc_info=True) + logger.warning("Lifecycle method error for %s: %s", plugin_id, lifecycle_error, exc_info=True) except Exception as hook_err: # Do not fail the save if hook fails; just log logger.warning("on_config_change failed: %s", hook_err) @@ -2226,48 +1949,9 @@ def _prepare_plugin_config_for_save(plugin_id, plugin_config, schema, schema_mgr Returns ``(regular_config, secrets_config, None)``, or ``(None, None, error_response)`` when validation fails. """ - # JSON path: fix numeric-keyed dicts that should be arrays. - # JS dotToNested() converts feeds.custom_feeds.0.name → {'0': {name:...}} - # instead of [{name:...}]. The form-data path has fix_array_structures for this; - # mirror that logic here for JSON submissions. + # The form path has already done this, before its checkbox pass. if is_json and schema and 'properties' in schema: - def _fix_json_arrays(cfg, props): - for k, ps in props.items(): - if not isinstance(cfg, dict) or k not in cfg: - continue - pt = ps.get('type') - val = cfg[k] - if pt == 'array': - items_schema = ps.get('items', {}) - item_type = items_schema.get('type') - if isinstance(val, dict): - keys = list(val.keys()) - if keys and all(str(x).isdigit() for x in keys): - sorted_keys = sorted(keys, key=lambda x: int(str(x))) - arr = [val[sk] for sk in sorted_keys] - if item_type in ('integer', 'number'): - converted = [] - for v in arr: - if isinstance(v, str): - try: - converted.append(int(v) if item_type == 'integer' else float(v)) - except (ValueError, TypeError, OverflowError): - converted.append(v) - else: - converted.append(v) - arr = converted - cfg[k] = arr - elif not keys: - cfg[k] = [] - # Recurse into each element when items are objects with properties, - # covering both freshly-converted and already-list values. - if item_type == 'object' and 'properties' in items_schema: - for elem in (cfg[k] if isinstance(cfg[k], list) else []): - if isinstance(elem, dict): - _fix_json_arrays(elem, items_schema['properties']) - elif pt == 'object' and 'properties' in ps and isinstance(val, dict): - _fix_json_arrays(val, ps['properties']) - _fix_json_arrays(plugin_config, schema['properties']) + coerce_array_shapes(plugin_config, schema['properties']) # PRE-PROCESSING: Preserve 'enabled' state if not in request # This prevents overwriting the enabled state when saving config from a form that doesn't include the toggle @@ -2276,7 +1960,6 @@ def _prepare_plugin_config_for_save(plugin_id, plugin_config, schema, schema_mgr current_config = api_v3.config_manager.load_config() if plugin_id in current_config and 'enabled' in current_config[plugin_id]: plugin_config['enabled'] = current_config[plugin_id]['enabled'] - # logger.debug(f"Preserving enabled state for {plugin_id}: {plugin_config['enabled']}") elif api_v3.plugin_manager: # Fallback to plugin instance if config doesn't have it plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) @@ -2311,9 +1994,8 @@ def _prepare_plugin_config_for_save(plugin_id, plugin_config, schema, schema_mgr defaults = schema_mgr.generate_default_config(plugin_id, use_cache=True) plugin_config = prepare_plugin_config(plugin_config, schema, defaults) - # After merging defaults, replace any None array values with their schema defaults. - # merge_with_defaults gives user config higher priority, so a None submitted by - # the client can survive the merge — this pass cleans those up. + # The defaults merge replaces a None only where the schema has a default, + # so an array the client sent as None, or left out, can still be one here. def _fix_none_arrays(cfg, props): for k, pschema in props.items(): if pschema.get('type') == 'array': @@ -2371,14 +2053,8 @@ def _prepare_plugin_config_for_save(plugin_id, plugin_config, schema, schema_mgr # Check integer first (more specific than number) if 'integer' in prop_type: if isinstance(value, str): - value_stripped = value.strip() - if value_stripped == '': - # Empty string with null allowed - already handled above, but double-check - if 'null' in prop_type: - normalized[key] = None - continue try: - normalized[key] = int(value_stripped) + normalized[key] = int(value.strip()) continue except (ValueError, TypeError, OverflowError): pass @@ -2389,14 +2065,8 @@ def _prepare_plugin_config_for_save(plugin_id, plugin_config, schema, schema_mgr # Check number (less specific, but handles floats) if 'number' in prop_type: if isinstance(value, str): - value_stripped = value.strip() - if value_stripped == '': - # Empty string with null allowed - already handled above, but double-check - if 'null' in prop_type: - normalized[key] = None - continue try: - normalized[key] = float(value_stripped) + normalized[key] = float(value.strip()) continue except (ValueError, TypeError, OverflowError): pass @@ -2410,21 +2080,7 @@ def _prepare_plugin_config_for_save(plugin_id, plugin_config, schema, schema_mgr normalized[key] = value.strip().lower() in ('true', '1', 'on', 'yes') continue - # If no conversion worked and null is allowed, try to set to None - # This handles cases where the value is an empty string or can't be converted - if 'null' in prop_type: - if isinstance(value, str): - value_stripped = value.strip() - if value_stripped == '' or value_stripped.lower() in ('null', 'none', 'undefined'): - normalized[key] = None - continue - # If it's already None, keep it - if value is None: - normalized[key] = None - continue - - # If no conversion worked, keep original value (will fail validation, but that's expected) - # Log a warning for debugging + # Nothing converted: keep the value for validation to report. logger.warning(f"Could not normalize field {field_path}: value={repr(value)}, type={type(value)}, schema_type={prop_type}") normalized[key] = value continue @@ -2550,37 +2206,24 @@ def _prepare_plugin_config_for_save(plugin_id, plugin_config, schema, schema_mgr enhanced_schema_for_filtering = _enhance_schema_with_core_properties(schema) plugin_config = _filter_config_by_schema(plugin_config, enhanced_schema_for_filtering) - # Debug logging for union type fields (temporary) - if 'rotation_settings' in plugin_config and 'random_seed' in plugin_config.get('rotation_settings', {}): - seed_value = plugin_config['rotation_settings']['random_seed'] - logger.debug(f"After normalization, random_seed value: {repr(seed_value)}, type: {type(seed_value)}") - - # Validate configuration against schema before saving + # A uniqueItems array can arrive with a repeat -- the form merges onto + # the stored list, so a stock symbol already saved and submitted again + # appears twice -- and validation would refuse the whole save for it. if schema: - # Log what we're validating for debugging - logger.info(f"Validating config for {plugin_id}") - # Only the shape. plugin_config still holds the submitted secret - # values at this point -- separate_secrets does not run until - # below -- so logging it wrote live credentials to the journal. - logger.info(f"Config keys being validated: {list(plugin_config.keys())}") - - # Schema keys including the injected core properties, for the error - enhanced_schema = _enhance_schema_with_core_properties(schema) + dedup_unique_arrays(plugin_config, schema) + if schema: is_valid, validation_errors = schema_mgr.validate_config_against_schema( plugin_config, schema, plugin_id ) if not is_valid: - # Log validation errors for debugging - logger.error(f"Config validation failed for {plugin_id}") - logger.error(f"Validation errors: {validation_errors}") - # Keys only, for the same reason as above. - logger.error(f"Config keys that failed: {list(plugin_config.keys())}") - logger.error(f"Schema properties: {list(enhanced_schema.get('properties', {}).keys())}") - - # Also print to console for immediate visibility - logger.warning("Config validation failed for plugin (see debug logs)") - + # Schema keys including the injected core properties, for the error + enhanced_schema = _enhance_schema_with_core_properties(schema) + # Keys, never values: plugin_config still holds the submitted + # secrets here (separate_secrets runs below), and logging it wrote + # live credentials to the journal. + logger.warning("Config validation failed for %s: %s (config keys: %s)", + plugin_id, validation_errors, list(plugin_config.keys())) return None, None, error_response( ErrorCode.CONFIG_VALIDATION_FAILED, 'Configuration validation failed', @@ -2704,8 +2347,13 @@ def reset_plugin_config(): # Replace all secrets with defaults current_secrets[plugin_id] = default_secrets - # Save updated configs - api_v3.config_manager.save_config(current_config) + success, error_msg = _pkg._save_config_atomic(api_v3.config_manager, current_config, create_backup=True) + if not success: + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + f"Failed to save configuration: {error_msg}", + status_code=500 + ) if default_secrets or not preserve_secrets: api_v3.config_manager.save_raw_file_content('secrets', current_secrets) @@ -2715,7 +2363,8 @@ def reset_plugin_config(): plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) if plugin_instance: merged_config = api_v3.config_manager.load_config() - plugin_full_config = merged_config.get(plugin_id, {}) + plugin_full_config = _pkg._prepared_plugin_config( + plugin_id, merged_config.get(plugin_id, {})) if hasattr(plugin_instance, 'on_config_change'): plugin_instance.on_config_change(plugin_full_config) except Exception as hook_err: @@ -2761,13 +2410,8 @@ def execute_plugin_action(): if plugin_id is None: return jsonify({'status': 'error', 'message': 'Invalid plugin_id'}), 400 - # Get plugin directory - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): + plugin_dir = _plugin_directory(plugin_id) + if not plugin_dir: return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 # Load manifest to get action definition @@ -2989,6 +2633,10 @@ sys.exit(proc.returncode) 'message': 'Could not generate authorization URL' }), 400 except Exception as e: + # Not a copy of the blueprint handler: without it, a + # TimeoutExpired from the plugin's script would reach + # this route's own `except subprocess.TimeoutExpired` + # and be answered as a 408 "Action timed out". logger.error("Error executing action step 1", exc_info=True) return jsonify({ 'status': 'error', @@ -3121,7 +2769,7 @@ def upload_plugin_asset(): if total_size + file_size > max_total_size: return jsonify({ 'status': 'error', - 'message': f'Upload would exceed 50MB total storage limit' + 'message': 'Upload would exceed 50MB total storage limit' }), 400 # Validate file is actually an image (check magic bytes) @@ -3222,13 +2870,8 @@ def serve_plugin_static(plugin_id, file_path): if not safe_parts: return jsonify({'status': 'error', 'message': 'Invalid file path'}), 400 - # Get plugin directory - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(safe_plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / safe_plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): + plugin_dir = _plugin_directory(safe_plugin_id) + if not plugin_dir: return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 # Containment is still checked after resolving: name validation cannot @@ -3300,14 +2943,8 @@ def upload_calendar_credentials(): 'message': 'File does not appear to be a valid Google OAuth credentials file' }), 400 - # Get plugin directory - plugin_id = 'calendar' - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): + plugin_dir = _plugin_directory('calendar') + if not plugin_dir: return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 # Save file to plugin directory @@ -3316,7 +2953,6 @@ def upload_calendar_credentials(): # Backup existing file if it exists if credentials_path.exists(): backup_path = Path(plugin_dir) / f'credentials.json.backup.{int(_pkg.time.time())}' - import shutil shutil.copy2(credentials_path, backup_path) _prune_credential_backups(Path(plugin_dir)) diff --git a/web_interface/blueprints/api_v3/starlark.py b/web_interface/blueprints/api_v3/starlark.py index 08d592c7..cdc21fa4 100644 --- a/web_interface/blueprints/api_v3/starlark.py +++ b/web_interface/blueprints/api_v3/starlark.py @@ -1,7 +1,7 @@ """Starlark / Tronbyte app management routes. -Routes decorate the shared `api_v3` Blueprint from ._common, so their -endpoint names are unchanged by living here. +Routes decorate the shared `api_v3` Blueprint from the package `__init__`, +so their endpoint names are unchanged by living here. """ import signal import threading @@ -333,7 +333,6 @@ def uninstall_starlark_app(app_id): else: # Standalone: remove app dir and manifest entry. app_dir is the # path _validate_starlark_app_path checked, not a fresh join. - import shutil if app_dir.exists(): shutil.rmtree(app_dir) with _starlark_manifest_lock(): @@ -716,7 +715,6 @@ def install_from_tronbyte_repository(): success = _install_star_file(app_id, temp_path, install_metadata, assets_dir=temp_assets_dir) finally: # Clean up temp assets directory - import shutil try: shutil.rmtree(temp_assets_dir) except OSError: diff --git a/web_interface/blueprints/api_v3/system.py b/web_interface/blueprints/api_v3/system.py index 405c425f..ab3403a2 100644 --- a/web_interface/blueprints/api_v3/system.py +++ b/web_interface/blueprints/api_v3/system.py @@ -1,7 +1,7 @@ """Service control, updates, versions and system status. -Routes decorate the shared `api_v3` Blueprint from ._common, so their -endpoint names are unchanged by living here. +Routes decorate the shared `api_v3` Blueprint from the package `__init__`, +so their endpoint names are unchanged by living here. """ from web_interface.blueprints.api_v3 import ( Any, Dict, PROJECT_ROOT, Path, _GIT, _UPDATE_CHECK_TTL, @@ -14,6 +14,8 @@ from web_interface.blueprints.api_v3 import ( ) import threading +from web_interface.cache import get_cached, set_cached +from web_interface.system_metrics import collect_system_metrics, format_uptime import web_interface.blueprints.api_v3 as _pkg # Read through the module rather than bound by value: tests patch these # as module attributes, and a value binding would not see the patch. @@ -23,91 +25,25 @@ import web_interface.blueprints.api_v3 as _pkg @api_v3.route('/system/status', methods=['GET']) def get_system_status(): - """Get system status""" - # Check cache first (10 second TTL for system status) - try: - from web_interface.cache import get_cached, set_cached - cached_result = get_cached('system_status', ttl_seconds=10) - if cached_result is not None: - return jsonify({'status': 'success', 'data': cached_result}) - except ImportError: - # Cache not available, continue without caching - get_cached = None - set_cached = None + """CPU, memory, disk, temperature and uptime, plus the display service state. - # Import psutil for system monitoring - try: - import psutil - except ImportError: - # Fallback if psutil not available - return jsonify({ - 'status': 'error', - 'message': 'psutil not available for system monitoring' - }), 503 + ``data`` carries every key of system_metrics.collect_system_metrics() -- + the numbers the live status stream sends -- and ``timestamp``, ``uptime`` + (formatted) and ``service_active``. A metric that cannot be read is null. + Cached for 10 seconds. + """ + cached_result = get_cached('system_status', ttl_seconds=10) + if cached_result is not None: + return jsonify({'status': 'success', 'data': cached_result}) - # Get system metrics using psutil - cpu_percent = psutil.cpu_percent(interval=0.1) # Short interval for responsiveness - memory = psutil.virtual_memory() - memory_percent = memory.percent - disk = psutil.disk_usage('/') - disk_percent = disk.percent - - # Calculate uptime - boot_time = psutil.boot_time() - uptime_seconds = _pkg.time.time() - boot_time - uptime_hours = uptime_seconds / 3600 - uptime_days = uptime_hours / 24 - - # Format uptime string - if uptime_days >= 1: - uptime_str = f"{int(uptime_days)}d {int(uptime_hours % 24)}h" - elif uptime_hours >= 1: - uptime_str = f"{int(uptime_hours)}h {int((uptime_seconds % 3600) / 60)}m" - else: - uptime_str = f"{int(uptime_seconds / 60)}m" - - # Get CPU temperature (Raspberry Pi) - cpu_temp = None - try: - temp_file = '/sys/class/thermal/thermal_zone0/temp' - if os.path.exists(temp_file): - with open(temp_file, 'r') as f: - temp_millidegrees = int(f.read().strip()) - cpu_temp = temp_millidegrees / 1000.0 # Convert to Celsius - except (IOError, ValueError, OSError): - # Temperature sensor not available or error reading - cpu_temp = None - - # Get display service status - service_status = _get_display_service_status() - - status = { - 'timestamp': _pkg.time.time(), - 'uptime': uptime_str, - 'uptime_seconds': int(uptime_seconds), - 'service_active': service_status.get('active', False), - 'cpu_percent': round(cpu_percent, 1), - 'memory_used_percent': round(memory_percent, 1), - 'memory_total_mb': round(memory.total / (1024 * 1024), 1), - 'memory_used_mb': round(memory.used / (1024 * 1024), 1), - # MemAvailable, not total-minus-used: it accounts for reclaimable - # page cache, so it is what actually predicts memory trouble. A - # board can read 70% "used" and be fine, or read the same and be - # about to fail fork(), and only this number tells them apart. - 'memory_available_mb': round(memory.available / (1024 * 1024), 1), - 'cpu_temp': round(cpu_temp, 1) if cpu_temp is not None else None, - 'disk_used_percent': round(disk_percent, 1), - 'disk_total_gb': round(disk.total / (1024 * 1024 * 1024), 1), - 'disk_used_gb': round(disk.used / (1024 * 1024 * 1024), 1) - } - - # Cache the result if available - if set_cached: - try: - set_cached('system_status', status, ttl_seconds=10) - except Exception: - pass # Cache write failed, but continue + # A short blocking sample: this may be the first cpu_percent call in the + # process, and a non-blocking first call has nothing to measure against. + status = collect_system_metrics(cpu_interval=0.1) + status['timestamp'] = _pkg.time.time() + status['uptime'] = format_uptime(status['uptime_seconds']) + status['service_active'] = _get_display_service_status().get('active', False) + set_cached('system_status', status, ttl_seconds=10) return jsonify({'status': 'success', 'data': status}) @api_v3.route('/system/version', methods=['GET']) def get_system_version(): diff --git a/web_interface/blueprints/api_v3/wifi.py b/web_interface/blueprints/api_v3/wifi.py index 0f03a878..f504a8a8 100644 --- a/web_interface/blueprints/api_v3/wifi.py +++ b/web_interface/blueprints/api_v3/wifi.py @@ -1,7 +1,7 @@ """Wi-Fi scanning, connection and status routes. -Routes decorate the shared `api_v3` Blueprint from ._common, so their -endpoint names are unchanged by living here. +Routes decorate the shared `api_v3` Blueprint from the package `__init__`, +so their endpoint names are unchanged by living here. """ import threading import time @@ -372,7 +372,12 @@ def set_auto_enable_ap_mode(): wifi_manager = WiFiManager() wifi_manager.config["auto_enable_ap_mode"] = auto_enable - wifi_manager._save_config() + if not wifi_manager._save_config(): + return jsonify({ + 'status': 'error', + 'message': (f'Could not save the setting to {wifi_manager.config_path}; ' + 'check that the web interface user can write it.'), + }), 500 return jsonify({ 'status': 'success', diff --git a/web_interface/blueprints/pages_v3.py b/web_interface/blueprints/pages_v3.py index e2b82c3d..3f55bc4c 100644 --- a/web_interface/blueprints/pages_v3.py +++ b/web_interface/blueprints/pages_v3.py @@ -1,6 +1,5 @@ from flask import Blueprint, Response, render_template, jsonify, url_for from jinja2 import TemplateNotFound -from markupsafe import escape from html.parser import HTMLParser import json import logging @@ -163,41 +162,13 @@ def index(): @pages_v3.route('/partials/') def load_partial(partial_name): - """Load HTMX partials dynamically""" + """One tab's HTML for HTMX, by the names in _PARTIAL_LOADERS; 404 otherwise.""" + loader = _PARTIAL_LOADERS.get(partial_name) + if loader is None: + return "Partial not found", 404 try: - # Map partial names to specific data loading - if partial_name == 'overview': - return _load_overview_partial() - elif partial_name == 'general': - return _load_general_partial() - elif partial_name == 'display': - return _load_display_partial() - elif partial_name == 'durations': - return _load_durations_partial() - elif partial_name == 'schedule': - return _load_schedule_partial() - elif partial_name == 'plugins': - return _load_plugins_partial() - elif partial_name == 'fonts': - return _load_fonts_partial() - elif partial_name == 'logs': - return _load_logs_partial() - elif partial_name == 'raw-json': - return _load_raw_json_partial() - elif partial_name == 'backup-restore': - return _load_backup_restore_partial() - elif partial_name == 'wifi': - return _load_wifi_partial() - elif partial_name == 'cache': - return _load_cache_partial() - elif partial_name == 'operation-history': - return _load_operation_history_partial() - elif partial_name == 'tools': - return _load_tools_partial() - else: - return "Partial not found", 404 - - except Exception as e: + return loader() + except Exception: logger.error("Error loading partial %s", partial_name, exc_info=True) return "Error loading partial", 500 @@ -297,19 +268,7 @@ def serve_plugin_web_ui(plugin_id, filename): return 'Plugin manager not available', 503, {'Content-Type': 'text/plain'} try: - _plugins_base = Path(pages_v3.plugin_manager.plugins_dir).resolve() - - _plugin_dir = resolve_under(_plugins_base, safe_id) - if _plugin_dir is None: - return 'Forbidden', 403, {'Content-Type': 'text/plain'} - - # Mirror PluginManager's ledmatrix- prefix fallback. - if not _plugin_dir.exists(): - _alt = resolve_under(_plugins_base, f'ledmatrix-{safe_id}') - if _alt is not None: - _plugin_dir = _alt - - web_ui_path = resolve_under(_plugin_dir / 'web_ui', safe_fn) + web_ui_path = resolve_under(_plugin_dir_for(safe_id) / 'web_ui', safe_fn) if web_ui_path is None: return 'Forbidden', 403, {'Content-Type': 'text/plain'} @@ -362,10 +321,11 @@ def serve_plugin_web_ui(plugin_id, filename): def _plugin_dir_for(safe_id): - """Resolve a sanitised plugin id to its directory, or None. + """A sanitised plugin id's directory, which may not exist. - Mirrors serve_plugin_web_ui: containment-guarded against the configured - plugins directory, with PluginManager's ``ledmatrix-`` prefix fallback. + Contained under the configured plugins directory, with PluginManager's + ``ledmatrix-`` prefix fallback. Raises ValueError for an id that would + leave it; the routes answer that with a 403. """ plugins_base = Path(pages_v3.plugin_manager.plugins_dir).resolve() plugin_dir = resolve_under(plugins_base, safe_id) @@ -479,45 +439,33 @@ def serve_plugin_widget(plugin_id, widget_name): def _load_overview_partial(): """Load overview partial with system stats""" - try: - if pages_v3.config_manager: - main_config = pages_v3.config_manager.load_config() - # This would be populated with real system stats via SSE - return render_template('v3/partials/overview.html', - main_config=main_config) - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + if pages_v3.config_manager: + main_config = pages_v3.config_manager.load_config() + # This would be populated with real system stats via SSE + return render_template('v3/partials/overview.html', + main_config=main_config) def _load_general_partial(): """Load general settings partial""" - try: - if pages_v3.config_manager: - main_config = pages_v3.config_manager.load_config() - try: - from web_interface.auto_update import describe_status - auto_update_status = describe_status(main_config) - except Exception: - logger.debug("Could not read auto-update status", exc_info=True) - auto_update_status = None - return render_template('v3/partials/general.html', - main_config=main_config, - auto_update_status=auto_update_status) - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + if pages_v3.config_manager: + main_config = pages_v3.config_manager.load_config() + try: + from web_interface.auto_update import describe_status + auto_update_status = describe_status(main_config) + except Exception: + logger.debug("Could not read auto-update status", exc_info=True) + auto_update_status = None + return render_template('v3/partials/general.html', + main_config=main_config, + auto_update_status=auto_update_status) def _load_display_partial(): """Load display settings partial""" - try: - if pages_v3.config_manager: - main_config = pages_v3.config_manager.load_config() - return render_template('v3/partials/display.html', - main_config=main_config, - is_pi5=is_raspberry_pi_5()) - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + if pages_v3.config_manager: + main_config = pages_v3.config_manager.load_config() + return render_template('v3/partials/display.html', + main_config=main_config, + is_pi5=is_raspberry_pi_5()) def _plugin_default_duration(plugin_id, plugin_config): """Seconds a plugin shows each screen when the Rotation page sets none. @@ -553,199 +501,168 @@ def _load_durations_partial(): value overrides the plugin (see DisplayController._get_display_duration). Pre-filling every mode would pin them all on the first save. """ - try: - if pages_v3.config_manager: - main_config = pages_v3.config_manager.load_config() - duration_groups = [] - covered_keys = set() - if pages_v3.plugin_manager: - try: - pages_v3.plugin_manager.discover_plugins() - saved = (main_config.get('display', {}) or {}).get('display_durations', {}) or {} - infos = sorted(pages_v3.plugin_manager.get_all_plugin_info(), - key=lambda i: (i.get('name') or i.get('id') or '').lower()) - for info in infos: - pid = info.get('id') - if not pid or not (main_config.get(pid, {}) or {}).get('enabled', False): - continue - modes = pages_v3.plugin_manager.get_plugin_display_modes(pid) or [pid] - covered_keys.update(modes) - default = _plugin_default_duration(pid, main_config.get(pid, {}) or {}) - duration_groups.append({ - 'plugin_id': pid, - 'plugin_name': info.get('name') or pid, - 'modes': [{'key': m, 'value': saved.get(m, ''), 'default': default} - for m in modes], - }) - # Saved keys not owned by any enabled plugin (disabled or - # uninstalled plugins) stay visible rather than vanishing. - leftovers = [{'key': k, 'value': v} for k, v in saved.items() - if k not in covered_keys] - if leftovers: - duration_groups.append({ - 'plugin_id': '', - 'plugin_name': 'Other saved entries', - 'modes': leftovers, - }) - except Exception: - logger.warning("durations: could not enumerate plugin modes", exc_info=True) - return render_template('v3/partials/durations.html', - main_config=main_config, - duration_groups=duration_groups) - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + if pages_v3.config_manager: + main_config = pages_v3.config_manager.load_config() + duration_groups = [] + covered_keys = set() + if pages_v3.plugin_manager: + try: + pages_v3.plugin_manager.discover_plugins() + saved = (main_config.get('display', {}) or {}).get('display_durations', {}) or {} + infos = sorted(pages_v3.plugin_manager.get_all_plugin_info(), + key=lambda i: (i.get('name') or i.get('id') or '').lower()) + for info in infos: + pid = info.get('id') + if not pid or not (main_config.get(pid, {}) or {}).get('enabled', False): + continue + modes = pages_v3.plugin_manager.get_plugin_display_modes(pid) or [pid] + covered_keys.update(modes) + default = _plugin_default_duration(pid, main_config.get(pid, {}) or {}) + duration_groups.append({ + 'plugin_id': pid, + 'plugin_name': info.get('name') or pid, + 'modes': [{'key': m, 'value': saved.get(m, ''), 'default': default} + for m in modes], + }) + # Saved keys not owned by any enabled plugin (disabled or + # uninstalled plugins) stay visible rather than vanishing. + leftovers = [{'key': k, 'value': v} for k, v in saved.items() + if k not in covered_keys] + if leftovers: + duration_groups.append({ + 'plugin_id': '', + 'plugin_name': 'Other saved entries', + 'modes': leftovers, + }) + except Exception: + logger.warning("durations: could not enumerate plugin modes", exc_info=True) + return render_template('v3/partials/durations.html', + main_config=main_config, + duration_groups=duration_groups) def _load_schedule_partial(): """Load schedule settings partial""" - try: - if pages_v3.config_manager: - main_config = pages_v3.config_manager.load_config() - schedule_config = main_config.get('schedule', {}) - dim_schedule_config = main_config.get('dim_schedule', {}) - # Get normal brightness for display in dim schedule UI - normal_brightness = main_config.get('display', {}).get('hardware', {}).get('brightness', 90) - return render_template('v3/partials/schedule.html', - schedule_config=schedule_config, - dim_schedule_config=dim_schedule_config, - normal_brightness=normal_brightness) - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + if pages_v3.config_manager: + main_config = pages_v3.config_manager.load_config() + schedule_config = main_config.get('schedule', {}) + dim_schedule_config = main_config.get('dim_schedule', {}) + # Get normal brightness for display in dim schedule UI + normal_brightness = main_config.get('display', {}).get('hardware', {}).get('brightness', 90) + return render_template('v3/partials/schedule.html', + schedule_config=schedule_config, + dim_schedule_config=dim_schedule_config, + normal_brightness=normal_brightness) def _load_plugins_partial(): """Load plugins management partial""" - try: - import json - from pathlib import Path - - # Load plugin data from the plugin system - plugins_data = [] + # Load plugin data from the plugin system + plugins_data = [] - # Get installed plugins if managers are available - if pages_v3.plugin_manager and pages_v3.plugin_store_manager: - try: - # Get all installed plugin info - all_plugin_info = pages_v3.plugin_manager.get_all_plugin_info() + # Get installed plugins if managers are available + if pages_v3.plugin_manager and pages_v3.plugin_store_manager: + try: + # Get all installed plugin info + all_plugin_info = pages_v3.plugin_manager.get_all_plugin_info() - # Load config once before the loop (not per-plugin) - full_config = pages_v3.config_manager.load_config() if pages_v3.config_manager else {} + # Load config once before the loop (not per-plugin) + full_config = pages_v3.config_manager.load_config() if pages_v3.config_manager else {} - # Format for the web interface - for plugin_info in all_plugin_info: - plugin_id = plugin_info.get('id') + # Format for the web interface + for plugin_info in all_plugin_info: + plugin_id = plugin_info.get('id') - # Re-read manifest from disk to ensure we have the latest metadata - manifest_path = Path(pages_v3.plugin_manager.plugins_dir) / plugin_id / "manifest.json" - if manifest_path.exists(): - try: - with open(manifest_path, 'r', encoding='utf-8') as f: - fresh_manifest = json.load(f) - # Update plugin_info with fresh manifest data - plugin_info.update(fresh_manifest) - except Exception as e: - # If we can't read the fresh manifest, use the cached one - logger.warning("Could not read fresh manifest for plugin: %s", plugin_id) + # Re-read manifest from disk to ensure we have the latest metadata + manifest_path = Path(pages_v3.plugin_manager.plugins_dir) / plugin_id / "manifest.json" + if manifest_path.exists(): + try: + with open(manifest_path, 'r', encoding='utf-8') as f: + fresh_manifest = json.load(f) + # Update plugin_info with fresh manifest data + plugin_info.update(fresh_manifest) + except Exception: + # If we can't read the fresh manifest, use the cached one + logger.warning("Could not read fresh manifest for plugin: %s", plugin_id) - # Get enabled status from config (source of truth) - # Read from config file first, fall back to plugin instance if config doesn't have the key - enabled = None - if pages_v3.config_manager: - plugin_config = full_config.get(plugin_id, {}) - # Check if 'enabled' key exists in config (even if False) - if 'enabled' in plugin_config: - enabled = bool(plugin_config['enabled']) - - # Fallback to plugin instance if config doesn't have enabled key - if enabled is None: - plugin_instance = pages_v3.plugin_manager.get_plugin(plugin_id) - if plugin_instance: - enabled = plugin_instance.enabled - else: - # Default to True if no config key and plugin not loaded (matches BasePlugin default) - enabled = True + # Get enabled status from config (source of truth) + # Read from config file first, fall back to plugin instance if config doesn't have the key + enabled = None + if pages_v3.config_manager: + plugin_config = full_config.get(plugin_id, {}) + # Check if 'enabled' key exists in config (even if False) + if 'enabled' in plugin_config: + enabled = bool(plugin_config['enabled']) + + # Fallback to plugin instance if config doesn't have enabled key + if enabled is None: + plugin_instance = pages_v3.plugin_manager.get_plugin(plugin_id) + if plugin_instance: + enabled = plugin_instance.enabled + else: + # Default to True if no config key and plugin not loaded (matches BasePlugin default) + enabled = True - # Get verified status from store registry (no GitHub API calls needed) - store_info = pages_v3.plugin_store_manager.get_registry_info(plugin_id) - verified = store_info.get('verified', False) if store_info else False + # Get verified status from store registry (no GitHub API calls needed) + store_info = pages_v3.plugin_store_manager.get_registry_info(plugin_id) + verified = store_info.get('verified', False) if store_info else False - last_updated = plugin_info.get('last_updated') - last_commit = plugin_info.get('last_commit') or plugin_info.get('last_commit_sha') - branch = plugin_info.get('branch') + last_updated = plugin_info.get('last_updated') + last_commit = plugin_info.get('last_commit') or plugin_info.get('last_commit_sha') + branch = plugin_info.get('branch') - if store_info: - last_updated = last_updated or store_info.get('last_updated') or store_info.get('last_updated_iso') - last_commit = last_commit or store_info.get('last_commit') or store_info.get('last_commit_sha') - branch = branch or store_info.get('branch') or store_info.get('default_branch') + if store_info: + last_updated = last_updated or store_info.get('last_updated') or store_info.get('last_updated_iso') + last_commit = last_commit or store_info.get('last_commit') or store_info.get('last_commit_sha') + branch = branch or store_info.get('branch') or store_info.get('default_branch') - plugins_data.append({ - 'id': plugin_id, - 'name': plugin_info.get('name', plugin_id), - 'author': plugin_info.get('author', 'Unknown'), - 'category': plugin_info.get('category', 'General'), - 'description': plugin_info.get('description', 'No description available'), - 'tags': plugin_info.get('tags', []), - 'enabled': enabled, - 'verified': verified, - 'loaded': plugin_info.get('loaded', False), - 'last_updated': last_updated, - 'last_commit': last_commit, - 'branch': branch - }) - except Exception as e: - logger.error("Error loading plugin data", exc_info=True) + plugins_data.append({ + 'id': plugin_id, + 'name': plugin_info.get('name', plugin_id), + 'author': plugin_info.get('author', 'Unknown'), + 'category': plugin_info.get('category', 'General'), + 'description': plugin_info.get('description', 'No description available'), + 'tags': plugin_info.get('tags', []), + 'enabled': enabled, + 'verified': verified, + 'loaded': plugin_info.get('loaded', False), + 'last_updated': last_updated, + 'last_commit': last_commit, + 'branch': branch + }) + except Exception: + logger.error("Error loading plugin data", exc_info=True) - return render_template('v3/partials/plugins.html', - plugins=plugins_data) - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + return render_template('v3/partials/plugins.html', + plugins=plugins_data) def _load_fonts_partial(): """Load fonts management partial""" - try: - # This would load font data from the font system - fonts_data = {} # Placeholder for font data - return render_template('v3/partials/fonts.html', - fonts=fonts_data) - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + # This would load font data from the font system + fonts_data = {} # Placeholder for font data + return render_template('v3/partials/fonts.html', + fonts=fonts_data) def _load_logs_partial(): """Load logs viewer partial""" - try: - return render_template('v3/partials/logs.html') - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + return render_template('v3/partials/logs.html') def _load_raw_json_partial(): """Load raw JSON editor partial""" - try: - if pages_v3.config_manager: - main_config_data = pages_v3.config_manager.get_raw_file_content('main') - secrets_config_data = pages_v3.config_manager.get_raw_file_content('secrets') - main_config_json = json.dumps(main_config_data, indent=4) - secrets_config_json = json.dumps(secrets_config_data, indent=4) + if pages_v3.config_manager: + main_config_data = pages_v3.config_manager.get_raw_file_content('main') + secrets_config_data = pages_v3.config_manager.get_raw_file_content('secrets') + main_config_json = json.dumps(main_config_data, indent=4) + secrets_config_json = json.dumps(secrets_config_data, indent=4) - return render_template('v3/partials/raw_json.html', - main_config_json=main_config_json, - secrets_config_json=secrets_config_json, - main_config_path=pages_v3.config_manager.get_config_path(), - secrets_config_path=pages_v3.config_manager.get_secrets_path()) - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + return render_template('v3/partials/raw_json.html', + main_config_json=main_config_json, + secrets_config_json=secrets_config_json, + main_config_path=pages_v3.config_manager.get_config_path(), + secrets_config_path=pages_v3.config_manager.get_secrets_path()) def _load_backup_restore_partial(): """Load backup & restore partial.""" - try: - return render_template('v3/partials/backup_restore.html') - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + return render_template('v3/partials/backup_restore.html') @pages_v3.route('/setup') def captive_setup(): @@ -754,27 +671,15 @@ def captive_setup(): def _load_wifi_partial(): """Load WiFi setup partial""" - try: - return render_template('v3/partials/wifi.html') - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + return render_template('v3/partials/wifi.html') def _load_cache_partial(): """Load cache management partial""" - try: - return render_template('v3/partials/cache.html') - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + return render_template('v3/partials/cache.html') def _load_operation_history_partial(): """Load operation history partial""" - try: - return render_template('v3/partials/operation_history.html') - except Exception as e: - logger.error("Error loading partial", exc_info=True) - return "Error loading partial", 500 + return render_template('v3/partials/operation_history.html') def _load_tools_partial(): @@ -789,6 +694,24 @@ def _load_tools_partial(): return "[Pages V3][Tools] Failed to load due to a file system error. Check logs.", 500 +_PARTIAL_LOADERS = { + 'overview': _load_overview_partial, + 'general': _load_general_partial, + 'display': _load_display_partial, + 'durations': _load_durations_partial, + 'schedule': _load_schedule_partial, + 'plugins': _load_plugins_partial, + 'fonts': _load_fonts_partial, + 'logs': _load_logs_partial, + 'raw-json': _load_raw_json_partial, + 'backup-restore': _load_backup_restore_partial, + 'wifi': _load_wifi_partial, + 'cache': _load_cache_partial, + 'operation-history': _load_operation_history_partial, + 'tools': _load_tools_partial, +} + + def _load_plugin_config_partial(plugin_id): """ Load plugin configuration partial - server-side rendered form. @@ -960,7 +883,7 @@ def _load_plugin_config_partial(plugin_id): web_ui_actions=web_ui_actions ) - except Exception as e: + except Exception: logger.error("Error loading plugin config partial for %s", plugin_id, exc_info=True) return '
Error loading plugin config; see logs for details
', 500 @@ -1041,6 +964,6 @@ def _load_starlark_config_partial(app_id): last_render_time=None, ) - except Exception as e: + except Exception: logger.error("[Pages V3] Error loading starlark config for app", exc_info=True) return '
Error loading starlark config; see logs for details
', 500 diff --git a/web_interface/cache.py b/web_interface/cache.py index 977600a2..643826b1 100644 --- a/web_interface/cache.py +++ b/web_interface/cache.py @@ -6,8 +6,8 @@ seconds or minutes (the font catalog, the system-status snapshot, systemctl checks). It is per-process and in-memory only; data shared with the display service goes through ``src.cache_manager.CacheManager`` instead. -Separated from app.py to avoid circular imports: blueprints import the -module-level helpers below lazily, inside their request handlers. +Separate from app.py so the blueprints can import it without importing the +app; it imports nothing from the project, so they import it at module top. """ import threading import time diff --git a/web_interface/display_preview.py b/web_interface/display_preview.py new file mode 100644 index 00000000..47f3790a --- /dev/null +++ b/web_interface/display_preview.py @@ -0,0 +1,36 @@ +"""The display preview the web UI shows, read from the display service's snapshot. + +GET /api/v3/display/current and the /api/v3/stream/display SSE stream both +answer with preview_payload(). Free of Flask and app imports, like +system_metrics, so a blueprint can use it without constructing the app. +""" + +import base64 +import time +from typing import Any, Dict, Optional + +#: Where DisplayManager writes the snapshot. Written atomically (a temp file +#: and os.replace), so a read never sees half a PNG. +SNAPSHOT_PATH = "/tmp/led_matrix_preview.png" # nosec B108 - fixed path shared with display_manager + + +def read_snapshot_base64(path: Optional[str] = None) -> str: + """The snapshot PNG's bytes, base64-encoded. + + The file already is a PNG, so it is passed through as it is: decoding and + re-encoding it produced the same image for more CPU on the Pi. Raises + OSError when there is no snapshot or it cannot be read. + """ + with open(path or SNAPSHOT_PATH, 'rb') as f: + return base64.b64encode(f.read()).decode('ascii') + + +def preview_payload(width: int, height: int, image: Optional[str]) -> Dict[str, Any]: + """``{timestamp, width, height, image}``; ``image`` is None when there is + no snapshot to show.""" + return { + 'timestamp': time.time(), + 'width': width, + 'height': height, + 'image': image, + } diff --git a/web_interface/start.py b/web_interface/start.py index f62ff086..7b37de78 100644 --- a/web_interface/start.py +++ b/web_interface/start.py @@ -11,10 +11,16 @@ import sys import logging from pathlib import Path +logger = logging.getLogger('web_interface.start') + +# No route to host, broken pipe, connection reset: a client went away. +_CLIENT_DISCONNECT_ERRNOS = (113, 32, 104) + + def get_local_ips(): """Get list of local IP addresses the service will be accessible on.""" ips = [] - + # Check if AP mode is active try: result = subprocess.run( @@ -27,7 +33,7 @@ def get_local_ips(): ips.append("192.168.4.1 (AP Mode)") except Exception: # nosec B110 - AP mode IP detection is non-critical startup info; systemctl may not exist pass - + # Get IPs from hostname -I try: result = subprocess.run( @@ -43,7 +49,7 @@ def get_local_ips(): ips.append(ip) except Exception: # nosec B110 - hostname -I output parsing; non-critical startup info pass - + # Fallback: try socket method if not ips: try: @@ -57,7 +63,7 @@ def get_local_ips(): s.close() except Exception: pass - + return ips if ips else ["localhost"] def main(): @@ -65,15 +71,15 @@ def main(): # Change to project root directory project_root = Path(__file__).parent.parent os.chdir(project_root) - + # Add to Python path sys.path.insert(0, str(project_root)) - + # Configure logging to suppress non-critical socket errors # These occur when clients disconnect and are harmless werkzeug_logger = logging.getLogger('werkzeug') original_log_exception = werkzeug_logger.error - + def log_exception_filtered(message, *args, **kwargs): """Filter out non-critical socket errors from werkzeug logs.""" if isinstance(message, str): @@ -90,51 +96,38 @@ def main(): if 'exc_info' in kwargs and kwargs['exc_info']: exc_type, exc_value, exc_tb = kwargs['exc_info'] if isinstance(exc_value, OSError): - # Suppress common non-critical socket errors - if exc_value.errno in (113, 32, 104): # No route to host, Broken pipe, Connection reset + if exc_value.errno in _CLIENT_DISCONNECT_ERRNOS: werkzeug_logger.debug(message, *args, **kwargs) return # Log everything else normally original_log_exception(message, *args, **kwargs) - + werkzeug_logger.error = log_exception_filtered - - # Import and run the Flask app + + # Importing the app also sets up logging, so the lines below reach the + # journal through it. from web_interface.app import app, start_auto_update_scheduler start_auto_update_scheduler() - print("Starting LED Matrix Web Interface V3...") - print("Web server binding to: 0.0.0.0:5000") - - # Get and display accessible IP addresses - ips = get_local_ips() - if ips: - print("Access the interface at:") - for ip in ips: - if "AP Mode" in ip: - print(" - http://192.168.4.1:5000 (AP Mode - connect to LEDMatrix-Setup WiFi)") - else: - print(f" - http://{ip}:5000") - else: - print(" - http://localhost:5000 (local only)") - print(" - http://:5000 (replace with your Pi's IP address)") - - # Run the web server with error handling for client disconnections + logger.info("Starting LED Matrix Web Interface V3, binding to 0.0.0.0:5000") + # get_local_ips() always returns at least "localhost". + logger.info("Access the interface at:") + for ip in get_local_ips(): + if "AP Mode" in ip: + logger.info(" - http://192.168.4.1:5000 (AP Mode - connect to LEDMatrix-Setup WiFi)") + else: + logger.info(" - http://%s:5000", ip) + try: # threaded=True is Flask's default since 1.0, but set it explicitly - # so it's self-documenting: the two /api/v3/stream/* SSE endpoints + # so it's self-documenting: the three /api/v3/stream/* SSE endpoints # hold long-lived connections and would starve other requests under # a single-threaded server. app.run(host='0.0.0.0', port=5000, debug=False, threaded=True) - except (OSError, BrokenPipeError) as e: - # Suppress non-critical socket errors (client disconnections) - if isinstance(e, OSError) and e.errno in (113, 32, 104): # No route to host, Broken pipe, Connection reset - werkzeug_logger.debug(f"Client disconnected: {e}", exc_info=True) - # Re-raise only if it's not a client disconnection error - if e.errno not in (113, 32, 104): - raise - else: + except OSError as e: + if e.errno not in _CLIENT_DISCONNECT_ERRNOS: raise + werkzeug_logger.debug("Client disconnected: %s", e, exc_info=True) if __name__ == '__main__': main() diff --git a/web_interface/static/v3/js/widgets/README.md b/web_interface/static/v3/js/widgets/README.md index be1f2189..36378fb5 100644 --- a/web_interface/static/v3/js/widgets/README.md +++ b/web_interface/static/v3/js/widgets/README.md @@ -1,20 +1,63 @@ # LEDMatrix Widget Development Guide -## Overview +Widgets are the controls the web UI draws for fields in a plugin's +`config_schema.json`. A field picks one with `"x-widget": ""`. This +directory holds the built-in widgets, the registry they register with, and +the loader for widgets a plugin ships itself. -The LEDMatrix Widget Registry system allows plugins to use reusable UI components (widgets) for configuration forms. This system enables: +The page loads every file here as one bundle, `/assets/widgets.js`, built by +[`web_interface/widget_bundle.py`](../../../../widget_bundle.py) in the order +set by `BUNDLE_ORDER` there. -- **Reusable Components**: Use existing widgets (file upload, checkboxes, etc.) without custom code -- **Custom Widgets**: Create plugin-specific widgets without modifying the LEDMatrix codebase -- **Backwards Compatibility**: Existing plugins continue to work without changes +## Built-in widgets -## Available Core Widgets +| `x-widget` | Field type | What it draws | +|---|---|---| +| `text-input` | string | Text field with optional length limits | +| `textarea` | string | Multi-line text | +| `email-input` | string | Email field with format check | +| `url-input` | string | URL field with format check | +| `password-input` | string | Password field with show/hide toggle | +| `select-dropdown` | string | Dropdown for an `enum` | +| `radio-group` | string | Radio buttons for an `enum` | +| `date-picker` | string | Date input | +| `time-picker` | string | Time input, `HH:MM` (24-hour) | +| `color-picker` | string or array | Colour picker; hex string, or `[r, g, b]` on an array field | +| `font-selector` | string | Font from `assets/fonts/` (TTF and BDF), fetched from the API | +| `timezone-selector` | string | IANA timezone, grouped by region | +| `file-upload-single` | string | One image upload; stores the uploaded file's relative path | +| `google-oauth` | string | Step 2 of the calendar plugin's Google sign-in | +| `plugin-file-manager` | null | Inline file manager driven by the plugin's `web_ui_actions` | +| `json-file-manager` | null | JSON data-file manager driven by `web_ui_actions` | +| `toggle-switch` | boolean | On/off switch | +| `slider` | integer / number | Range slider using `minimum` / `maximum` | +| `number-input` | integer / number | Number field with min/max check | +| `file-upload` | array | Multi-image upload with preview, delete and scheduling | +| `checkbox-group` | array | Checkboxes for an array of `enum` items | +| `day-selector` | array | Days of the week | +| `custom-feeds` | array | RSS feed table with per-feed logo upload | +| `array-table` | array | Table editor for an array of objects | +| `google-calendar-picker` | array | Calendars from the user's Google account | +| `schedule-picker` | object | Enable toggle, global/per-day mode and times | +| `time-range` | object | Start and end time pair | +| `style-editor` | object | One row per display element: font, size, colour, alignment, offsets | -### 1. File Upload Widget (`file-upload`) +Other files here: -Upload and manage image files with drag-and-drop support, preview, delete, and scheduling. +| File | Purpose | +|---|---| +| `registry.js` | `window.LEDMatrixWidgets`: `register()`, `get()` | +| `base-widget.js` | Shared helpers (`escapeHtml`, `sanitizeId`) other widgets use | +| `notification.js` | Toast notifications; owns `window.showNotification` | +| `plugin-order-list.js` | Drag-and-drop plugin order list used by the Display and Durations tabs (`window.PluginOrderList`) | +| `plugin-loader.js` | Loads a plugin-supplied widget on demand | +| `example-color-picker.js` | Example custom widget. Not bundled: it registers `color-picker` and would replace the real one | + +Each widget file's header comment gives its schema options. The sections +below cover the ones that need more than a line. + +### `file-upload` -**Schema Configuration:** ```json { "type": "array", @@ -28,45 +71,39 @@ Upload and manage image files with drag-and-drop support, preview, delete, and s } ``` -**Features:** -- Drag and drop file upload -- Image preview with thumbnails -- Delete functionality -- Schedule images to show at specific times -- Progress indicators during upload +### `file-upload-single` -### 2. Checkbox Group Widget (`checkbox-group`) +Uploads one image to the plugin's asset folder +(`assets/plugins//uploads/`) and stores the returned relative path +in a string field. `plugin_id` is filled in from the page; don't put it in +the schema. Use it for per-row images inside an `array-table`. -Multi-select checkboxes for array fields with enum items. - -**Schema Configuration:** ```json { - "type": "array", - "x-widget": "checkbox-group", - "items": { + "image_path": { "type": "string", - "enum": ["option1", "option2", "option3"] - }, - "x-options": { - "labels": { - "option1": "Option 1 Label", - "option2": "Option 2 Label" + "x-widget": "file-upload-single", + "x-upload-config": { + "allowed_types": ["image/png", "image/jpeg", "image/bmp", "image/gif"], + "max_size_mb": 5 } } } ``` -**Features:** -- Multiple selection from enum list -- Custom labels for each option -- Automatic JSON array serialization +### `checkbox-group` -### 3. Custom Feeds Widget (`custom-feeds`) +```json +{ + "type": "array", + "x-widget": "checkbox-group", + "items": {"type": "string", "enum": ["option1", "option2", "option3"]}, + "x-options": {"labels": {"option1": "Option 1 Label", "option2": "Option 2 Label"}} +} +``` -Table-based RSS feed editor with logo uploads. +### `custom-feeds` -**Schema Configuration:** ```json { "type": "array", @@ -74,539 +111,260 @@ Table-based RSS feed editor with logo uploads. "items": { "type": "object", "properties": { - "name": { "type": "string" }, - "url": { "type": "string", "format": "uri" }, - "enabled": { "type": "boolean" }, - "logo": { "type": "object" } + "name": {"type": "string"}, + "url": {"type": "string", "format": "uri"}, + "enabled": {"type": "boolean"}, + "logo": {"type": "object"} } }, "maxItems": 50 } ``` -**Features:** -- Add/remove feed rows -- Logo upload per feed -- Enable/disable individual feeds -- Automatic row re-indexing +### `plugin-file-manager` -### Other Built-in Widgets - -In addition to the three documented above, these widgets are -registered and ready to use via `x-widget`: - -**Inputs:** -- `text-input` — Plain text field with optional length constraints -- `textarea` — Multi-line text input -- `number-input` — Numeric input with min/max validation -- `email-input` — Email field with format validation -- `url-input` — URL field with format validation -- `password-input` — Password field with show/hide toggle - -**Selectors:** -- `select-dropdown` — Single-select dropdown for `enum` fields -- `radio-group` — Radio buttons for `enum` fields (alternative to dropdown) -- `toggle-switch` — Boolean toggle (alternative to a checkbox) -- `slider` — Numeric range slider for `integer`/`number` with `min`/`max` -- `color-picker` — RGB color picker; outputs `[r, g, b]` arrays -- `font-selector` — Picks from fonts in `assets/fonts/` (TTF + BDF) -- `timezone-selector` — IANA timezone picker - -**Date / time / scheduling:** -- `date-picker` — Single date input -- `day-selector` — Days-of-week multi-select (Mon–Sun checkboxes) -- `time-range` — Start/end time pair (e.g. for dim schedules) -- `schedule-picker` — Full cron-style or weekday/time schedule editor - -**Composite / data-source:** -- `array-table` — Generic table editor for arrays of objects -- `google-calendar-picker` — Picks from the user's authenticated Google - Calendars (used by the calendar plugin) - -**Internal (typically not used directly by plugins):** -- `notification` — Toast notification helper -- `base-widget` — Base class other widgets extend - -The canonical source for each widget's exact schema and options is the -file in this directory (e.g., `slider.js`, `color-picker.js`). If you -need a feature one of these doesn't support, see "Creating Custom -Widgets" below. - -## Using Existing Widgets - -To use an existing widget in your plugin's `config_schema.json`, simply add the `x-widget` property to your field definition: +A card grid, upload zone, create/delete dialogs and a table editor for the +plugin's data files, rendered inline. File operations call +`/api/v3/plugins/action` as soon as the user acts; they are not part of +**Save Configuration**. `plugin_id` is filled in from the page. ```json { - "properties": { - "my_images": { - "type": "array", - "x-widget": "file-upload", - "x-upload-config": { - "plugin_id": "my-plugin", - "max_files": 5 - } + "file_manager": { + "type": "null", + "title": "Data Files", + "x-widget": "plugin-file-manager", + "x-widget-config": { + "actions": { + "list": "list-files", + "get": "get-file", + "save": "save-file", + "upload": "upload-file", + "delete": "delete-file", + "create": "create-file", + "toggle": "toggle-category" + }, + "upload_hint": "JSON files with day numbers 1–365 as keys", + "directory_label": "my_data/", + "create_fields": [ + {"key": "category_name", "label": "Category Name", + "pattern": "^[a-z0-9_]+$", "hint": "Lowercase letters, numbers, underscores"}, + {"key": "display_name", "label": "Display Name", "hint": "Optional"} + ] } } } ``` -The widget will be automatically rendered when the plugin configuration form is loaded. +The action ids refer to entries in the plugin's `web_ui_actions` +([docs/PLUGIN_WEB_UI_ACTIONS.md](../../../../../docs/PLUGIN_WEB_UI_ACTIONS.md)). +`list` is required: without it the widget stays on its loading state. Leave +out any other action to hide its control. The editor shows a table when a +file is an object of objects with the same keys, otherwise a JSON text area. -## Creating Custom Widgets +## Schema keywords the form understands -### Step 1: Create Widget File +These work on any field, with or without a widget. -Create a JavaScript file in your plugin directory (e.g., `widgets/my-widget.js`): +### Option labels: `x-options.labels` + +A plain `enum` renders as a dropdown whose option text is the value with +underscores replaced and title case applied (`day_first` → "Day First"). +`x-options.labels` sets the visible text instead: + +```json +{ + "date_format": { + "type": "string", + "enum": ["abbrev", "numeric", "day_first"], + "default": "abbrev", + "x-options": {"labels": {"abbrev": "Sep 19", "numeric": "9/19", "day_first": "19 Sep"}} + } +} +``` + +Labels are display only; the stored value is still the enum value. The map +may be partial. Older cores ignore `x-options` and show the fallback text. +`array-table` columns accept the same `x-options.labels`, but their fallback +is the raw value (so a ticker symbol `aapl` stays `aapl`). + +### Advanced settings: `x-advanced` + +`"x-advanced": true` on a top-level, non-object property moves it into a +collapsed **Advanced Settings** section at the bottom of the plugin's page. +Use it for settings most users never change (timeouts, cache TTLs, styling +overrides); keep anything needed to get the plugin working in the main form. +The settings search still finds and expands advanced fields. It is ignored +on `object` properties and by older cores. + +### Hidden fields: `x-display: "hidden"` + +`"x-display": "hidden"` keeps a property in the schema without drawing a +control, for a deprecated key that existing configs still carry or an +internal value such as a generated row id. + +- Not rendered at any depth: top level, inside an object section, or as a + column or row-editor field of an array of objects. Hidden fields are left + out of Advanced Settings and the settings search. +- Saving the form never changes a hidden value. Array rows carry it through; + a new row gets no value. +- A JSON `POST /api/v3/plugins/config` can still set it. +- Older cores ignore the flag and render the field. + +## Creating a custom widget + +### 1. Write the widget + +Put it in your plugin's `widgets/` directory as `widgets/.js`. That +directory is the only place the core serves plugin widgets from. ```javascript -// Ensure LEDMatrixWidgets registry is available -if (typeof window.LEDMatrixWidgets === 'undefined') { - console.error('LEDMatrixWidgets registry not found'); - return; -} +(function () { + 'use strict'; + if (typeof window.LEDMatrixWidgets === 'undefined') { + console.error('LEDMatrixWidgets registry not found'); + return; + } -// Register your widget -window.LEDMatrixWidgets.register('my-custom-widget', { - name: 'My Custom Widget', - version: '1.0.0', - - /** - * Render the widget HTML - * @param {HTMLElement} container - Container element to render into - * @param {Object} config - Widget configuration from schema - * @param {*} value - Current value - * @param {Object} options - Additional options (fieldId, pluginId, etc.) - */ - render: function(container, config, value, options) { - const fieldId = options.fieldId || container.id; - // Sanitize fieldId for safe use in DOM IDs and selectors - const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - const safeFieldId = sanitizeId(fieldId); - - const html = ` -
- -
- `; - container.innerHTML = html; - - // Attach event listeners - const input = container.querySelector(`#${safeFieldId}_input`); - if (input) { - input.addEventListener('change', (e) => { - this.handlers.onChange(fieldId, e.target.value); - }); - } - }, - - /** - * Get current value from widget - * @param {string} fieldId - Field ID - * @returns {*} Current value - */ - getValue: function(fieldId) { - // Sanitize fieldId for safe selector use - const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - const safeFieldId = sanitizeId(fieldId); - const input = document.querySelector(`#${safeFieldId}_input`); - return input ? input.value : null; - }, - - /** - * Set value programmatically - * @param {string} fieldId - Field ID - * @param {*} value - Value to set - */ - setValue: function(fieldId, value) { - // Sanitize fieldId for safe selector use - const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - const safeFieldId = sanitizeId(fieldId); - const input = document.querySelector(`#${safeFieldId}_input`); - if (input) { - input.value = value || ''; - } - }, - - /** - * Event handlers - */ - handlers: { - onChange: function(fieldId, value) { - // Trigger form change event - const event = new CustomEvent('widget-change', { - detail: { fieldId, value }, - bubbles: true - }); - document.dispatchEvent(event); - } - }, - - /** - * Helper: Escape HTML to prevent XSS - */ - escapeHtml: function(text) { + const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); + const escapeHtml = (text) => { const div = document.createElement('div'); div.textContent = text; return div.innerHTML; - }, - - /** - * Helper: Sanitize identifier for use in DOM IDs and CSS selectors - */ - sanitizeId: function(id) { - return String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - } -}); + }; + + window.LEDMatrixWidgets.register('my-custom-widget', { + name: 'My Custom Widget', + version: '1.0.0', + + render: function (container, config, value, options) { + const fieldId = options.fieldId || container.id; + const safeId = sanitizeId(fieldId); + container.innerHTML = ` + `; + const input = container.querySelector(`#${safeId}_input`); + input.addEventListener('change', (e) => { + this.handlers.onChange(fieldId, e.target.value); + }); + }, + + getValue: function (fieldId) { + const input = document.querySelector(`#${sanitizeId(fieldId)}_input`); + return input ? input.value : null; + }, + + setValue: function (fieldId, value) { + const input = document.querySelector(`#${sanitizeId(fieldId)}_input`); + if (input) input.value = value || ''; + }, + + handlers: { + onChange: function (fieldId, value) { + document.dispatchEvent(new CustomEvent('widget-change', { + detail: { fieldId, value }, bubbles: true + })); + } + } + }); +})(); ``` -### Step 2: Reference Widget in Schema +[`example-color-picker.js`](example-color-picker.js) is a longer example. -In your plugin's `config_schema.json`: +### 2. Reference it in the schema ```json { "properties": { - "my_field": { - "type": "string", - "description": "My custom field", - "x-widget": "my-custom-widget", - "default": "" - } + "my_field": {"type": "string", "x-widget": "my-custom-widget", "default": ""} } } ``` -### Step 3: Declare the Widget in `manifest.json` +### 3. Declare it in `manifest.json` -The manifest is the allowlist -- a widget is served only if the plugin declares -it, so shipping a file under `widgets/` does not by itself publish it: +The manifest is the allowlist: a widget is served only if the plugin +declares it. ```json { "widgets": [ - { "name": "my-custom-widget", "script": "my-custom-widget.js" } + {"name": "my-custom-widget", "script": "my-custom-widget.js", + "description": "What this widget is for"} ] } ``` -`script` is optional and defaults to `[name].js`. It must be a plain filename -directly inside the plugin's `widgets/` directory. +`name` is what `x-widget` uses. `script` is optional, defaults to +`.js`, and must be a plain filename directly inside `widgets/`. Both +are validated against `schema/manifest_schema.json`. -### Step 4: Widget Loading +### 4. How it loads -The widget is loaded on demand when the config form renders a field that -references it. The system will: +When the config form reaches a field whose `x-widget` is not a built-in: -1. Check if the widget is registered in the core registry -2. If not, fetch `/static/plugin-widgets/[plugin-id]/[widget-name].js`, which - serves the declared script from the plugin's `widgets/` directory -3. Render the widget using the registered `render` function +1. If the name is already registered, that widget renders the field. +2. Otherwise the page fetches `/static/plugin-widgets//.js` + (`serve_plugin_widget` in + [`web_interface/blueprints/pages_v3.py`](../../../../blueprints/pages_v3.py)), + which serves the declared script from the plugin's `widgets/` directory. +3. The widget's `render()` draws the field. -The fetch is a dynamic `import()`, so the file must parse as an ES module (a -plain IIFE does). If anything fails, the field falls back to a plain text input +The fetch is a dynamic `import()`, so the file must parse as an ES module. +An IIFE does; modules are strict mode, and a `return` outside a function is a +syntax error. + +If the widget fails to load (not declared, file missing, script throws, or +it never calls `register`), the field falls back to a plain text input holding the current value, so a broken widget never costs the user their -configured value. +setting. -Only `string`-typed fields take this path today; see `docs/widget-guide.md` -for the full details and limitations. +**Limitation:** only `string` fields without an `enum` take this path. +[`plugin_config.html`](../../../../templates/v3/partials/plugin_config.html) +renders `object`, `array`, `boolean`, `integer`, `number` and `enum` fields +with its own branches, which only know the built-in names, so a plugin's own +widget on one of those is ignored. -## Widget API Reference - -### Widget Definition Object +## Widget API ```javascript { - name: string, // Human-readable widget name - version: string, // Widget version - render: function, // Required: Render function - getValue: function, // Optional: Get current value - setValue: function, // Optional: Set value programmatically - handlers: object // Optional: Event handlers + name: string, // human-readable name + version: string, + render: function, // required: render(container, config, value, options) + getValue: function, // optional: getValue(fieldId) -> value + setValue: function, // optional: setValue(fieldId, value) + handlers: object // optional: e.g. onChange(fieldId, value) } ``` -### Render Function +`render()` arguments: -```javascript -render(container, config, value, options) -``` +- `container` — element to render into +- `config` — the field's schema, including `x-widget-config` / `x-options` +- `value` — current value +- `options` — `fieldId`, `pluginId`, `fullKey` (dotted path of the field) -**Parameters:** -- `container` (HTMLElement): Container element to render into -- `config` (Object): Widget configuration from schema (`x-widget-config` or schema properties) -- `value` (*): Current field value -- `options` (Object): Additional options - - `fieldId` (string): Field ID - - `pluginId` (string): Plugin ID - - `fullKey` (string): Full field key path +## Guidelines -### Get Value Function - -```javascript -getValue(fieldId) -``` - -**Returns:** Current widget value - -### Set Value Function - -```javascript -setValue(fieldId, value) -``` - -**Parameters:** -- `fieldId` (string): Field ID -- `value` (*): Value to set - -### Event Handlers - -Widgets can define custom event handlers in the `handlers` object: - -```javascript -handlers: { - onChange: function(fieldId, value) { - // Handle value change - }, - onFocus: function(fieldId) { - // Handle focus - } -} -``` - -## Best Practices - -### Security - -1. **Always escape HTML**: Use `escapeHtml()` or `textContent` to prevent XSS -2. **Validate inputs**: Validate user input before processing -3. **Sanitize values**: Clean values before storing -4. **Sanitize identifiers**: Always sanitize identifiers (like `fieldId`) used as element IDs and in CSS selectors to prevent selector injection/XSS: - - Use `sanitizeId()` helper function (available in BaseWidget) or create your own - - Allow only safe characters: `[A-Za-z0-9_-]` - - Replace or remove invalid characters before using in: - - `getElementById()`, `querySelector()`, `querySelectorAll()` - - Setting `id` attributes - - Building CSS selectors - - Never interpolate raw `fieldId` into HTML strings or selectors without sanitization - - Example: `const safeId = fieldId.replace(/[^a-zA-Z0-9_-]/g, '_');` - -### Performance - -1. **Lazy loading**: Load widget scripts only when needed -2. **Event delegation**: Use event delegation for dynamic content -3. **Debounce**: Debounce frequent events (e.g., input changes) - -### Accessibility - -1. **Labels**: Always associate labels with inputs -2. **ARIA attributes**: Use appropriate ARIA attributes -3. **Keyboard navigation**: Ensure keyboard accessibility - -### Error Handling - -1. **Graceful degradation**: Handle missing dependencies -2. **User feedback**: Show clear error messages -3. **Logging**: Log errors for debugging - -## Examples - -### Example 1: Color Picker Widget - -```javascript -window.LEDMatrixWidgets.register('color-picker', { - name: 'Color Picker', - version: '1.0.0', - - render: function(container, config, value, options) { - const fieldId = options.fieldId; - // Sanitize fieldId for safe use in DOM IDs and selectors - const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - const sanitizedFieldId = sanitizeId(fieldId); - - container.innerHTML = ` -
- - -
- `; - - const colorInput = container.querySelector(`#${sanitizedFieldId}_color`); - const hexInput = container.querySelector(`#${sanitizedFieldId}_hex`); - - if (colorInput && hexInput) { - colorInput.addEventListener('change', (e) => { - hexInput.value = e.target.value; - this.handlers.onChange(fieldId, e.target.value); - }); - - hexInput.addEventListener('change', (e) => { - if (/^#[0-9A-Fa-f]{6}$/.test(e.target.value)) { - colorInput.value = e.target.value; - this.handlers.onChange(fieldId, e.target.value); - } - }); - } - }, - - getValue: function(fieldId) { - // Sanitize fieldId for safe selector use - const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - const sanitizedFieldId = sanitizeId(fieldId); - const colorInput = document.querySelector(`#${sanitizedFieldId}_color`); - return colorInput ? colorInput.value : null; - }, - - setValue: function(fieldId, value) { - // Sanitize fieldId for safe selector use - const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - const sanitizedFieldId = sanitizeId(fieldId); - const colorInput = document.querySelector(`#${sanitizedFieldId}_color`); - const hexInput = document.querySelector(`#${sanitizedFieldId}_hex`); - if (colorInput && hexInput) { - colorInput.value = value; - hexInput.value = value; - } - }, - - handlers: { - onChange: function(fieldId, value) { - const event = new CustomEvent('widget-change', { - detail: { fieldId, value }, - bubbles: true - }); - document.dispatchEvent(event); - } - } -}); -``` - -### Example 2: Slider Widget - -```javascript -window.LEDMatrixWidgets.register('slider', { - name: 'Slider Widget', - version: '1.0.0', - - render: function(container, config, value, options) { - const fieldId = options.fieldId; - // Sanitize fieldId for safe use in DOM IDs and selectors - const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - const sanitizedFieldId = sanitizeId(fieldId); - - const min = config.minimum || 0; - const max = config.maximum || 100; - const step = config.step || 1; - const currentValue = value !== undefined ? value : (config.default || min); - - container.innerHTML = ` -
- -
- ${min} - ${currentValue} - ${max} -
-
- `; - - const slider = container.querySelector(`#${sanitizedFieldId}_slider`); - const valueDisplay = container.querySelector(`#${sanitizedFieldId}_value`); - - if (slider && valueDisplay) { - slider.addEventListener('input', (e) => { - valueDisplay.textContent = e.target.value; - this.handlers.onChange(fieldId, parseFloat(e.target.value)); - }); - } - }, - - getValue: function(fieldId) { - // Sanitize fieldId for safe selector use - const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - const sanitizedFieldId = sanitizeId(fieldId); - const slider = document.querySelector(`#${sanitizedFieldId}_slider`); - return slider ? parseFloat(slider.value) : null; - }, - - setValue: function(fieldId, value) { - // Sanitize fieldId for safe selector use - const sanitizeId = (id) => String(id).replace(/[^a-zA-Z0-9_-]/g, '_'); - const sanitizedFieldId = sanitizeId(fieldId); - const slider = document.querySelector(`#${sanitizedFieldId}_slider`); - const valueDisplay = document.querySelector(`#${sanitizedFieldId}_value`); - if (slider) { - slider.value = value; - if (valueDisplay) { - valueDisplay.textContent = value; - } - } - }, - - handlers: { - onChange: function(fieldId, value) { - const event = new CustomEvent('widget-change', { - detail: { fieldId, value }, - bubbles: true - }); - document.dispatchEvent(event); - } - } -}); -``` +- Escape values before putting them in HTML (`textContent` or an + `escapeHtml` helper). +- Sanitise `fieldId` before using it in an `id`, `getElementById()` or a CSS + selector: allow only `[A-Za-z0-9_-]`. `BaseWidget` has `sanitizeId()`. +- Associate labels with inputs and keep the widget usable from the keyboard. +- Debounce events that fire on every keystroke. ## Troubleshooting -### Widget Not Loading +**Widget not loading** +- Check the browser console. +- The widget must be declared in `manifest.json` and live in `widgets/`. +- The name passed to `register()` must match `x-widget`. +- The field must be a non-enum `string` (see the limitation above). -1. Check browser console for errors -2. Verify widget file path is correct -3. Ensure `LEDMatrixWidgets.register()` is called -4. Check that widget name matches schema `x-widget` value - -### Widget Not Rendering - -1. Verify `render` function is defined -2. Check container element exists -3. Ensure widget is registered before form loads -4. Check for JavaScript errors in console - -### Value Not Saving - -1. Ensure widget triggers `widget-change` event -2. Verify form submission includes widget value -3. Check `getValue` function returns correct type -4. Verify field name matches schema property - -## Migration from Server-Side Rendering - -Currently, widgets are server-side rendered via Jinja2 templates. The registry system provides: - -1. **Backwards Compatibility**: Existing server-side rendered widgets continue to work -2. **Future Enhancement**: Client-side rendering support for custom widgets -3. **Handler Availability**: All widget handlers are available globally - -Future versions may support full client-side rendering, but server-side rendering remains the primary method for core widgets. - -## Support - -For questions or issues: -- Check existing widget implementations for examples -- Review browser console for errors -- Test with simple widget first before complex implementations +**Value not saving** +- Fire a `widget-change` event on change. +- `getValue()` must return the type the schema expects. +- Check the field name matches the schema property. diff --git a/web_interface/system_metrics.py b/web_interface/system_metrics.py index 723bd85f..2d7ef585 100644 --- a/web_interface/system_metrics.py +++ b/web_interface/system_metrics.py @@ -1,4 +1,4 @@ -"""System metrics for the status stream. +"""System metrics for the status stream and GET /api/v3/system/status. Deliberately free of Flask and app imports: importing web_interface.app constructs the Flask application and a CacheManager, and the latter claims the @@ -11,9 +11,20 @@ the UI can render '--', because a confident wrong number is worse than a blank: on a card that was filling up. """ +import time from typing import Any, Dict, Optional _THERMAL_ZONE = '/sys/class/thermal/thermal_zone0/temp' +_MB = 1024 * 1024 +_GB = 1024 * 1024 * 1024 + +#: Every key collect_system_metrics() returns. +METRIC_KEYS = ( + 'cpu_percent', 'cpu_temp', + 'memory_used_percent', 'memory_total_mb', 'memory_used_mb', 'memory_available_mb', + 'disk_used_percent', 'disk_total_gb', 'disk_used_gb', + 'uptime_seconds', +) def _cpu_temp_c() -> Optional[float]: @@ -25,38 +36,53 @@ def _cpu_temp_c() -> Optional[float]: return None -def collect_system_metrics() -> Dict[str, Any]: +def collect_system_metrics(cpu_interval: Optional[float] = None) -> Dict[str, Any]: """Collect the numbers that predict trouble on a small board. + ``cpu_interval`` is passed to ``psutil.cpu_percent``. None does not block + and measures since the previous call (app startup primes it); a request + that may be the first call in its process passes a short interval instead. + ``memory_available_mb`` is MemAvailable rather than a used percentage: it accounts for reclaimable page cache, so it is what separates a board at 70% "used" that is fine from one at 70% that is about to fail fork(). A 1GB Pi can sit at either and only this number tells them apart. """ + metrics: Dict[str, Any] = dict.fromkeys(METRIC_KEYS) + metrics['cpu_temp'] = _cpu_temp_c() try: import psutil except ImportError: - return { - 'cpu_percent': 0, - 'memory_used_percent': 0, - 'memory_available_mb': None, - 'disk_used_percent': None, - 'cpu_temp': _cpu_temp_c() or 0, - } + return metrics + + metrics['cpu_percent'] = round(psutil.cpu_percent(interval=cpu_interval), 1) - # interval=None is non-blocking; app startup primes psutil's internal state. - cpu_percent = round(psutil.cpu_percent(interval=None), 1) memory = psutil.virtual_memory() + metrics['memory_used_percent'] = round(memory.percent, 1) + metrics['memory_total_mb'] = round(memory.total / _MB, 1) + metrics['memory_used_mb'] = round(memory.used / _MB, 1) + metrics['memory_available_mb'] = round(memory.available / _MB, 1) try: - disk_used_percent: Optional[float] = round(psutil.disk_usage('/').percent, 1) + disk = psutil.disk_usage('/') except OSError: - disk_used_percent = None + pass + else: + metrics['disk_used_percent'] = round(disk.percent, 1) + metrics['disk_total_gb'] = round(disk.total / _GB, 1) + metrics['disk_used_gb'] = round(disk.used / _GB, 1) - return { - 'cpu_percent': cpu_percent, - 'memory_used_percent': round(memory.percent, 1), - 'memory_available_mb': round(memory.available / (1024 * 1024), 1), - 'disk_used_percent': disk_used_percent, - 'cpu_temp': _cpu_temp_c() or 0, - } + metrics['uptime_seconds'] = int(time.time() - psutil.boot_time()) + return metrics + + +def format_uptime(uptime_seconds: Optional[float]) -> Optional[str]: + """'3d 4h', '5h 12m' or '42m'; None when the uptime is unknown.""" + if uptime_seconds is None: + return None + hours = uptime_seconds / 3600 + if hours >= 24: + return f"{int(hours / 24)}d {int(hours % 24)}h" + if hours >= 1: + return f"{int(hours)}h {int((uptime_seconds % 3600) / 60)}m" + return f"{int(uptime_seconds / 60)}m"