From 2df273ecfc03815ce4f663bf0e9c08a854afe23a Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:42:00 -0400 Subject: [PATCH 01/16] fix(web): default web_display_autostart to true, as the installer already does (#556) start_web_conditionally.py read the flag with `config_data.get("web_display_autostart", False)`, so a config that simply lacked the key got no web interface. Both config/config.template.json and first_time_install.sh ship the key as true, so the code default contradicted the shipped default in two places: absence means an older or hand-edited config, not a request to stay down. The failure mode was silent in the worst way. The "not starting" path exits 0, so `systemctl status ledmatrix-web` reported the unit as successfully started while nothing was listening on the port, and the only trace was one journal line saying the flag was "false or not set" -- which reads as a deliberate setting rather than a missing key. Also start the web interface when config.json is missing or unparseable, instead of exiting. The web interface is how a config gets created and repaired, so a broken config is exactly when the user needs it most; leaving it down means there is no way back in. Only an explicit false/off disables autostart now, and the disabled message says "explicitly disabled" so the journal distinguishes a real setting from a default. Co-authored-by: Claude Opus 5 --- scripts/utils/start_web_conditionally.py | 42 +++++++++++----- test/test_web_autostart_default.py | 63 ++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 12 deletions(-) create mode 100644 test/test_web_autostart_default.py diff --git a/scripts/utils/start_web_conditionally.py b/scripts/utils/start_web_conditionally.py index b3c3c99d..b1492ac0 100644 --- a/scripts/utils/start_web_conditionally.py +++ b/scripts/utils/start_web_conditionally.py @@ -74,23 +74,41 @@ def install_dependencies(): print(f"Failed to install dependencies: {e}") return False +#: String spellings that turn autostart OFF. Anything else -- including the key +#: being absent entirely -- leaves it on. +DISABLED_STRINGS = ("off", "false", "no", "0") + + +def autostart_enabled(config_data): + """Whether to bring the web interface up. Defaults to True. + + config.template.json and first_time_install.sh both ship + ``web_display_autostart`` as true, so a config that lacks the key is an + older or hand-edited one rather than a request to stay down. Defaulting to + False meant any such config silently got no web interface -- and because + the "not starting" path exits 0, systemd reported the unit as successfully + started while nothing was listening. Only an explicit false/off disables it. + """ + value = config_data.get("web_display_autostart", True) + if isinstance(value, str): + return value.strip().lower() not in DISABLED_STRINGS + return bool(value) + + def main(): try: with open(CONFIG_FILE, 'r') as f: config_data = json.load(f) except FileNotFoundError: - print(f"Config file {CONFIG_FILE} not found. Web interface will not start.") - sys.exit(0) # Exit gracefully, don't start - except Exception as e: - print(f"Error reading config file {CONFIG_FILE}: {e}. Web interface will not start.") - sys.exit(1) # Exit with error, service might restart depending on config + # The web interface is how a config gets created and repaired, so a + # missing one is the case where the user needs it most. + print(f"Config file {CONFIG_FILE} not found. Starting the web interface so it can be configured.") + config_data = {} + except (json.JSONDecodeError, OSError) as e: + print(f"Error reading config file {CONFIG_FILE}: {e}. Starting the web interface anyway so the config can be repaired.") + config_data = {} - autostart_enabled = config_data.get("web_display_autostart", False) - - # Handle both boolean True and string "on"/"true" values - is_enabled = (autostart_enabled is True) or (isinstance(autostart_enabled, str) and autostart_enabled.lower() in ("on", "true", "yes", "1")) - - if is_enabled: + if autostart_enabled(config_data): print("Configuration 'web_display_autostart' is enabled. Starting web interface...") # Only install dependencies if not already done during first-time setup @@ -116,7 +134,7 @@ def main(): print(f"Failed to exec web interface: {e}") sys.exit(1) # Failed to start else: - print("Configuration 'web_display_autostart' is false or not set. Web interface will not be started.") + print("Configuration 'web_display_autostart' is explicitly disabled. Web interface will not be started.") sys.exit(0) # Exit gracefully, service considered successful if __name__ == '__main__': diff --git a/test/test_web_autostart_default.py b/test/test_web_autostart_default.py new file mode 100644 index 00000000..9119b7ed --- /dev/null +++ b/test/test_web_autostart_default.py @@ -0,0 +1,63 @@ +"""Tests for the web interface's autostart default. + +Regression under test: ``start_web_conditionally.py`` read the flag with +``config_data.get("web_display_autostart", False)``, so a config that simply +lacked the key got no web interface. Both config.template.json and +first_time_install.sh ship the key as ``true``, so absence means an older or +hand-edited config -- not a request to stay down. + +The failure was silent in the worst way: the "not starting" path exits 0, so +``systemctl status ledmatrix-web`` reported the unit as successfully started +while nothing was listening on the port, and the journal's only trace was one +line saying the flag was "false or not set". +""" + +import importlib.util +from pathlib import Path + +import pytest + +SCRIPT = (Path(__file__).resolve().parents[1] + / "scripts" / "utils" / "start_web_conditionally.py") + + +@pytest.fixture(scope="module") +def mod(): + spec = importlib.util.spec_from_file_location("start_web_conditionally", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TestAutostartDefault: + def test_absent_key_still_starts(self, mod): + # The regression: this returned False and the UI silently stayed down. + assert mod.autostart_enabled({}) is True + + def test_unreadable_config_still_starts(self, mod): + # main() falls back to {} when the config is missing or corrupt, because + # the web interface is how a broken config gets repaired. + assert mod.autostart_enabled({}) is True + + def test_explicit_true_starts(self, mod): + assert mod.autostart_enabled({"web_display_autostart": True}) is True + + def test_explicit_false_does_not_start(self, mod): + assert mod.autostart_enabled({"web_display_autostart": False}) is False + + @pytest.mark.parametrize("value", ["off", "false", "no", "0", "OFF", " False "]) + def test_disabling_strings_do_not_start(self, mod, value): + assert mod.autostart_enabled({"web_display_autostart": value}) is False + + @pytest.mark.parametrize("value", ["on", "true", "yes", "1", "TRUE"]) + def test_enabling_strings_start(self, mod, value): + assert mod.autostart_enabled({"web_display_autostart": value}) is True + + +class TestShippedDefaultsAgree: + def test_template_ships_autostart_true(self): + """The code default must match what the installer actually writes.""" + import json + template = json.loads( + (SCRIPT.parents[2] / "config" / "config.template.json").read_text(encoding="utf-8")) + assert template["web_display_autostart"] is True From 12f3790994f24d4dce205a07b9f6c7aaf75ef04b Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:41:19 -0400 Subject: [PATCH 02/16] fix(install): render the systemd units from their templates, not from heredocs (#547) * fix(install): render the systemd units from their templates, not from heredocs The installers carried their own inline copies of units that also exist as templates under systemd/, and the copies drifted. install_service.sh renders ledmatrix.service from the template correctly, then wrote ledmatrix-web.service from a heredoc that predated it -- missing Wants=network-online.target, RestartSec=10, SyslogIdentifier, CacheDirectory, CacheDirectoryMode and Environment=USE_THREADING=1. install_web_service.sh had a third copy, and install_wifi_monitor.sh a fourth, that one already differing from its template (syslog where the template says journal). startup_validator.py compares the installed unit against the template, so a rig installed this way warned on every boot -- and the remedy the warning names, "re-run scripts/install/install_service.sh", reinstalled the same stale copy. The warning could never clear. Reproduced on a live rig running exactly that unit. All three installers now render systemd/*.service through the same placeholder substitution. The template gains a __USER__ placeholder rather than hardcoding User=root, because the web interface runs as whoever installed it. That last point was a second, independent cause of a permanent warning: the validator substituted a fixed "root", so any non-root install reported drift forever. It now reads User= from the installed unit -- an install-time decision, not something the template dictates -- and compares everything else strictly. first_time_install.sh already reads the installed User= the same way. Tests cover a non-root web unit not warning, a genuinely changed directive in that unit still warning, the User= fallback, and a grep-based guard that no installer under scripts/install/ contains an inline unit body. That guard is what found the install_wifi_monitor.sh copy. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 * fix(install): escape sed replacements, use mktemp, and make render failures fatal Address CodeRabbit findings on install_service.sh, install_web_service.sh and install_wifi_monitor.sh: - Values interpolated into each script's sed expression (project root path, username) were not escaped, so a value containing &, \ or the | delimiter would corrupt the rendered systemd unit. Add a shared sed_escape_replacement() helper in the new scripts/install/lib_systemd_render.sh (sourced by all three scripts) and apply it to every sed replacement. - install_service.sh rendered the main and web units to the predictable path /tmp/ledmatrix.service.tmp before installing them -- a symlink/TOCTOU race (CWE-377). Use mktemp for both, with a trap to clean up on exit. - install_service.sh treated a missing template as a mere warning and then checked only whether a unit already existed at the destination before enabling/starting it, so a render failure could silently fall back to enabling a stale, previously-installed unit. Both unit blocks now exit non-zero on a missing template or a failed render. Also rename the ambiguous loop variable `l` to `line` in test/test_systemd_unit_drift.py (Ruff E741); ruff isn't wired into any CI workflow in this repo today, so this isn't currently CI-blocking, but the rename is trivial and correct regardless. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01S3bPMESe2TfrGvbs1ef9c5 * test(install): cover sed_escape_replacement against sed-special characters CodeRabbit asked for regression coverage using a project path containing an ampersand; the earlier commits on this branch already fixed the escaping, mktemp usage, and enable/start-on-fatal-render-failure findings, and the l->line rename was already applied -- this closes the one remaining gap. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 (1M context) --- scripts/install/install_service.sh | 98 ++++++++++++------- scripts/install/install_web_service.sh | 46 ++++----- scripts/install/install_wifi_monitor.sh | 34 +++---- scripts/install/lib_systemd_render.sh | 27 ++++++ src/startup_validator.py | 33 +++++-- systemd/ledmatrix-web.service | 16 ++-- test/test_systemd_unit_drift.py | 121 ++++++++++++++++++++++++ 7 files changed, 277 insertions(+), 98 deletions(-) create mode 100644 scripts/install/lib_systemd_render.sh diff --git a/scripts/install/install_service.sh b/scripts/install/install_service.sh index cd4144cc..0a7955d8 100755 --- a/scripts/install/install_service.sh +++ b/scripts/install/install_service.sh @@ -16,20 +16,39 @@ USER_HOME=$(eval echo ~$ACTUAL_USER) # Determine the Project Root Directory (parent of scripts/install/) PROJECT_ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd) +# shellcheck source=scripts/install/lib_systemd_render.sh +source "$PROJECT_ROOT_DIR/scripts/install/lib_systemd_render.sh" + echo "Installing LED Matrix Display Service for user: $ACTUAL_USER" echo "Using home directory: $USER_HOME" echo "Project root directory: $PROJECT_ROOT_DIR" -# Create a temporary service file for the main display with the correct paths -# Assuming ledmatrix.service template exists and uses /home/ledpi as a placeholder for user home +# Render the main display unit from its template. The display service runs as +# root (it needs GPIO), so __USER__ is always root here -- unlike the web unit +# below, which runs as whoever installed it. +# +# A missing template or a failed render is fatal: falling through would leave +# whatever unit already sits at /etc/systemd/system/ledmatrix.service (from a +# previous install) untouched, and the enable/start step below would then +# silently reuse that stale unit instead of the one this run was asked to +# install. if [ -f "$PROJECT_ROOT_DIR/systemd/ledmatrix.service" ]; then - sed "s|/home/ledpi|$USER_HOME|g; s|__PROJECT_ROOT_DIR__|$PROJECT_ROOT_DIR|g; s|__USER__|root|g" "$PROJECT_ROOT_DIR/systemd/ledmatrix.service" > /tmp/ledmatrix.service.tmp + ESCAPED_PROJECT_ROOT_DIR=$(sed_escape_replacement "$PROJECT_ROOT_DIR") + MAIN_UNIT_TMP=$(mktemp) + trap 'rm -f "$MAIN_UNIT_TMP"' EXIT + if ! sed "s|__PROJECT_ROOT_DIR__|$ESCAPED_PROJECT_ROOT_DIR|g; s|__USER__|root|g" \ + "$PROJECT_ROOT_DIR/systemd/ledmatrix.service" > "$MAIN_UNIT_TMP"; then + echo "ERROR: failed to render ledmatrix.service from its template." >&2 + exit 1 + fi # Copy the service file to the systemd directory - sudo cp /tmp/ledmatrix.service.tmp /etc/systemd/system/ledmatrix.service + sudo cp "$MAIN_UNIT_TMP" /etc/systemd/system/ledmatrix.service # Clean up - rm /tmp/ledmatrix.service.tmp + rm -f "$MAIN_UNIT_TMP" + trap - EXIT else - echo "WARNING: ledmatrix.service template not found at $PROJECT_ROOT_DIR/systemd/ledmatrix.service. Main display service not configured." + echo "ERROR: ledmatrix.service template not found at $PROJECT_ROOT_DIR/systemd/ledmatrix.service." >&2 + exit 1 fi @@ -48,42 +67,49 @@ fi # === LEDMatrix Web Interface service (ledmatrix-web.service) === echo "Installing LEDMatrix Web Interface service (ledmatrix-web.service)..." -WEB_SERVICE_FILE_CONTENT=$(cat < /dev/null +# Rendered from systemd/ledmatrix-web.service, the same template +# install_web_service.sh uses. This was an inline heredoc until it drifted from +# the template: it had lost Wants=network-online.target, RestartSec, +# SyslogIdentifier, CacheDirectory and Environment=USE_THREADING. Because +# src/startup_validator.py compares the installed unit against the template, +# every boot warned "re-run install_service.sh" -- and doing so reinstalled the +# same stale copy, so the warning could never clear. +# +# As with the main unit above, a missing template or a failed render is +# fatal -- otherwise the enable/start check below would fall back to +# whatever unit (possibly stale) already exists at the destination path. +if [ -f "$PROJECT_ROOT_DIR/systemd/ledmatrix-web.service" ]; then + ESCAPED_ACTUAL_USER=$(sed_escape_replacement "$ACTUAL_USER") + WEB_UNIT_TMP=$(mktemp) + trap 'rm -f "$WEB_UNIT_TMP"' EXIT + if ! sed "s|__PROJECT_ROOT_DIR__|$ESCAPED_PROJECT_ROOT_DIR|g; s|__USER__|$ESCAPED_ACTUAL_USER|g" \ + "$PROJECT_ROOT_DIR/systemd/ledmatrix-web.service" > "$WEB_UNIT_TMP"; then + echo "ERROR: failed to render ledmatrix-web.service from its template." >&2 + exit 1 + fi + sudo cp "$WEB_UNIT_TMP" /etc/systemd/system/ledmatrix-web.service + rm -f "$WEB_UNIT_TMP" + trap - EXIT +else + echo "ERROR: ledmatrix-web.service template not found at $PROJECT_ROOT_DIR/systemd/ledmatrix-web.service." >&2 + exit 1 +fi echo "Reloading systemd daemon for web service..." sudo systemctl daemon-reload -echo "Enabling ledmatrix-web.service to start on boot..." -sudo systemctl enable ledmatrix-web.service +if [ -f "/etc/systemd/system/ledmatrix-web.service" ]; then + echo "Enabling ledmatrix-web.service to start on boot..." + sudo systemctl enable ledmatrix-web.service -echo "Starting ledmatrix-web.service..." -sudo systemctl start ledmatrix-web.service + echo "Starting ledmatrix-web.service..." + sudo systemctl start ledmatrix-web.service -echo "LEDMatrix Web Interface service (ledmatrix-web.service) installation complete." -echo "It will start based on the 'web_display_autostart' setting in config/config.json." + echo "LEDMatrix Web Interface service (ledmatrix-web.service) installation complete." + echo "It will start based on the 'web_display_autostart' setting in config/config.json." +else + echo "Skipping enable/start for ledmatrix-web.service as it was not configured." +fi # === End of LEDMatrix Web Interface service === diff --git a/scripts/install/install_web_service.sh b/scripts/install/install_web_service.sh index 6df16d71..1e7a9476 100755 --- a/scripts/install/install_web_service.sh +++ b/scripts/install/install_web_service.sh @@ -17,6 +17,9 @@ fi # Determine the Project Root Directory (parent of scripts/install/) PROJECT_ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +# shellcheck source=scripts/install/lib_systemd_render.sh +source "$PROJECT_ROOT_DIR/scripts/install/lib_systemd_render.sh" + echo "Installing for user: $ACTUAL_USER" echo "Project root directory: $PROJECT_ROOT_DIR" @@ -26,37 +29,22 @@ if [ "$EUID" -ne 0 ]; then exit 1 fi -# Generate the service file dynamically with the correct paths -echo "Generating service file with dynamic paths..." -WEB_SERVICE_FILE_CONTENT=$(cat < /etc/systemd/system/ledmatrix-web.service +ESCAPED_PROJECT_ROOT_DIR=$(sed_escape_replacement "$PROJECT_ROOT_DIR") +ESCAPED_ACTUAL_USER=$(sed_escape_replacement "$ACTUAL_USER") +sed "s|__PROJECT_ROOT_DIR__|$ESCAPED_PROJECT_ROOT_DIR|g; s|__USER__|$ESCAPED_ACTUAL_USER|g" \ + "$TEMPLATE" > /etc/systemd/system/ledmatrix-web.service # Ensure cache directory exists with proper permissions # This is a fallback for older systemd versions that don't support CacheDirectory diff --git a/scripts/install/install_wifi_monitor.sh b/scripts/install/install_wifi_monitor.sh index 5d224c03..0c24b0b8 100755 --- a/scripts/install/install_wifi_monitor.sh +++ b/scripts/install/install_wifi_monitor.sh @@ -18,6 +18,9 @@ USER_HOME=$(eval echo ~$ACTUAL_USER) # Determine the Project Root Directory (parent of scripts/install/) PROJECT_ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd) +# shellcheck source=scripts/install/lib_systemd_render.sh +source "$PROJECT_ROOT_DIR/scripts/install/lib_systemd_render.sh" + echo "Installing LED Matrix WiFi Monitor Service for user: $ACTUAL_USER" echo "Using home directory: $USER_HOME" echo "Project root directory: $PROJECT_ROOT_DIR" @@ -64,30 +67,19 @@ if [ ${#MISSING_PACKAGES[@]} -gt 0 ]; then echo "✓ Package installation completed" fi -# Create service file with correct paths +# Render the unit from systemd/ledmatrix-wifi-monitor.service rather than +# inlining a second copy here. The copy this replaced had already drifted -- +# it wrote StandardOutput/StandardError=syslog where the template says journal. echo "" echo "Creating systemd service file..." -SERVICE_FILE_CONTENT=$(cat < /dev/null diff --git a/scripts/install/lib_systemd_render.sh b/scripts/install/lib_systemd_render.sh new file mode 100644 index 00000000..d07dcec1 --- /dev/null +++ b/scripts/install/lib_systemd_render.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Shared helper for rendering systemd unit templates via sed. +# +# Sourced by install_service.sh, install_web_service.sh and +# install_wifi_monitor.sh so all three escape sed replacement text the same +# way instead of carrying three copies of the same fix. + +# sed_escape_replacement VALUE +# +# Print VALUE escaped for safe use as the replacement side of `sed +# s|pattern|replacement|`. Every one of these scripts builds its sed +# expression by interpolating a shell variable (a path, a username, ...) +# straight into the replacement text. sed gives three characters special +# meaning there: backslash (escape character), & (whole match) and the +# delimiter itself (here `|`). A value containing any of them -- e.g. a +# username or path with an `&`, a literal backslash, or a `|` -- would +# otherwise corrupt the rendered unit file instead of being substituted +# literally. Escape the backslash first so the later escapes aren't +# double-escaped. +sed_escape_replacement() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//&/\\&}" + value="${value//|/\\|}" + printf '%s' "$value" +} diff --git a/src/startup_validator.py b/src/startup_validator.py index 79275dc2..36065bf0 100644 --- a/src/startup_validator.py +++ b/src/startup_validator.py @@ -111,17 +111,25 @@ class StartupValidator: if not template.is_file() or not installed.is_file(): continue - # The template carries placeholders the installer substitutes, - # so compare the substituted form rather than the raw file. - expected = template.read_text(encoding="utf-8") - expected = expected.replace("__PROJECT_ROOT_DIR__", str(project_root)) - expected = expected.replace("__USER__", "root") - try: actual = installed.read_text(encoding="utf-8") except PermissionError: continue + # The template carries placeholders the installer substitutes, + # so compare the substituted form rather than the raw file. + expected = template.read_text(encoding="utf-8") + expected = expected.replace("__PROJECT_ROOT_DIR__", str(project_root)) + # User= is an install-time decision, not something the template + # dictates: the installers write whoever ran them, which on a + # non-root install is never "root". Substituting a fixed "root" + # here reported drift on every such install, permanently -- and + # re-running the installer, which is what the warning tells you + # to do, could not clear it. Taking the installed unit's own + # value keeps the comparison on the directives the template + # actually controls. + expected = expected.replace("__USER__", self._installed_user(actual)) + if self._unit_body(expected) != self._unit_body(actual): self.warnings.append( f"{installed.name} differs from {template_rel}; the " @@ -132,6 +140,19 @@ class StartupValidator: except OSError as e: self.logger.debug("Could not compare systemd units: %s", e) + @staticmethod + def _installed_user(unit_text: str) -> str: + """The installed unit's ``User=``, or "root" when it does not set one. + + systemd itself defaults to root for a system unit with no User=, so that + is the right fallback rather than an empty string. + """ + for line in unit_text.splitlines(): + stripped = line.strip() + if stripped.startswith("User="): + return stripped.split("=", 1)[1].strip() + return "root" + @staticmethod def _unit_body(text: str) -> str: """A unit's meaningful lines, in order: no comments, no blanks. diff --git a/systemd/ledmatrix-web.service b/systemd/ledmatrix-web.service index 4dcb9fd7..3b74b569 100644 --- a/systemd/ledmatrix-web.service +++ b/systemd/ledmatrix-web.service @@ -1,9 +1,13 @@ -# This is a template file. The actual service file is generated dynamically -# by install_web_service.sh with the correct paths based on where the project -# is installed. Do not use this file directly - use install_web_service.sh instead. +# This is a template. Do not install it directly -- run +# scripts/install/install_web_service.sh (or install_service.sh, which +# renders this same file). Both substitute: +# __PROJECT_ROOT_DIR__ -> the directory the project is installed in +# __USER__ -> the user the web interface runs as # -# Template - paths will be replaced dynamically: -# __PROJECT_ROOT_DIR__ will be replaced with the actual project directory +# This file is the only description of the unit. It used to be duplicated as +# an inline heredoc in both installers, which drifted apart; src/startup_validator.py +# compares the installed unit against this template, so a second copy means a +# permanent, unclearable drift warning on every boot. [Unit] Description=LED Matrix Web Interface Service @@ -12,7 +16,7 @@ Wants=network-online.target [Service] Type=simple -User=root +User=__USER__ WorkingDirectory=__PROJECT_ROOT_DIR__ Environment=USE_THREADING=1 ExecStart=/usr/bin/python3 __PROJECT_ROOT_DIR__/scripts/utils/start_web_conditionally.py diff --git a/test/test_systemd_unit_drift.py b/test/test_systemd_unit_drift.py index 21cf56e0..8afaadb0 100644 --- a/test/test_systemd_unit_drift.py +++ b/test/test_systemd_unit_drift.py @@ -17,6 +17,8 @@ editing files under /etc and restarting services is the installer's job, not something a display process should do to a machine while it boots. """ import logging +import shlex +import subprocess from pathlib import Path from unittest.mock import MagicMock @@ -150,3 +152,122 @@ def test_a_missing_installed_unit_is_silent(validator, tmp_path): validator._validate_systemd_units() assert not validator.warnings assert not validator.errors + + +# --- the web unit: one template, three copies, and a warning that never cleared --- +# +# install_service.sh and install_web_service.sh each carried their own inline +# heredoc of ledmatrix-web.service. install_service.sh's had drifted -- no +# Wants=network-online.target, RestartSec, SyslogIdentifier or CacheDirectory -- +# and that is what was installed on real rigs. The validator correctly reported +# the drift and told the user to re-run install_service.sh, which reinstalled the +# same stale copy, so the warning could never clear. Separately, the template +# hardcoded User=root while the installers write whoever ran them, so even the +# *correct* installer produced a permanent warning on any non-root install. + + +def _render(template_text, project_root, user): + """Exactly what the installers' sed does.""" + return (template_text + .replace("__PROJECT_ROOT_DIR__", str(project_root)) + .replace("__USER__", user)) + + +def test_the_web_unit_installed_as_a_non_root_user_is_not_drift(validator, tmp_path): + """The web interface runs as whoever installed it, not as root. + + This is the case that warned forever: nothing the user could do would make + an installed `User=pi` match a template that said `User=root`. + """ + project_root = Path("src/startup_validator.py").resolve().parent.parent + template_rel = "systemd/ledmatrix-web.service" + template = project_root / template_rel + if not template.is_file(): + pytest.skip("repo unit template not present") + + installed = tmp_path / "ledmatrix-web.service" + installed.write_text( + _render(template.read_text(encoding="utf-8"), project_root, "hdpi"), + encoding="utf-8") + + validator._UNITS = ((template_rel, str(installed)),) + validator._validate_systemd_units() + assert not validator.warnings, ( + f"a correctly installed non-root web unit warned: {validator.warnings}") + + +def test_the_web_unit_still_reports_a_real_changed_directive(validator, tmp_path): + """Ignoring User= must not make the check blind to everything else.""" + project_root = Path("src/startup_validator.py").resolve().parent.parent + template_rel = "systemd/ledmatrix-web.service" + template = project_root / template_rel + if not template.is_file(): + pytest.skip("repo unit template not present") + + rendered = _render(template.read_text(encoding="utf-8"), project_root, "hdpi") + # Drop RestartSec -- one of the directives the stale heredoc was missing. + stale = "\n".join(line for line in rendered.splitlines() if not line.startswith("RestartSec=")) + installed = tmp_path / "ledmatrix-web.service" + installed.write_text(stale + "\n", encoding="utf-8") + + validator._UNITS = ((template_rel, str(installed)),) + validator._validate_systemd_units() + assert validator.warnings, "a web unit missing RestartSec= produced no warning" + assert not validator.errors + + +def test_installed_user_falls_back_to_root(): + """systemd defaults a system unit with no User= to root, so we must too.""" + assert StartupValidator._installed_user("[Service]\nExecStart=/x\n") == "root" + assert StartupValidator._installed_user("[Service]\nUser=pi\n") == "pi" + assert StartupValidator._installed_user("[Service]\n User=hdpi \n") == "hdpi" + + +def test_sed_escape_replacement_preserves_special_characters(): + """A project path or username containing sed-special characters must render literally. + + The installers build their sed expression by interpolating a shell + variable into the replacement side of `sed s|pattern|replacement|`. + Unescaped, sed treats `&` as "insert the whole match" and `\\` as an + escape character, so a path like `/opt/led&matrix` would corrupt the + rendered unit instead of being substituted as-is. lib_systemd_render.sh's + sed_escape_replacement exists to prevent exactly that. + """ + project_root = Path("src/startup_validator.py").resolve().parent.parent + helper = project_root / "scripts" / "install" / "lib_systemd_render.sh" + if not helper.is_file(): + pytest.skip("install helper not present") + + value = "/opt/led&matrix\\pi|two" + escape_cmd = f'source {shlex.quote(str(helper))}; sed_escape_replacement {shlex.quote(value)}' + escaped = subprocess.run( + ["bash", "-c", escape_cmd], capture_output=True, text=True, check=True + ).stdout + + rendered = subprocess.run( + ["sed", f"s|__X__|{escaped}|g"], + input="path=__X__\n", capture_output=True, text=True, check=True, + ).stdout + + assert rendered == f"path={value}\n", ( + "a sed-special character in the replacement was not preserved literally") + + +def test_no_installer_carries_its_own_copy_of_a_unit(): + """The regression guard. + + Both installers used to inline the unit as a heredoc, and the two copies + drifted from the template and from each other. A unit body in a shell script + is the bug, so assert there isn't one rather than asserting the current + contents match -- matching contents is exactly what silently stops being + true. + """ + project_root = Path("src/startup_validator.py").resolve().parent.parent + offenders = [] + for script in sorted((project_root / "scripts" / "install").glob("*.sh")): + text = script.read_text(encoding="utf-8", errors="replace") + if "[Unit]" in text and "Description=" in text: + offenders.append(script.name) + assert not offenders, ( + f"{offenders} contain an inline systemd unit; render " + f"systemd/*.service instead so there is one source of truth") From 28bc79566ff2f2f09efd9739529cd7b9dfb1dca4 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:41:32 -0400 Subject: [PATCH 03/16] fix(logo): remember a missing logo instead of re-warning every rotation (#548) * fix(logo): remember a missing logo instead of re-warning every rotation load_logo() stat'd the path and logged a WARNING on every call, and the positive cache never covered it because a miss returns None and caches nothing. A file that is simply not there therefore produced one warning per rotation for as long as the process ran -- measured on a live rig at 114 lines in 24 hours for a single missing ticker icon, for a file nobody was going to add. Misses are now remembered for 10 minutes: warn once, then return None without touching the disk. Bounded rather than permanent because logo_downloader writes logos at runtime, so a file that appears later must still be picked up without a restart. Downloads through load_logo_with_download() clear the entry outright -- load_logo() consults the miss record before it stats the disk, so without that a freshly downloaded logo would stay invisible for the whole window. This is in the core rather than in ledmatrix-stocks, where it was found, so every plugin that goes through LogoHelper gets it. _cache_order stays a list. Swapping the pair for an OrderedDict would shave an O(n) scan per cache hit, but n is capped at cache_size (100 by default) and test_logo_helper.py pins the current structure; not worth the churn. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 * fix(logo): make the miss TTL longer than the rotation it is meant to outlast Deployed the previous commit to a live rig and measured it: no change at all. "Logo not found for VOO" stayed at ~6 lines an hour, exactly the baseline. The TTL was 600s and the display rotation is ~618s, so every recheck expired just as the plugin came round again and the negative cache never once got to suppress a warning. The fix was correct in shape and useless in practice, which only measuring on the rig would show. An hour instead. That is safe because the TTL is not the main way an entry clears: load_logo_with_download() drops it the moment a download succeeds and clear_cache() drops all of them. The TTL only covers a file that appeared some other way -- someone copying one in by hand -- and waiting up to an hour for that, or restarting, is a fair trade for not re-warning about a file nobody is going to add. The general lesson is in the comment: a TTL has to be long relative to the loop that does the asking, not merely "a while". Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 --------- Co-authored-by: Claude Opus 5 (1M context) --- src/common/logo_helper.py | 40 ++++++++++++++++++++++ test/test_logo_helper.py | 70 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/src/common/logo_helper.py b/src/common/logo_helper.py index a64ca340..0361a08b 100644 --- a/src/common/logo_helper.py +++ b/src/common/logo_helper.py @@ -8,6 +8,7 @@ Extracted from LEDMatrix core to provide reusable functionality for plugins. import logging import os import tempfile +import time from pathlib import Path from typing import Dict, List, Optional, Union @@ -20,6 +21,21 @@ from src.common.permission_utils import ( get_assets_file_mode ) +# How long a missing logo stays remembered as missing. +# +# This was 600s, and measured on a live rig that turned out to suppress nothing: +# the display rotation is ~618s, so every recheck landed just as the plugin came +# round again and the warning rate was unchanged at ~6/hour. A TTL has to be long +# relative to the loop that does the asking, not merely "a while". +# +# An hour is safe because the TTL is not the main way an entry clears. A download +# through load_logo_with_download() drops it immediately, and clear_cache() drops +# all of them; the TTL only covers a file that appeared some other way -- someone +# copying one in by hand. Waiting up to an hour for that, or restarting, is a fair +# trade for not re-warning about a file nobody is going to add. +MISSING_LOGO_RECHECK_SECONDS = 3600.0 + + # Well above any real team logo; bounds what a remote URL can write to disk. MAX_LOGO_BYTES = 10 * 1024 * 1024 @@ -56,6 +72,14 @@ class LogoHelper: # In-memory logo cache self._logo_cache: Dict[str, Image.Image] = {} self._cache_order: List[str] = [] # For LRU cache management + + # Misses, so an absent file is stat'd and warned about once rather than + # on every call. Without this a permanently missing logo produced a + # warning per rotation forever -- measured at 114 lines in 24 hours for + # a single missing ticker icon, for a file nobody was going to add. + # Time-bounded rather than permanent so a logo that appears later (the + # downloader writes them at runtime) is still picked up. + self._missing_logos: Dict[str, float] = {} # Session for HTTP requests self.session = requests.Session() @@ -100,9 +124,19 @@ class LogoHelper: self._cache_order.append(cache_key) return self._logo_cache[cache_key] + # A known-missing file: skip the stat and stay quiet until the entry + # ages out. Checked after the positive cache so a logo that has since + # been loaded always wins. + missed_at = self._missing_logos.get(cache_key) + if missed_at is not None: + if time.time() - missed_at < MISSING_LOGO_RECHECK_SECONDS: + return None + del self._missing_logos[cache_key] + try: logo_path = Path(logo_path) if not logo_path.exists(): + self._missing_logos[cache_key] = time.time() self.logger.warning(f"Logo not found for {team_abbr} at {logo_path}") return None @@ -179,6 +213,11 @@ class LogoHelper: self._logo_cache.pop(key, None) if key in self._cache_order: self._cache_order.remove(key) + # The file exists now, so any record of it being missing is wrong -- + # and load_logo() consults that record before it stats the disk, so + # leaving it would hide a logo we just downloaded. + for key in [k for k in self._missing_logos if k.startswith(prefix)]: + del self._missing_logos[key] @staticmethod def _refresh_stale_placeholder(logo_path: Path) -> None: @@ -270,6 +309,7 @@ class LogoHelper: """Clear the logo cache.""" self._logo_cache.clear() self._cache_order.clear() + self._missing_logos.clear() self.logger.debug("Logo cache cleared") def get_cache_stats(self) -> Dict[str, int]: diff --git a/test/test_logo_helper.py b/test/test_logo_helper.py index cb5824b2..0ae7ec44 100644 --- a/test/test_logo_helper.py +++ b/test/test_logo_helper.py @@ -24,7 +24,8 @@ import pytest import requests from PIL import Image, UnidentifiedImageError -from src.common.logo_helper import MAX_LOGO_BYTES, LogoHelper +from src.common.logo_helper import (MAX_LOGO_BYTES, MISSING_LOGO_RECHECK_SECONDS, + LogoHelper) @pytest.fixture(autouse=True) @@ -494,3 +495,70 @@ class TestStalePlaceholderHandling: helper.load_logo_with_download("COLL", path, "http://x/c.png") assert should_attempt_download(path) is False + + +class TestMissingLogosAreRememberedNotRewarned: + """A file that is not there does not become there by being asked again. + + load_logo() stat'd the path and logged a WARNING on every call, so a + permanently absent logo produced one warning per rotation for as long as + the process ran -- measured at 114 lines in 24 hours for a single missing + ticker icon. The positive cache never covered this because a miss returns + None and caches nothing. + """ + + def test_a_missing_logo_warns_once(self, helper, tmp_path, caplog): + absent = tmp_path / "VOO.png" + with caplog.at_level(logging.WARNING): + for _ in range(50): + assert helper.load_logo("VOO", absent) is None + assert caplog.text.count("Logo not found for VOO") == 1, ( + f"warned {caplog.text.count('Logo not found for VOO')} times in 50 calls") + + def test_the_miss_is_not_remembered_forever(self, helper, tmp_path, monkeypatch): + """A logo written later -- logo_downloader does this at runtime -- must + still be picked up without a restart.""" + path = tmp_path / "LATE.png" + assert helper.load_logo("LATE", path) is None + + write_logo(path) + # Still inside the recheck window: the remembered miss stands. + assert helper.load_logo("LATE", path) is None + + # Once it ages out, the next call stats the disk again and finds it. + for key in helper._missing_logos: + helper._missing_logos[key] -= MISSING_LOGO_RECHECK_SECONDS + 1 + assert helper.load_logo("LATE", path) is not None + + def test_a_download_clears_the_miss_immediately(self, helper, tmp_path): + """load_logo_with_download() must not be defeated by its own miss record. + + load_logo() consults _missing_logos before it stats the disk, so a + logo we just downloaded would stay invisible for the whole recheck + window without the explicit invalidation. + """ + path = tmp_path / "NEW.png" + assert helper.load_logo("NEW", path) is None + assert any(k.startswith("NEW_") for k in helper._missing_logos) + + def _fake_download(url, file_path): + write_logo(Path(file_path)) + + with patch.object(helper, "_download_logo", side_effect=_fake_download): + got = helper.load_logo_with_download("NEW", path, logo_url="http://x/NEW.png") + assert got is not None, "a freshly downloaded logo was hidden by the miss record" + assert not any(k.startswith("NEW_") for k in helper._missing_logos) + + def test_clear_cache_forgets_misses_too(self, helper, tmp_path): + helper.load_logo("GONE", tmp_path / "GONE.png") + assert helper._missing_logos + helper.clear_cache() + assert helper._missing_logos == {} + + def test_a_present_logo_is_unaffected(self, helper, tmp_path, caplog): + path = write_logo(tmp_path / "OK.png") + with caplog.at_level(logging.WARNING): + for _ in range(10): + assert helper.load_logo("OK", path) is not None + assert "Logo not found" not in caplog.text + assert helper._missing_logos == {} From 8da13f02f81a1b83e7b547e5d8a1bfd53da66ba6 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:41:44 -0400 Subject: [PATCH 04/16] chore: ignore team logos fetched at runtime (#551) logo_downloader.py and LogoHelper write into assets/sports/_logos/ whenever a plugin meets a team whose logo is not on disk. Those directories are also tracked -- 209 NCAA logos and 153 soccer ones ship with the repo -- so every rig accumulates untracked files nobody intended to commit. This checkout had 62; hdpi shows the same. The cost is not the files, it is that a permanently dirty `git status` trains everyone to ignore the one signal that says a checkout is not what you think it is. That is how a stale tree sat unnoticed on a rig for hours until a restart surfaced four sports plugins that could no longer import. Ignoring a directory does not untrack what is already in it, so the logos that ship keep shipping -- verified: 209 and 153 still tracked, no deletions in the diff. Only new downloads are hidden. Adding a logo on purpose stays possible and is what the escape hatch in the comment documents. It is also rare: the last deliberate addition was #415, four named NCAA logos a plugin needed, and `git log` finds no other in a year. So the common case is noise and the rare case is explicit, which is the right way round. Untracked files: 62 -> 0. Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 Co-authored-by: Claude Opus 5 (1M context) --- .gitignore | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.gitignore b/.gitignore index fdfaa90b..641d7c44 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,25 @@ skin_renders/ # JS test deps (test/js) node_modules/ package-lock.json + +# Team logos fetched at runtime. +# +# src/logo_downloader.py and LogoHelper write into assets/sports/_logos/ +# whenever a plugin meets a team whose logo is not on disk. Those directories are +# also tracked -- 209 NCAA logos and 153 soccer ones ship with the repo -- so +# every rig accumulated untracked files it was never meant to commit and +# `git status` was permanently dirty. That noise is not harmless: it trains +# everyone to ignore the one signal that says a checkout is not what you think +# it is, which is how a stale tree sat unnoticed on a rig until a restart +# surfaced four dead sports plugins. +# +# Ignoring a directory does not untrack what is already in it, so the logos that +# ship keep shipping. Only new downloads are hidden. +# +# Adding a logo on purpose is rare and deliberate -- the last time was #415, four +# named NCAA logos a plugin needed, and there has been no other in a year. Do it +# with an explicit override: +# git add -f assets/sports/ncaa_logos/DUKE.png +assets/sports/*_logos/ +assets/stocks/ticker_icons/ +assets/stocks/crypto_icons/ From fb3b293ace64df31e478a47628be0d65d6790905 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:41:58 -0400 Subject: [PATCH 05/16] fix(plugins): let a plugin ask to be polled faster while it has live content (#555) * fix(plugins): let a plugin ask to be polled faster while it has live content Reported: "the football plugin with live games only updates the live game in progress if I restart the display." The data path was never the problem. NFLLiveManager fetches ESPN with no cache, SportsLive.update() refreshes current_game in place when the game IDs are unchanged, and the scorebug redraws from the game dict every frame -- which is why the reporter's logs look healthy. The problem is cadence. _get_plugin_update_interval() read only the manifest's static update_interval, football's manifest pins that to 60, and the plugin's own live_update_interval (15s) was invisible to the scheduler. Measured on a rig during the fourth quarter of the game in the report: 23:21:49 23:22:50 23:23:50 23:24:50 23:25:50 <- exactly 60s apart A clock and score up to a minute stale during a two-minute drill reads as a frozen panel, and a restart is the one moment it is ever current. A single static number cannot say "every 15 seconds while a game is on, every 15 minutes in July", and only the plugin knows which is true. get_update_interval() lets it say so per tick; returning None means "no opinion" and the existing manifest/config resolution applies, so every plugin that predates this is unaffected. Requests are clamped to MIN_DYNAMIC_UPDATE_INTERVAL (5s): a plugin returning 0 would otherwise be re-entered on every tick of the render loop, busy-waiting against its own API. A hook that raises or returns a non-number is ignored rather than propagated -- a scheduler that fails on one plugin's bug stops updating all the others. Deliberately NOT changed: the manifest still beats config in the static path. That looked like the obvious fix -- user config being silently ignored -- until checking a real rig, where football and baseball both carry update_interval 3600 in config against a manifest 60, and weather 1800 against 60. Those values are stale precisely because nothing has been honouring them; making config win would have slowed three plugins by 60x, turning a one-minute lag into an hour. The dynamic hook makes the flip unnecessary. There is a test pinning the current precedence with that reasoning attached. Full suite: 4,283 passed, 68 skipped. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 * test(plugins): drive the real scheduler, not just the interval resolver test_plugin_dynamic_update_interval.py asserts that _get_plugin_update_interval() returns the number the plugin asked for. That is not the same claim as "the plugin gets updated more often", and the gap between those two is exactly where the original bug lived: the plugin knew it wanted 15s, said so in live_update_interval, and nothing downstream acted on it. So this ticks the real run_scheduled_updates() through a simulated hour and counts dispatches. Against pre-fix core it reports "10 updates in 10 minutes of a live game" -- the 60s manifest cadence, matching what was measured on a rig during the reported game. Against the fix it reports ~40. Also pins the regression that would be worse than the bug: an idle hour must still be ~60 updates, not 240. Asking for the live interval year-round would poll ESPN four times a minute all summer. Scope note, since it is easy to over-read this fix: the *switch* display path already refreshed the manager immediately before drawing, via _try_manager_display() -> _ensure_manager_updated(), which honours the manager's own 15s interval. So a switch-mode card was already <=15s stale at draw time before this change. What this fixes is the background cadence, which is what live-priority detection, Vegas content and scroll preparation all read. Full suite: 4,288 passed, 68 skipped. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 * fix(plugins): reject bool and -inf hook results in dynamic interval get_update_interval() ran bool through float() (bool is an int subclass, so True/False became 1.0/0.0) and only checked for +inf, not -inf. Both cases landed on the MIN_DYNAMIC_UPDATE_INTERVAL floor by coincidence instead of falling back to the static/manifest interval as invalid input should. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_017Tst9cied2ri9bH4QRWa6H --------- Co-authored-by: Claude Opus 5 (1M context) --- src/plugin_system/base_plugin.py | 32 ++++ src/plugin_system/plugin_manager.py | 62 +++++++- test/test_live_update_cadence_integration.py | 159 +++++++++++++++++++ test/test_plugin_dynamic_update_interval.py | 153 ++++++++++++++++++ 4 files changed, 402 insertions(+), 4 deletions(-) create mode 100644 test/test_live_update_cadence_integration.py create mode 100644 test/test_plugin_dynamic_update_interval.py diff --git a/src/plugin_system/base_plugin.py b/src/plugin_system/base_plugin.py index dbe14a49..54aa7426 100644 --- a/src/plugin_system/base_plugin.py +++ b/src/plugin_system/base_plugin.py @@ -520,6 +520,38 @@ class BasePlugin(ABC): """ return + def get_update_interval(self) -> Optional[float]: + """ + How often this plugin wants update() called, right now, in seconds. + + The manifest's ``update_interval`` is a single static number, which + cannot say "poll me every 15 seconds while a game is in progress and + every 15 minutes when nothing is on". Only the plugin knows which is + true at any moment, so override this to say so. + + Return None (the default) to accept the manifest/config value. + + Two constraints, both because the scheduler calls this on every tick of + the render loop: + + - It must be cheap. Attribute reads only -- no config lookups, no I/O, + no locks that a fetch might be holding. + - It must not raise. A raising hook is ignored and the static interval + used, but a hook that raises every tick also logs every tick. + + Values below PluginManager.MIN_DYNAMIC_UPDATE_INTERVAL are clamped up: + a plugin asking for 0 would otherwise busy-wait against its own API. + + Example:: + + def get_update_interval(self): + # Fast while something is actually live, manifest default otherwise. + if any(m.live_games for m in self._live_managers): + return self.config.get("live_update_interval", 15) + return None + """ + return None + def has_live_priority(self) -> bool: """ Check if this plugin has live priority enabled. diff --git a/src/plugin_system/plugin_manager.py b/src/plugin_system/plugin_manager.py index 0dc7a428..2c8de821 100644 --- a/src/plugin_system/plugin_manager.py +++ b/src/plugin_system/plugin_manager.py @@ -8,6 +8,7 @@ API Version: 1.0.0 """ import json +import math import queue import sys import time @@ -779,15 +780,68 @@ class PluginManager: return None + def _dynamic_update_interval(self, plugin_id: str, plugin_instance: Any) -> Optional[float]: + """The interval a plugin asks for right now, or None if it has no view.""" + hook = getattr(plugin_instance, 'get_update_interval', None) + if not callable(hook): + return None + try: + requested = hook() + except Exception as exc: # pylint: disable=broad-except + self.logger.debug( + "get_update_interval() failed for %s, using the static interval: %s", + plugin_id, exc) + return None + if requested is None: + return None + if isinstance(requested, bool): + self.logger.debug( + "get_update_interval() returned a bool for %s, which is not a number", + plugin_id) + return None + try: + requested = float(requested) + except (TypeError, ValueError): + self.logger.debug( + "get_update_interval() returned %r for %s, which is not a number", + requested, plugin_id) + return None + if not math.isfinite(requested): # NaN / +inf / -inf + return None + return max(requested, self.MIN_DYNAMIC_UPDATE_INTERVAL) + + #: Floor for a plugin-requested interval. A plugin asking for 0 (or a + #: negative) would otherwise be re-entered on every tick of the render + #: loop, which is a busy-wait against whatever API it fetches. + MIN_DYNAMIC_UPDATE_INTERVAL = 5.0 + def _get_plugin_update_interval(self, plugin_id: str, plugin_instance: Any) -> Optional[float]: """ Get the data-fetch interval for a plugin (seconds between update() calls). - Result is cached per plugin_id after the first lookup to avoid calling - config_manager.get_config() — which returns a full dict copy — on every - tick of the 30-fps display loop. The cache is invalidated when a plugin - is loaded or unloaded. + A plugin may implement ``get_update_interval()`` to vary its own cadence + at runtime, which the static manifest value cannot express. The case + this exists for: a sports scoreboard needs to poll every 15s while a + game is in progress and every 15 minutes when nothing is on, and only + the plugin knows which is true right now. Returning None from the hook + means "no opinion", and the static resolution below applies. + + The hook is called on every scheduling tick, so implementations must be + cheap — attribute reads, no config lookups and no I/O. A raising or + non-numeric hook is ignored rather than allowed to stop the plugin + updating, since a scheduler that propagates a plugin bug stops every + other plugin too. + + The static result is cached per plugin_id after the first lookup to + avoid calling config_manager.get_config() — which returns a full dict + copy — on every tick of the 30-fps display loop. The cache is + invalidated when a plugin is loaded or unloaded. The dynamic hook is + deliberately *not* cached: caching it would defeat its only purpose. """ + dynamic = self._dynamic_update_interval(plugin_id, plugin_instance) + if dynamic is not None: + return dynamic + if plugin_id in self._update_interval_cache: return self._update_interval_cache[plugin_id] diff --git a/test/test_live_update_cadence_integration.py b/test/test_live_update_cadence_integration.py new file mode 100644 index 00000000..20724b27 --- /dev/null +++ b/test/test_live_update_cadence_integration.py @@ -0,0 +1,159 @@ +"""The scheduler actually polls faster while a plugin reports live content. + +test_plugin_dynamic_update_interval.py proves _get_plugin_update_interval() +returns the plugin's requested number. That is not the same claim as "the +plugin gets updated more often", and the gap between those two is exactly where +the original bug lived: the plugin knew it wanted 15s, said so in +live_update_interval, and nothing downstream acted on it. + +So this drives the real run_scheduled_updates() over a simulated hour of ticks +and counts dispatches. It is the test that would have caught the reported bug: + + "the football plugin with live games only updates the live game in + progress if I restart the display" + +Measured on a rig before the fix, during an NFL fourth quarter, ESPN was polled +at 23:21:49 / 23:22:50 / 23:23:50 / 23:24:50 -- exactly the manifest's 60s, +never the configured 15. +""" + +from unittest.mock import MagicMock + +import pytest + +from src.plugin_system.plugin_manager import PluginManager +from src.plugin_system.plugin_state import PluginState + + +class _StateManager: + """Enough of the real state machine for reservation to behave.""" + + def __init__(self): + self.states = {} + + def can_execute(self, plugin_id): + return self.states.get(plugin_id, PluginState.ENABLED) != PluginState.RUNNING + + def set_state(self, plugin_id, state): + self.states[plugin_id] = state + + +class _Plugin: + """A sports plugin: fast while something is live, quiet otherwise.""" + + enabled = True + + def __init__(self, live_interval=15): + self.live = False + self._live_interval = live_interval + + def get_update_interval(self): + return self._live_interval if self.live else None + + def update(self): + pass + + +@pytest.fixture +def scheduler(): + pm = PluginManager.__new__(PluginManager) + import threading + pm._update_interval_cache = {} + pm.plugin_manifests = {"sports": {"update_interval": 60}} + pm.config_manager = None + pm.logger = MagicMock() + pm.state_manager = _StateManager() + pm.health_tracker = None + pm._synchronous_updates = False + pm._reservation_lock = threading.Lock() + pm._plugin_last_update_lock = threading.Lock() + pm.plugin_last_update = {} + pm.dispatched = [] + + # Stand in for the background worker: record the dispatch and complete it, + # so the next tick can reserve the plugin again. + def _enqueue(plugin_id, scheduled_time): + pm.dispatched.append(scheduled_time) + pm.plugin_last_update[plugin_id] = scheduled_time + pm.state_manager.set_state(plugin_id, PluginState.ENABLED) + + pm._enqueue_update = _enqueue + return pm + + +def _run_for(pm, plugin, seconds, start=1_000_000.0, step=1.0): + """Tick the real scheduler once a second for `seconds`.""" + pm.plugins = {"sports": plugin} + t = start + end = start + seconds + while t < end: + pm.run_scheduled_updates(current_time=t) + t += step + return len(pm.dispatched) + + +class TestTheReportedBug: + def test_a_live_game_is_polled_at_the_live_interval(self, scheduler): + """15s while live, not the manifest's 60s. This is the fix.""" + plugin = _Plugin(live_interval=15) + plugin.live = True + count = _run_for(scheduler, plugin, seconds=600) # ten minutes + # 600s / 15s = 40, allowing one for the first tick's free run. + assert 38 <= count <= 41, ( + f"{count} updates in 10 minutes of a live game; expected ~40 at 15s. " + f"At the manifest's 60s it would be ~10 -- the reported bug.") + + def test_the_same_window_without_the_hook_is_the_old_behaviour(self, scheduler): + """Pin what the bug actually looked like, so the contrast is asserted.""" + plugin = _Plugin() + plugin.live = True + plugin.get_update_interval = lambda: None # pre-fix: no opinion + count = _run_for(scheduler, plugin, seconds=600) + assert 9 <= count <= 11, ( + f"{count} updates in 10 minutes; expected ~10 at the manifest's 60s") + + def test_idle_polling_is_unchanged(self, scheduler): + """The regression that would be worse than the bug. + + Asking for 15s year-round would poll ESPN four times a minute all + summer. Idle must stay exactly on the manifest. + """ + plugin = _Plugin() + plugin.live = False + count = _run_for(scheduler, plugin, seconds=3600) # a full hour idle + assert 59 <= count <= 61, ( + f"{count} updates in an idle hour; expected ~60 at the manifest's 60s") + + +class TestTheCadenceTracksTheGame: + def test_it_speeds_up_when_a_game_starts_and_slows_when_it_ends(self, scheduler): + """No reload, no restart -- the whole point of a per-tick hook.""" + plugin = _Plugin(live_interval=15) + pm = scheduler + + _run_for(pm, plugin, seconds=300, start=1_000_000.0) + idle_before = len(pm.dispatched) + + plugin.live = True + _run_for(pm, plugin, seconds=300, start=1_000_300.0) + during = len(pm.dispatched) - idle_before + + plugin.live = False + _run_for(pm, plugin, seconds=300, start=1_000_600.0) + idle_after = len(pm.dispatched) - idle_before - during + + assert during > idle_before * 3, ( + f"live window got {during} updates vs {idle_before} idle; " + "the hook did not speed anything up") + assert idle_after <= idle_before + 1, ( + f"{idle_after} updates after the game ended vs {idle_before} before; " + "the fast cadence leaked past the live window") + + +def test_a_broken_hook_does_not_stop_the_plugin_updating(scheduler): + """A scheduler that propagates one plugin's bug stops every other plugin.""" + plugin = _Plugin() + plugin.get_update_interval = MagicMock(side_effect=RuntimeError("boom")) + count = _run_for(scheduler, plugin, seconds=600) + assert 9 <= count <= 11, ( + f"{count} updates; a raising hook should fall back to the manifest's 60s") diff --git a/test/test_plugin_dynamic_update_interval.py b/test/test_plugin_dynamic_update_interval.py new file mode 100644 index 00000000..bd70fce0 --- /dev/null +++ b/test/test_plugin_dynamic_update_interval.py @@ -0,0 +1,153 @@ +"""A plugin can ask to be polled faster while it has something live. + +The manifest carries one static `update_interval`, and until now that was the +only thing the scheduler would look at. A sports scoreboard needs 15 seconds +while a game is in progress and 15 minutes when nothing is on, and no single +number expresses that. + +Measured consequence, on a live rig during an NFL game in its fourth quarter: +football-scoreboard's manifest pins update_interval to 60, so ESPN was polled +exactly once a minute -- + + 23:21:49 23:22:50 23:23:50 23:24:50 23:25:50 + +-- while the plugin's own live_update_interval said 15. The clock and score on +the panel therefore lagged by up to a minute during a two-minute drill, which +reads to a viewer as a frozen display. + +get_update_interval() lets the plugin say what it needs per tick. These tests +pin the parts that are easy to get wrong: that the hook wins, that "no opinion" +falls back cleanly, that a broken hook cannot stop a plugin updating, and that +the hook is not accidentally cached (which would defeat the whole point). +""" + +from unittest.mock import MagicMock + +import pytest + +from src.plugin_system.plugin_manager import PluginManager + + +@pytest.fixture +def manager(): + pm = PluginManager.__new__(PluginManager) + pm._update_interval_cache = {} + pm.plugin_manifests = {"sports": {"update_interval": 60}} + pm.config_manager = None + pm.logger = MagicMock() + return pm + + +class _Plugin: + def __init__(self, wants=None, raises=False): + self._wants = wants + self._raises = raises + self.calls = 0 + + def get_update_interval(self): + self.calls += 1 + if self._raises: + raise RuntimeError("plugin is broken") + return self._wants + + +def test_the_hook_overrides_the_manifest(manager): + """The whole point: 15s while live, not the manifest's 60.""" + assert manager._get_plugin_update_interval("sports", _Plugin(wants=15)) == 15.0 + + +def test_none_means_no_opinion_and_the_manifest_applies(manager): + """A plugin with nothing live should not have to restate the default.""" + assert manager._get_plugin_update_interval("sports", _Plugin(wants=None)) == 60.0 + + +def test_a_plugin_without_the_hook_is_unaffected(manager): + """Every existing plugin predates this and must keep its manifest value.""" + assert manager._get_plugin_update_interval("sports", MagicMock(spec=[])) == 60.0 + + +def test_the_hook_is_consulted_every_time(manager): + """Caching the answer would make it exactly as static as the manifest. + + The static path *is* cached -- reading config is expensive -- so it would be + an easy mistake to let the dynamic answer ride along in that cache. + """ + plugin = _Plugin(wants=15) + for _ in range(5): + manager._get_plugin_update_interval("sports", plugin) + assert plugin.calls == 5, f"hook called {plugin.calls} times in 5 lookups" + + +def test_the_cadence_can_change_between_ticks(manager): + """A game going final must slow the polling back down without a reload.""" + plugin = _Plugin(wants=15) + assert manager._get_plugin_update_interval("sports", plugin) == 15.0 + plugin._wants = None # game ended + assert manager._get_plugin_update_interval("sports", plugin) == 60.0 + plugin._wants = 15 # another game started + assert manager._get_plugin_update_interval("sports", plugin) == 15.0 + + +class TestABrokenHookCannotStopUpdates: + """A scheduler that propagates a plugin's bug stops every other plugin too.""" + + def test_a_raising_hook_falls_back_to_the_manifest(self, manager): + assert manager._get_plugin_update_interval("sports", _Plugin(raises=True)) == 60.0 + + @pytest.mark.parametrize("junk", ["soon", object(), [], {}]) + def test_a_non_numeric_hook_falls_back(self, manager, junk): + assert manager._get_plugin_update_interval("sports", _Plugin(wants=junk)) == 60.0 + + @pytest.mark.parametrize("junk", [float("nan"), float("inf"), float("-inf")]) + def test_nan_and_infinity_fall_back(self, manager, junk): + assert manager._get_plugin_update_interval("sports", _Plugin(wants=junk)) == 60.0 + + @pytest.mark.parametrize("junk", [True, False]) + def test_a_bool_hook_falls_back(self, manager, junk): + """bool is a subclass of int, so it must be rejected before float() + turns it into 0.0/1.0 and the floor silently swallows the mistake.""" + assert manager._get_plugin_update_interval("sports", _Plugin(wants=junk)) == 60.0 + + +class TestTheFloor: + """A plugin asking for 0 would be re-entered on every tick of the render + loop, which is a busy-wait against whatever API it fetches.""" + + @pytest.mark.parametrize("wants", [0, 0.5, -10]) + def test_small_and_negative_requests_are_clamped(self, manager, wants): + got = manager._get_plugin_update_interval("sports", _Plugin(wants=wants)) + assert got == PluginManager.MIN_DYNAMIC_UPDATE_INTERVAL + + def test_a_reasonable_request_is_not_clamped(self, manager): + assert manager._get_plugin_update_interval("sports", _Plugin(wants=15)) == 15.0 + + +def test_the_static_path_still_prefers_the_manifest_over_config(): + """Deliberately unchanged, and not an oversight. + + Config values that have sat inert behind the manifest are stale by + definition -- nothing has been honouring them. On one rig, football and + baseball both carry update_interval 3600 in config against a manifest 60; + making config win would slow those plugins by 60x. The dynamic hook is the + supported way for a plugin to vary its own cadence, so flipping this is + both risky and no longer necessary. + """ + pm = PluginManager.__new__(PluginManager) + pm._update_interval_cache = {} + pm.plugin_manifests = {"sports": {"update_interval": 60}} + cfg = MagicMock() + cfg.get_config.return_value = {"sports": {"update_interval": 3600}} + pm.config_manager = cfg + pm.logger = MagicMock() + assert pm._get_plugin_update_interval("sports", MagicMock(spec=[])) == 60.0 + + +def test_config_is_still_used_when_the_manifest_is_silent(): + pm = PluginManager.__new__(PluginManager) + pm._update_interval_cache = {} + pm.plugin_manifests = {"sports": {}} + cfg = MagicMock() + cfg.get_config.return_value = {"sports": {"update_interval": 120}} + pm.config_manager = cfg + pm.logger = MagicMock() + assert pm._get_plugin_update_interval("sports", MagicMock(spec=[])) == 120.0 From ad5bc4b8190a3f2b774fbf089716b5a488861150 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:42:18 -0400 Subject: [PATCH 06/16] perf(sports): LRU-bound the decoded logo cache (#559) SportsCore._logo_cache was a plain dict keyed by team abbreviation with no eviction. Its entries are not file bytes but decoded RGBA thumbnails sized to display*1.5 -- roughly 36KB on a 256x64 panel, more for wide wordmarks -- and assets/sports/ncaa_logos ships 307 of them. A plugin that walked a full league held the whole league resident: about 11-18MB per manager instance, and a league runs three (live/recent/upcoming) that each keep their own cache, so the same logos were duplicated across them. On the 1GB Pi 3B+ this was measured on, one board was sitting at 439MB resident with ~290MB available, so tens of megabytes of duplicated league logos is real money. Bounded to 64 entries, which holds a full "other games" cycle (on the order of 20 games, 40 teams) without thrashing while capping the cache well below a 307-team league. Eviction is LRU rather than clear-when-full, using the OrderedDict/popitem pattern the neighbouring caches in this codebase already use (_IMAGE_CACHE_MAX, _FIT_CACHE_MAX, _TEXT_WIDTH_CACHE_MAX). That ordering matters: the logos on screen right now are precisely the ones that must not be discarded, so a cache hit moves the entry to the end. Co-authored-by: Claude Opus 5 --- src/base_classes/sports/core.py | 18 +++- test/test_sports_logo_cache_bounded.py | 120 +++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 test/test_sports_logo_cache_bounded.py diff --git a/src/base_classes/sports/core.py b/src/base_classes/sports/core.py index 251ba5cb..452e9bcc 100644 --- a/src/base_classes/sports/core.py +++ b/src/base_classes/sports/core.py @@ -8,6 +8,7 @@ import os import tempfile import time from abc import ABC, abstractmethod +from collections import OrderedDict from datetime import datetime, timedelta from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -167,7 +168,14 @@ class SportsCore(ABC): self.session.mount("https://", adapter) self.session.mount("http://", adapter) - self._logo_cache = {} + # LRU-bounded: entries are decoded RGBA thumbnails, not file bytes. + # Each is up to display_width*1.5 x display_height*1.5 -- about 36KB on + # a 256x64 panel, more for wide wordmarks. The key is a team + # abbreviation and assets/sports/ncaa_logos alone ships 307 of them, so + # an unbounded dict here held the whole league: ~11-18MB per manager + # instance, and a league runs three (live/recent/upcoming) each with + # its own cache. That is real money on a 1GB Pi. + self._logo_cache: "OrderedDict[str, Image.Image]" = OrderedDict() # Font caches for _load_custom_font_from_element_config: per-frame # callers (font-ladder walks) resolve the same (name, size) over and @@ -560,11 +568,17 @@ class SportsCore(ABC): draw.text((x + dx, y + dy), text, font=font, fill=outline_color) draw.text((x, y), text, font=font, fill=fill) + #: Decoded logos to keep. A scroll of "other games" shows on the order of + #: 20 games (40 teams), so this holds a full cycle without thrashing while + #: capping the cache well below a 307-team league. + _LOGO_CACHE_MAX = 64 + def _load_and_resize_logo(self, team_id: str, team_abbrev: str, logo_path: Path, logo_url: str | None ) -> Optional[Image.Image]: """Load and resize a team logo, with caching and automatic download if missing.""" self.logger.debug(f"Logo path: {logo_path}") if team_abbrev in self._logo_cache: self.logger.debug(f"Using cached logo for {team_abbrev}") + self._logo_cache.move_to_end(team_abbrev) return self._logo_cache[team_abbrev] try: @@ -604,6 +618,8 @@ class SportsCore(ABC): max_height = int(self.display_height * 1.5) logo.thumbnail((max_width, max_height), Image.Resampling.LANCZOS) self._logo_cache[team_abbrev] = logo + while len(self._logo_cache) > self._LOGO_CACHE_MAX: + self._logo_cache.popitem(last=False) return logo except Exception as e: diff --git a/test/test_sports_logo_cache_bounded.py b/test/test_sports_logo_cache_bounded.py new file mode 100644 index 00000000..8570b81e --- /dev/null +++ b/test/test_sports_logo_cache_bounded.py @@ -0,0 +1,120 @@ +"""Tests that SportsCore's decoded-logo cache is LRU-bounded. + +``self._logo_cache`` was a plain dict keyed by team abbreviation, with no +eviction. The entries are decoded RGBA thumbnails sized to display*1.5 -- about +36KB on a 256x64 panel, more for wide wordmarks -- and +assets/sports/ncaa_logos ships 307 of them. A plugin that walked a full league +therefore held the whole league in memory: roughly 11-18MB per manager +instance, and a league runs three of them (live/recent/upcoming) each with its +own cache. On a 1GB Pi 3B+ with ~290MB available that is worth recovering. + +The bound has to be LRU rather than "clear when full": the logos on screen +right now are exactly the ones that must not be thrown away. +""" + +import logging +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from PIL import Image + +# src.base_classes.sports transitively imports the hardware matrix driver. +sys.modules.setdefault("rgbmatrix", MagicMock()) + +from src.base_classes.sports import SportsCore # noqa: E402 + +LOGGER = logging.getLogger("test_sports_logo_cache_bounded") + + +class _StubSports(SportsCore): + def _fetch_data(self): + return None + + def _extract_game_details(self, game_event): + return None + + +@pytest.fixture +def host(monkeypatch, tmp_path): + monkeypatch.setattr( + SportsCore, "_initialize_logo_dir", lambda self, configured: tmp_path) + monkeypatch.setattr( + "src.base_classes.sports.core.get_background_service", + lambda *args, **kwargs: MagicMock()) + # Never reach for the network: every logo these tests ask for exists. + monkeypatch.setattr( + "src.base_classes.sports.core.download_missing_logo", + lambda *a, **k: None) + + display_manager = MagicMock() + display_manager.matrix.width = 128 + display_manager.matrix.height = 32 + display_manager.width = 128 + display_manager.height = 32 + display_manager.image = Image.new("RGB", (128, 32)) + cache_manager = MagicMock() + cache_manager.cache_dir = str(tmp_path) + instance = _StubSports({"timezone": "UTC"}, display_manager, + cache_manager, LOGGER, "nhl") + instance._logo_dir = tmp_path + return instance, tmp_path + + +def _load(host, abbrev): + """Create a real logo file for `abbrev` and load it through the cache.""" + instance, tmp_path = host + path = tmp_path / f"{abbrev}.png" + if not path.exists(): + Image.new("RGBA", (64, 64), (1, 2, 3, 255)).save(path) + return instance._load_and_resize_logo(abbrev, abbrev, path, None) + + +class TestTheCacheIsBounded: + def test_it_never_exceeds_the_limit(self, host): + instance, _ = host + limit = instance._LOGO_CACHE_MAX + for i in range(limit + 40): + assert _load(host, f"T{i}") is not None + # The regression: this grew to limit + 40, and for NCAA to 307. + assert len(instance._logo_cache) == limit + + def test_the_limit_is_smaller_than_a_real_league(self, host): + """ncaa_logos ships 307 files; the cap has to be well under that.""" + instance, _ = host + assert instance._LOGO_CACHE_MAX < 307 + + +class TestEvictionIsLRUNotArbitrary: + def test_the_least_recently_used_goes_first(self, host): + instance, _ = host + limit = instance._LOGO_CACHE_MAX + for i in range(limit): + _load(host, f"T{i}") + _load(host, "NEW") + assert "T0" not in instance._logo_cache, "oldest should have been evicted" + assert "NEW" in instance._logo_cache + + def test_touching_an_entry_saves_it(self, host): + instance, _ = host + limit = instance._LOGO_CACHE_MAX + for i in range(limit): + _load(host, f"T{i}") + _load(host, "T0") # a cache hit -- T0 is on screen again + _load(host, "NEW") # forces one eviction + + assert "T0" in instance._logo_cache, "a logo in use was thrown away" + assert "T1" not in instance._logo_cache, "T1 was the true LRU entry" + + +class TestHitsStillAvoidDiskWork: + def test_a_second_request_returns_the_cached_object(self, host, monkeypatch): + instance, _ = host + first = _load(host, "TB") + + def _boom(*a, **k): + raise AssertionError("cache hit re-opened the file from disk") + + monkeypatch.setattr(Image, "open", _boom) + assert _load(host, "TB") is first From dcd6e39c96eac4fa0aafed53c2d846551cb118a5 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:42:30 -0400 Subject: [PATCH 07/16] fix(web): report real disk usage and MemAvailable on the live status stream (#558) The SSE status stream sent 'disk_used_percent': 0 as a literal, so every consumer of the live view showed 0% disk no matter how full the card was. /api/v3/system/status computed it correctly; the stream that the dashboard actually watches did not. On a Pi with a modest SD card that is the warning a user most needs, and it was guaranteed to never appear. The stream also omitted memory_available_mb. /api/v3/system/status carries it with a comment spelling out why it matters: MemAvailable accounts for reclaimable page cache, so it is what separates a board reading 70% "used" that is fine from one reading 70% that is about to fail fork(). A 1GB Pi 3B+ can sit at either. The number that predicts the failure was missing from the live view. An unreadable disk now reports None rather than 0. The UI already renders null as '--'; a confident 0 reads as "plenty of room", which is worse than a blank. Metric collection moves to web_interface/system_metrics.py, with no Flask or app imports. That is not cosmetic: importing web_interface.app constructs the Flask application and a CacheManager, and the latter claims the cache directory with a cleanup thread. The first version of these tests imported the generator directly and broke test_cache_cleanup_thread_ownership ("one thread per directory") plus four starlark route tests through that side effect. Reading a CPU percentage should not boot a web application, and testing it should not either. Co-authored-by: Claude Opus 5 --- test/test_live_status_fields.py | 103 ++++++++++++++++++++++++++++++++ web_interface/app.py | 31 ++++------ web_interface/system_metrics.py | 62 +++++++++++++++++++ 3 files changed, 175 insertions(+), 21 deletions(-) create mode 100644 test/test_live_status_fields.py create mode 100644 web_interface/system_metrics.py diff --git a/test/test_live_status_fields.py b/test/test_live_status_fields.py new file mode 100644 index 00000000..571683de --- /dev/null +++ b/test/test_live_status_fields.py @@ -0,0 +1,103 @@ +"""Tests for the metrics the live status stream reports. + +Two gaps made a struggling 1GB Pi look healthy on the dashboard: + +* ``disk_used_percent`` was the literal ``0`` in the SSE generator, so every + consumer of the live stream displayed 0% disk no matter how full the card was. + /api/v3/system/status computed it properly, but the live view -- the one people + actually watch -- did not. + +* ``memory_available_mb`` was absent. /api/v3/system/status carries it with a + comment explaining that MemAvailable, not used%, is what separates a board + that is fine from one about to fail fork(). The live stream omitted the one + number that predicts the failure. + +A wrong number is worse than a missing one, so an unreadable disk now reports +None (the UI renders '--') rather than a confident 0. + +These import web_interface.system_metrics rather than web_interface.app on +purpose: importing the app constructs a Flask application and a CacheManager, +and the latter claims the cache directory with a cleanup thread -- which broke +test_cache_cleanup_thread_ownership and the starlark route tests when this file +first reached for the generator directly. +""" + +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +PROJECT_ROOT = Path(__file__).parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +from web_interface.system_metrics import collect_system_metrics # noqa: E402 + + +@pytest.fixture +def metrics(monkeypatch): + """collect_system_metrics() with psutil's readings under our control.""" + def _collect(disk_percent=12.0, available_bytes=512 * 1024 * 1024, + used_percent=67.7, raise_disk=False): + import psutil + if raise_disk: + monkeypatch.setattr(psutil, "disk_usage", + lambda _p: (_ for _ in ()).throw(OSError("no such mount"))) + else: + monkeypatch.setattr(psutil, "disk_usage", + lambda _p: SimpleNamespace(percent=disk_percent)) + monkeypatch.setattr(psutil, "virtual_memory", + lambda: SimpleNamespace(percent=used_percent, + available=available_bytes)) + return collect_system_metrics() + return _collect + + +class TestDiskUsageIsReal: + def test_reported_value_comes_from_the_filesystem(self, metrics): + # The regression: this field was the literal 0. + assert metrics(disk_percent=91.4)["disk_used_percent"] == 91.4 + + def test_a_full_disk_is_not_reported_as_zero(self, metrics): + assert metrics(disk_percent=99.9)["disk_used_percent"] != 0 + + def test_unreadable_disk_is_null_not_a_confident_zero(self, metrics): + # null renders as '--'; 0 would read as "plenty of room". + assert metrics(raise_disk=True)["disk_used_percent"] is None + + +class TestMemAvailableIsReported: + def test_mem_available_is_present_and_in_mb(self, metrics): + got = metrics(available_bytes=292 * 1024 * 1024)["memory_available_mb"] + assert got == pytest.approx(292.0, abs=0.1) + + def test_the_1gb_danger_case_is_visible(self, metrics): + """67% used can be fine or nearly out of memory; only MemAvailable says.""" + m = metrics(used_percent=67.7, available_bytes=40 * 1024 * 1024) + assert m["memory_used_percent"] == 67.7 + assert m["memory_available_mb"] == pytest.approx(40.0, abs=0.1) + + +class TestEveryPredictiveFieldIsPresent: + def test_nothing_the_dashboard_needs_is_missing(self, metrics): + m = metrics() + for field in ("cpu_percent", "cpu_temp", "memory_used_percent", + "memory_available_mb", "disk_used_percent"): + assert field in m, f"{field} missing from collected metrics" + + def test_absent_psutil_still_yields_every_key(self, monkeypatch): + """The no-psutil fallback must not drop fields the UI reads.""" + import builtins + real_import = builtins.__import__ + + def _no_psutil(name, *args, **kwargs): + if name == "psutil": + raise ImportError("psutil missing") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _no_psutil) + m = collect_system_metrics() + for field in ("cpu_percent", "cpu_temp", "memory_used_percent", + "memory_available_mb", "disk_used_percent"): + assert field in m + assert m["disk_used_percent"] is None diff --git a/web_interface/app.py b/web_interface/app.py index 98bf0f71..13529a1d 100644 --- a/web_interface/app.py +++ b/web_interface/app.py @@ -32,6 +32,8 @@ _JOURNALCTL = shutil.which('journalctl') _SYSTEMCTL = shutil.which('systemctl') _VCGENCMD = shutil.which('vcgencmd') +from web_interface.system_metrics import collect_system_metrics + # Create Flask app app = Flask(__name__) app.secret_key = os.urandom(24) @@ -606,26 +608,12 @@ def system_status_generator(): """Generate system status updates""" while True: try: - # Try to import psutil for system stats - try: - import psutil - # interval=None is non-blocking; primed at module startup above - cpu_percent = round(psutil.cpu_percent(interval=None), 1) - memory = psutil.virtual_memory() - memory_used_percent = round(memory.percent, 1) - - # Try to get CPU temperature (Raspberry Pi specific) - cpu_temp = 0 - try: - with open('/sys/class/thermal/thermal_zone0/temp', 'r') as f: - cpu_temp = round(float(f.read()) / 1000.0, 1) - except (OSError, ValueError): - pass - - except ImportError: - cpu_percent = 0 - memory_used_percent = 0 - cpu_temp = 0 + metrics = collect_system_metrics() + cpu_percent = metrics['cpu_percent'] + memory_used_percent = metrics['memory_used_percent'] + memory_available_mb = metrics['memory_available_mb'] + disk_used_percent = metrics['disk_used_percent'] + cpu_temp = metrics['cpu_temp'] # Check if display service is running (cached to avoid per-client subprocess forks) now = time.time() @@ -646,8 +634,9 @@ def system_status_generator(): 'service_active': service_active, 'cpu_percent': cpu_percent, 'memory_used_percent': memory_used_percent, + 'memory_available_mb': memory_available_mb, 'cpu_temp': cpu_temp, - 'disk_used_percent': 0, + 'disk_used_percent': disk_used_percent, 'power': _get_power_status() } yield status diff --git a/web_interface/system_metrics.py b/web_interface/system_metrics.py new file mode 100644 index 00000000..723bd85f --- /dev/null +++ b/web_interface/system_metrics.py @@ -0,0 +1,62 @@ +"""System metrics for the status stream. + +Deliberately free of Flask and app imports: importing web_interface.app +constructs the Flask application and a CacheManager, and the latter claims the +cache directory with a cleanup thread. Reading a CPU percentage should not do +that, and neither should testing it. + +Every value is best-effort. A number that cannot be taken comes back as None so +the UI can render '--', because a confident wrong number is worse than a blank: +``disk_used_percent`` used to be hardcoded to 0, which read as "plenty of room" +on a card that was filling up. +""" + +from typing import Any, Dict, Optional + +_THERMAL_ZONE = '/sys/class/thermal/thermal_zone0/temp' + + +def _cpu_temp_c() -> Optional[float]: + """CPU temperature in degrees C, or None off a Raspberry Pi.""" + try: + with open(_THERMAL_ZONE, 'r') as f: + return round(float(f.read()) / 1000.0, 1) + except (OSError, ValueError): + return None + + +def collect_system_metrics() -> Dict[str, Any]: + """Collect the numbers that predict trouble on a small board. + + ``memory_available_mb`` is MemAvailable rather than a used percentage: it + accounts for reclaimable page cache, so it is what separates a board at 70% + "used" that is fine from one at 70% that is about to fail fork(). A 1GB Pi + can sit at either and only this number tells them apart. + """ + try: + import psutil + except ImportError: + return { + 'cpu_percent': 0, + 'memory_used_percent': 0, + 'memory_available_mb': None, + 'disk_used_percent': None, + 'cpu_temp': _cpu_temp_c() or 0, + } + + # interval=None is non-blocking; app startup primes psutil's internal state. + cpu_percent = round(psutil.cpu_percent(interval=None), 1) + memory = psutil.virtual_memory() + + try: + disk_used_percent: Optional[float] = round(psutil.disk_usage('/').percent, 1) + except OSError: + disk_used_percent = None + + return { + 'cpu_percent': cpu_percent, + 'memory_used_percent': round(memory.percent, 1), + 'memory_available_mb': round(memory.available / (1024 * 1024), 1), + 'disk_used_percent': disk_used_percent, + 'cpu_temp': _cpu_temp_c() or 0, + } From 577f5501a698933e402cbe3c65f7b8ba8ee1bcae Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:42:49 -0400 Subject: [PATCH 08/16] perf(plugins): stop re-deriving a display() signature the caller already cached (#549) display_controller resolves once, and caches, whether a plugin's display() takes a display_mode keyword -- self._plugin_accepts_display_mode, populated right before the dispatch. It then handed the executor a types.SimpleNamespace wrapping a closure, and execute_display() ran inspect.signature() on that to work out the same thing. Because the SimpleNamespace is rebuilt per call, the callable was new every time, so nothing inside the executor could ever cache it either. Measured at ~39us per dispatch on a Pi 4, for a value the caller had a line earlier. execute_display() now takes accepts_display_mode, falling back to inspecting only when a caller does not pass it, so existing callers are unaffected. Also documents two things that read as bugs and are not: - execute_with_timeout()'s timeout is advisory. Nothing cancels the thread -- Python cannot -- so on expiry the operation runs to completion in the background and only the caller gives up. A permanently hung plugin leaks a daemon thread per attempt. This is why callers holding a lock across the call must release it from inside the wrapped callable, as run()'s _release_display_lock already does. - Only the first display() of each mode goes through the executor; the per-frame loops call display() directly. That is deliberate: a thread per frame would cost more than an advisory timeout buys. Both loops now say so, so the asymmetry does not read as an oversight. Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 Co-authored-by: Claude Opus 5 (1M context) --- src/display_controller.py | 27 +++++++++++++++++++++++- src/plugin_system/plugin_executor.py | 31 +++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/display_controller.py b/src/display_controller.py index 8a902570..44875f63 100644 --- a/src/display_controller.py +++ b/src/display_controller.py @@ -2169,7 +2169,14 @@ class DisplayController: types.SimpleNamespace(display=_display_target), plugin_id, force_clear=self.force_change, - display_mode=active_mode if _accepts_display_mode else None + display_mode=active_mode if _accepts_display_mode else None, + # Already resolved and cached above. + # Without this the executor re-derives + # it with inspect.signature() against + # the SimpleNamespace built two lines + # up -- a fresh callable every call, so + # nothing there can ever cache. + accepts_display_mode=_accepts_display_mode ) except Exception: # pragma: no cover - defensive; # execute_display catches everything @@ -2492,6 +2499,15 @@ class DisplayController: 1.0 / display_interval ) + # Deliberate: frames after the first call + # display() directly rather than through + # PluginExecutor. The executor spawns a thread per + # call, which at this loop's frame rate would cost + # more than the advisory timeout it buys -- and + # that timeout cannot cancel a hung plugin anyway + # (see execute_with_timeout). The first dispatch + # above still goes through it, so load-time + # failures are still caught and recorded. while True: _frame_start = time.perf_counter() try: @@ -2564,6 +2580,15 @@ class DisplayController: display_interval ) + # Deliberate: frames after the first call + # display() directly rather than through + # PluginExecutor. The executor spawns a thread per + # call, which at this loop's frame rate would cost + # more than the advisory timeout it buys -- and + # that timeout cannot cancel a hung plugin anyway + # (see execute_with_timeout). The first dispatch + # above still goes through it, so load-time + # failures are still caught and recorded. while True: time.sleep(display_interval) self._tick_plugin_updates() diff --git a/src/plugin_system/plugin_executor.py b/src/plugin_system/plugin_executor.py index 2fac5c5e..c1a23b49 100644 --- a/src/plugin_system/plugin_executor.py +++ b/src/plugin_system/plugin_executor.py @@ -76,6 +76,13 @@ class PluginExecutor: thread.start() thread.join(timeout=timeout) + # NB: this timeout is advisory. Nothing cancels the thread -- Python + # has no way to -- so on expiry the operation keeps running to + # completion in the background and only this caller gives up waiting. + # A plugin that hangs permanently leaks one daemon thread per attempt. + # Callers that hold a resource across the call must release it from + # inside the wrapped callable rather than after this returns; see the + # _release_display_lock guard inside DisplayController.run(). if not result_container['completed']: error_msg = f"{plugin_context} operation timed out after {timeout}s" self.logger.error(error_msg) @@ -148,7 +155,8 @@ class PluginExecutor: plugin_id: str, force_clear: bool = False, display_mode: Optional[str] = None, - timeout: Optional[float] = None + timeout: Optional[float] = None, + accepts_display_mode: Optional[bool] = None ) -> bool: """ Execute plugin display() method with error handling. @@ -159,6 +167,9 @@ class PluginExecutor: force_clear: Whether to force clear display display_mode: Optional display mode parameter timeout: Timeout in seconds (None = use default) + accepts_display_mode: Whether plugin.display() takes a + display_mode keyword. Pass it when the caller already knows; + None falls back to inspecting the callable. Returns: True if display succeeded, False otherwise @@ -166,10 +177,20 @@ class PluginExecutor: try: start_time = time.time() - # Check if plugin accepts display_mode parameter - import inspect - sig = inspect.signature(plugin.display) - has_display_mode = 'display_mode' in sig.parameters + # Does display() take a display_mode keyword? The caller usually + # knows and caches the answer, so prefer what it passed. + # + # Inspecting here was not merely redundant, it could never be + # cached: display_controller wraps the real plugin in a fresh + # SimpleNamespace per call, so inspect.signature() saw a new + # callable every time and paid ~55us on a Pi 4 to re-derive a + # value the caller had computed one line earlier and stored in + # self._plugin_accepts_display_mode. + if accepts_display_mode is None: + import inspect + accepts_display_mode = ( + 'display_mode' in inspect.signature(plugin.display).parameters) + has_display_mode = accepts_display_mode # Capture the return value from the plugin's display() method if has_display_mode and display_mode: From aba96e25b3a4a196ddeef7e8fa492f042192d575 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:45:07 -0400 Subject: [PATCH 09/16] chore: delete three functions nothing calls (#550) src/base_classes/baseball.py _get_baseball_display_text 45 lines src/web_interface/api_helpers.py validate_request_params 22 web_interface/blueprints/api_v3.py _validate_time_range 14 Each has exactly one occurrence across both repositories -- its own definition. No decorator, no __all__, no getattr dispatch, nothing in templates or JavaScript. A fourth candidate was dropped after checking: _unshare_element_fonts in src/common/sports_shared.py looked unreferenced, but eight scoreboard plugins call SportsCore._unshare_element_fonts directly from their test_element_text_colors.py, plus their own copies at runtime. It is live API. The earlier reading came from a plugins checkout 84 commits behind main, which is a good argument for re-verifying this kind of claim against a fresh tree rather than trusting an earlier scan. Full suite: 4,265 passed, 68 skipped. Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 Co-authored-by: Claude Opus 5 (1M context) --- src/base_classes/baseball.py | 45 ------------------------------ src/web_interface/api_helpers.py | 24 ---------------- web_interface/blueprints/api_v3.py | 14 ---------- 3 files changed, 83 deletions(-) diff --git a/src/base_classes/baseball.py b/src/base_classes/baseball.py index ac84bfb7..f36ca9db 100644 --- a/src/base_classes/baseball.py +++ b/src/base_classes/baseball.py @@ -37,51 +37,6 @@ class Baseball(SportsCore): self.data_source = ESPNDataSource(logger) self.sport = "baseball" - def _get_baseball_display_text(self, game: Dict) -> str: - """Get baseball-specific display text.""" - try: - display_parts = [] - - # Inning information - if self.show_innings: - inning = game.get("inning", "") - if inning: - display_parts.append(f"Inning: {inning}") - - # Outs information - if self.show_outs: - outs = game.get("outs", 0) - if outs is not None: - display_parts.append(f"Outs: {outs}") - - # Bases information - if self.show_bases: - bases = game.get("bases", "") - if bases: - display_parts.append(f"Bases: {bases}") - - # Count information - if self.show_count: - strikes = game.get("strikes", 0) - balls = game.get("balls", 0) - if strikes is not None and balls is not None: - display_parts.append(f"Count: {balls}-{strikes}") - - # Pitcher/Batter information - if self.show_pitcher_batter: - pitcher = game.get("pitcher", "") - batter = game.get("batter", "") - if pitcher: - display_parts.append(f"Pitcher: {pitcher}") - if batter: - display_parts.append(f"Batter: {batter}") - - return " | ".join(display_parts) if display_parts else "" - - except Exception as e: - self.logger.error(f"Error getting baseball display text: {e}") - return "" - def _is_baseball_game_live(self, game: Dict) -> bool: """Check if a baseball game is currently live.""" try: diff --git a/src/web_interface/api_helpers.py b/src/web_interface/api_helpers.py index 3cff5293..be2b0771 100644 --- a/src/web_interface/api_helpers.py +++ b/src/web_interface/api_helpers.py @@ -105,27 +105,3 @@ def validate_request_json(required_fields: list, data: Optional[Dict] = None) -> ) return data, None - - -def validate_request_params(required_params: list) -> Tuple[Optional[Dict], Optional[Any]]: - """ - Validate request has required query parameters. - - Args: - required_params: List of required parameter names - - Returns: - Tuple of (params_dict, error_response) or (params_dict, None) if valid - """ - missing_params = [param for param in required_params if param not in request.args] - if missing_params: - return None, error_response( - ErrorCode.INVALID_INPUT, - f"Missing required parameters: {', '.join(missing_params)}", - context={'missing_params': missing_params}, - status_code=400 - ) - - params = {param: request.args.get(param) for param in required_params} - return params, None - diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index 865789f0..d5aeae3c 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -352,20 +352,6 @@ def _validate_time_format(time_str): except (ValueError, TypeError): return False, f"Invalid time format: {time_str}. Expected HH:MM format." -def _validate_time_range(start_time_str, end_time_str, allow_overnight=True): - """Validate time range. Returns (is_valid, error_message)""" - try: - start_time = datetime.strptime(start_time_str, '%H:%M').time() - end_time = datetime.strptime(end_time_str, '%H:%M').time() - - # Allow overnight schedules (start > end) or same-day schedules - if not allow_overnight and start_time >= end_time: - return False, f"Start time ({start_time_str}) must be before end time ({end_time_str}) for same-day schedules" - - return True, None - except (ValueError, TypeError) as e: - return False, f"Invalid time format: {str(e)}" - @api_v3.route('/config/schedule', methods=['POST']) def save_schedule_config(): """Save schedule configuration""" From 39f27d285da25d32b47f4ed4c48e30072f393ab8 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:45:21 -0400 Subject: [PATCH 10/16] fix(plugins): stop reconciliation inventing plugins and telling users to delete real config (#557) On a device running four installed, configured, working plugins, the overview banner read: Stale plugin config entries found: football-scoreboard, odds-ticker, data, ledmatrix-weather, starlark-apps. Remove them from config.json or reinstall via the Plugin Store. Every claim in that sentence was wrong, and following its advice would have deleted 4.9KB of working league settings. Four separate defects combined. 1. Secrets keys became phantom plugins. load_config() merges config_secrets.json into the config it returns, and the ignore list named only 'github' and 'youtube'. A 'data' key in that file therefore read as a plugin id and was reported as "in config but not on disk" forever. Read the secrets file's own top-level keys instead of hardcoding two of them. 2. The auto-fix clobbered real config. The handler for "on disk but not in config" assigned `config[plugin_id] = {'enabled': False}` unconditionally, so whenever detection was wrong it replaced a plugin's entire configuration with a stub. On the reported device it only failed to do so because the write hit EACCES. Now it refuses to overwrite an entry that already exists. 3. The banner gave backwards advice. plugin_missing_in_config ("on disk, not in config") and plugin_missing_on_disk ("in config, not on disk") are opposite problems, and both were rendered as "stale config entries ... remove them from config.json" -- which is correct for the second and destructive for the first. They are now reported separately, each with the advice that fits. 4. A stale verdict was served indefinitely. The result is a snapshot written once per run to a status file, and a run that fails to apply a fix also declares it will not retry. A condition that had since resolved kept being reported for hours. The status endpoint now re-checks stored findings against current state, dropping only what it can prove stale and keeping any kind it cannot re-verify. The secrets-key lookup is deliberately fail-safe: an unreadable, absent, malformed or non-path secrets location narrows the ignore set rather than raising. An earlier revision let TypeError escape, which the broad handler in _get_config_state() swallowed as "Error reading config state" -- emptying the config state and making every downstream detection wrong. The existing reconciliation tests caught it; there is now a regression test for it too. Co-authored-by: Claude Opus 5 --- src/plugin_system/state_reconciliation.py | 66 ++++++- test/test_reconciliation_phantom_plugins.py | 170 ++++++++++++++++++ web_interface/blueprints/api_v3.py | 26 +++ .../templates/v3/partials/overview.html | 31 +++- 4 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 test/test_reconciliation_phantom_plugins.py diff --git a/src/plugin_system/state_reconciliation.py b/src/plugin_system/state_reconciliation.py index afbd2e1d..e967fbdf 100644 --- a/src/plugin_system/state_reconciliation.py +++ b/src/plugin_system/state_reconciliation.py @@ -8,6 +8,7 @@ Detects and fixes inconsistencies between: - State manager state """ +import json from typing import Dict, Any, List, Set from dataclasses import dataclass from enum import Enum @@ -55,6 +56,35 @@ class ReconciliationResult: message: str +def still_unresolved(entries: List[Dict[str, Any]], + config_keys: Set[str], + installed_ids: Set[str]) -> List[Dict[str, Any]]: + """Drop stored reconciliation findings that are no longer true. + + The verdict is written once to a status file and served to the web UI from + there, and a run that fails to apply a fix also declares it "will not retry + automatically". Together those froze a single moment forever: a device whose + plugins were all present in config kept being told, for hours, that four of + them were missing and should be removed from config.json. + + Entry kinds this cannot re-check are kept, so filtering only ever removes + findings that are provably stale. + """ + live: List[Dict[str, Any]] = [] + for entry in entries: + kind = entry.get('type') + plugin_id = entry.get('plugin_id') + if kind == InconsistencyType.PLUGIN_MISSING_IN_CONFIG.value: + if plugin_id not in config_keys: + live.append(entry) + elif kind == InconsistencyType.PLUGIN_MISSING_ON_DISK.value: + if plugin_id not in installed_ids: + live.append(entry) + else: + live.append(entry) + return live + + class StateReconciliation: """ State reconciliation system. @@ -200,15 +230,37 @@ class StateReconciliation: 'github', 'youtube', }) + def _secrets_top_level_keys(self) -> Set[str]: + """Top-level keys that load_config() merges in from the secrets file. + + load_config() merges config_secrets.json into the config it returns, so + those keys sit alongside plugin ids. _SYSTEM_CONFIG_KEYS named them + individually ('github', 'youtube'), which broke the moment anything else + was written there: a 'data' key became a phantom plugin, permanently + reported as "in config but not on disk". Reading the file keeps this + correct no matter what it holds. + """ + try: + path = self.config_manager.get_secrets_path() + with open(path, 'r') as f: + secrets = json.load(f) + except (AttributeError, OSError, TypeError, ValueError): + # Deliberately broad: an unreadable, absent, malformed or + # non-path secrets location must narrow this set, never break + # reconciliation. ValueError covers json.JSONDecodeError. + return set() + return set(secrets) if isinstance(secrets, dict) else set() + def _get_config_state(self) -> Dict[str, Dict[str, Any]]: """Get plugin state from config file.""" state = {} try: config = self.config_manager.load_config() + ignored = self._SYSTEM_CONFIG_KEYS | self._secrets_top_level_keys() for plugin_id, plugin_config in config.items(): if not isinstance(plugin_config, dict): continue - if plugin_id in self._SYSTEM_CONFIG_KEYS: + if plugin_id in ignored: continue state[plugin_id] = { 'enabled': plugin_config.get('enabled', True), @@ -367,8 +419,18 @@ class StateReconciliation: """Attempt to fix an inconsistency.""" try: if inconsistency.inconsistency_type == InconsistencyType.PLUGIN_MISSING_IN_CONFIG: - # Add plugin to config with default disabled state config = self.config_manager.load_config() + if inconsistency.plugin_id in config: + # Detection said "not in config" but it is there -- the + # config changed under us, or the id came from a key merged + # in from elsewhere. Assigning the stub below would replace + # the real entry: one reported case would have traded 4.9KB + # of league settings for {'enabled': False}. Nothing to fix. + self.logger.info( + "Skipped: %s is already in config; not overwriting it", + inconsistency.plugin_id) + return True + # Add plugin to config with default disabled state config[inconsistency.plugin_id] = { 'enabled': False } diff --git a/test/test_reconciliation_phantom_plugins.py b/test/test_reconciliation_phantom_plugins.py new file mode 100644 index 00000000..d05e80fa --- /dev/null +++ b/test/test_reconciliation_phantom_plugins.py @@ -0,0 +1,170 @@ +"""Tests for plugin-state reconciliation reporting phantom inconsistencies. + +Observed on a device running four installed, configured, working plugins: the +web UI showed "Stale plugin config entries found: football-scoreboard, +odds-ticker, data, ledmatrix-weather, starlark-apps. Remove them from +config.json" for hours. Three separate defects combined to produce that: + +1. load_config() merges config_secrets.json into the config it returns, and the + ignore list named only 'github' and 'youtube'. A 'data' key in the secrets + file therefore became a phantom plugin, reported as in-config-not-on-disk. + +2. The auto-fix for "on disk but not in config" assigned + ``config[plugin_id] = {'enabled': False}`` unconditionally. When detection was + wrong, that traded a plugin's real configuration (4.9KB of league settings in + the reported case) for a stub. It only avoided firing there because the write + failed with EACCES. + +3. The verdict is a snapshot written once per run and served to the UI from a + status file, and a run that fails to apply a fix also declines to retry. So a + condition that had since resolved was reported indefinitely. +""" + +import json +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from src.plugin_system.state_reconciliation import ( + InconsistencyType, + StateReconciliation, + still_unresolved, +) + + +class _ConfigManager: + """Minimal config manager with the secrets path the real one exposes.""" + + def __init__(self, config, secrets_path=None): + self._config = config + self._secrets_path = secrets_path + self.saved = [] + + def load_config(self): + return dict(self._config) + + def save_config(self, config): + self.saved.append(config) + self._config = dict(config) + + def get_secrets_path(self): + return self._secrets_path + + +def _reconciler(config, plugins_dir, secrets_path=None): + return StateReconciliation( + state_manager=Mock(), + config_manager=_ConfigManager(config, secrets_path), + plugin_manager=Mock(), + plugins_dir=plugins_dir, + ) + + +class TestSecretsKeysAreNotPlugins: + def test_secrets_key_is_not_treated_as_a_plugin(self, tmp_path): + secrets = tmp_path / "config_secrets.json" + # The shape actually found on the device: display state written here. + secrets.write_text(json.dumps({"data": {"mode": "nfl_recent"}, + "timestamp": 1.0}), encoding="utf-8") + config = { + "data": {"mode": "nfl_recent"}, + "football-scoreboard": {"enabled": True, "nfl": {"favorite_teams": ["TB"]}}, + } + state = _reconciler(config, tmp_path, secrets).\ + _get_config_state() + + assert "data" not in state, "secrets key leaked in as a phantom plugin" + assert "football-scoreboard" in state + + def test_missing_secrets_file_is_survivable(self, tmp_path): + state = _reconciler({"odds-ticker": {"enabled": True}}, tmp_path, + tmp_path / "does-not-exist.json")._get_config_state() + assert "odds-ticker" in state + + def test_non_path_secrets_location_is_survivable(self, tmp_path): + """Regression: a non-path return raised TypeError out of the helper, + which the broad handler in _get_config_state() swallowed as "Error + reading config state" -- emptying the config state and making every + downstream detection wrong.""" + r = _reconciler({"odds-ticker": {"enabled": True}}, tmp_path, + secrets_path=Mock()) + assert "odds-ticker" in r._get_config_state() + + def test_config_manager_without_secrets_path_is_survivable(self, tmp_path): + r = StateReconciliation(state_manager=Mock(), + config_manager=Mock(spec=["load_config", "save_config"]), + plugin_manager=Mock(), plugins_dir=tmp_path) + r.config_manager.load_config.return_value = {"odds-ticker": {"enabled": True}} + assert "odds-ticker" in r._get_config_state() + + +class TestFixNeverClobbersRealConfig: + def _missing_in_config(self, plugin_id): + inc = Mock() + inc.inconsistency_type = InconsistencyType.PLUGIN_MISSING_IN_CONFIG + inc.plugin_id = plugin_id + return inc + + def test_existing_entry_is_not_overwritten(self, tmp_path): + rich = {"enabled": True, "nfl": {"favorite_teams": ["TB"], "show_odds": True}} + r = _reconciler({"football-scoreboard": dict(rich)}, tmp_path) + + assert r._fix_inconsistency(self._missing_in_config("football-scoreboard")) is True + # The regression: this wrote {'enabled': False} over the real settings. + assert r.config_manager.saved == [] + assert r.config_manager.load_config()["football-scoreboard"] == rich + + def test_genuinely_absent_plugin_is_still_added(self, tmp_path): + r = _reconciler({"other-plugin": {"enabled": True}}, tmp_path) + + assert r._fix_inconsistency(self._missing_in_config("new-plugin")) is True + assert len(r.config_manager.saved) == 1 + assert r.config_manager.load_config()["new-plugin"] == {"enabled": False} + # and it must not disturb what was already there + assert r.config_manager.load_config()["other-plugin"] == {"enabled": True} + + +class TestStaleFindingsAreDropped: + IN_CONFIG = InconsistencyType.PLUGIN_MISSING_IN_CONFIG.value + ON_DISK = InconsistencyType.PLUGIN_MISSING_ON_DISK.value + + def test_resolved_missing_in_config_is_dropped(self): + entries = [{"plugin_id": "football-scoreboard", "type": self.IN_CONFIG}] + assert still_unresolved(entries, {"football-scoreboard"}, set()) == [] + + def test_genuinely_missing_in_config_is_kept(self): + entries = [{"plugin_id": "football-scoreboard", "type": self.IN_CONFIG}] + assert still_unresolved(entries, set(), set()) == entries + + def test_resolved_missing_on_disk_is_dropped(self): + entries = [{"plugin_id": "odds-ticker", "type": self.ON_DISK}] + assert still_unresolved(entries, set(), {"odds-ticker"}) == [] + + def test_genuinely_missing_on_disk_is_kept(self): + entries = [{"plugin_id": "data", "type": self.ON_DISK}] + assert still_unresolved(entries, set(), {"odds-ticker"}) == entries + + def test_unrecheckable_kinds_are_kept(self): + # Filtering must only ever remove what it can prove stale. + entries = [{"plugin_id": "x", "type": "plugin_version_mismatch"}] + assert still_unresolved(entries, set(), set()) == entries + + def test_the_reported_device_state_clears_every_false_finding(self): + """The exact verdict the device served, against its real state.""" + entries = [ + {"plugin_id": "football-scoreboard", "type": self.IN_CONFIG}, + {"plugin_id": "odds-ticker", "type": self.IN_CONFIG}, + {"plugin_id": "data", "type": self.ON_DISK}, + {"plugin_id": "ledmatrix-weather", "type": self.IN_CONFIG}, + {"plugin_id": "starlark-apps", "type": self.IN_CONFIG}, + ] + installed = {"football-scoreboard", "ledmatrix-weather", "odds-ticker", + "starlark-apps", "web-ui-info"} + live = still_unresolved(entries, installed, installed) + + # All four "installed but not in config" findings were false and clear. + # 'data' legitimately is not installed, so this filter keeps it; it stops + # being reported because _get_config_state() no longer invents it from the + # secrets file, which takes effect on the next reconciliation run. + assert [e["plugin_id"] for e in live] == ["data"] diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index d5aeae3c..a42d712f 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -3464,6 +3464,30 @@ def reconcile_plugin_state(): status_code=500 ) +def _drop_stale_reconciliation_findings(unresolved): + """Re-check a stored reconciliation verdict against current state. + + The verdict is a snapshot written once per run, and a run that could not + apply a fix also refuses to retry -- so a resolved condition was reported + indefinitely. Best-effort: any failure here returns the list untouched, + because showing a stale warning beats failing the endpoint. + """ + try: + from src.plugin_system.state_reconciliation import still_unresolved + + config_keys = set(api_v3.config_manager.load_config() or {}) + installed = set() + plugins_dir = getattr(api_v3.plugin_manager, 'plugins_dir', None) + if plugins_dir: + for entry in Path(plugins_dir).iterdir(): + if entry.is_dir() and (entry / 'manifest.json').exists(): + installed.add(entry.name) + return still_unresolved(unresolved, config_keys, installed) + except Exception: + logger.debug("[Reconciliation] Could not re-check stored findings", exc_info=True) + return unresolved + + @api_v3.route('/plugins/reconciliation-status', methods=['GET']) def get_reconciliation_status(): """Return the result of the last startup reconciliation from /tmp status file.""" @@ -3478,6 +3502,8 @@ def get_reconciliation_status(): try: with open(_recon_path) as _f: data = json.load(_f) + if data.get('unresolved'): + data['unresolved'] = _drop_stale_reconciliation_findings(data['unresolved']) return jsonify({'status': 'success', 'data': data}) except json.JSONDecodeError: logger.exception("[Reconciliation] Failed to parse status file: %s", _recon_path) diff --git a/web_interface/templates/v3/partials/overview.html b/web_interface/templates/v3/partials/overview.html index e351c90f..340fd35d 100644 --- a/web_interface/templates/v3/partials/overview.html +++ b/web_interface/templates/v3/partials/overview.html @@ -30,10 +30,33 @@ if (!d.unresolved || d.unresolved.length === 0) return; var key = d.unresolved.map(function (i) { return i.plugin_id; }).sort().join(','); if (sessionStorage.getItem(DISMISS_KEY) === key) return; - var ids = d.unresolved.map(function (i) { return i.plugin_id; }).join(', '); - document.getElementById('reconciliation-banner-text').textContent = - 'Stale plugin config entries found: ' + ids + - '. Remove them from config.json or reinstall via the Plugin Store.'; + // These are opposite problems and need opposite advice. Lumping + // them together told users to delete config.json entries for + // plugins that were installed and correctly configured. + function idsOfType(t) { + return d.unresolved.filter(function (i) { return i.type === t; }) + .map(function (i) { return i.plugin_id; }); + } + var notConfigured = idsOfType('plugin_missing_in_config'); + var notInstalled = idsOfType('plugin_missing_on_disk'); + var other = d.unresolved.filter(function (i) { + return i.type !== 'plugin_missing_in_config' + && i.type !== 'plugin_missing_on_disk'; + }).map(function (i) { return i.plugin_id; }); + + var parts = []; + if (notConfigured.length) { + parts.push('Installed but missing from config: ' + notConfigured.join(', ') + + '. Configure them in the Plugin Store - do not remove anything from config.json.'); + } + if (notInstalled.length) { + parts.push('In config but not installed: ' + notInstalled.join(', ') + + '. Reinstall via the Plugin Store, or remove these entries from config.json.'); + } + if (other.length) { + parts.push('Needs attention: ' + other.join(', ') + '.'); + } + document.getElementById('reconciliation-banner-text').textContent = parts.join(' '); var banner = document.getElementById('reconciliation-banner'); banner.dataset.dismissKey = key; banner.style.setProperty('display', 'flex', 'important'); From 0ab95586fb89c5a57f61d2365c914bbc43012943 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 08:45:40 -0400 Subject: [PATCH 11/16] fix(web): say when a system action failed for want of passwordless sudo (#560) * fix(web): say when a system action failed for want of passwordless sudo POSTing reboot_system to a Pi returns, in full: {"message": "Action failed; see logs for details", "status": "error"} The cause is that the web interface runs unprivileged, and its systemctl/reboot/journalctl calls only work once scripts/install/configure_web_sudo.sh has granted NOPASSWD. first_time_install.sh never invokes that script and no user-facing doc mentions it, so on a fresh device every privileged action fails -- start_display, stop_display, the autostart toggles, reboot, and the log viewer. That last one closes the loop: "see logs for details" is unreachable advice when journalctl is refused for the same reason. This is exactly the failure src/web_interface/error_handler.py's describe_exception() was written to break, and /system/action's exception handler was still discarding the cause instead of using the helper the module already imports. Two changes, no behaviour change when things work: - The exception path now returns 'details': describe_exception(e), matching how the other handlers in this blueprint already report. - A failure whose stderr or exception text is sudo refusing to prompt ("a password is required", "no tty present", "a terminal is required") reports what to do about it, naming configure_web_sudo.sh. Unrelated failures keep the generic message and their stderr, so a missing unit is not blamed on sudo. Granting the sudo rights is left alone deliberately: auto-running a script that hands out NOPASSWD is a security decision for the maintainer, not something to slip into an installer. Making the refusal legible is the part that is unambiguously an improvement. Co-Authored-By: Claude Opus 5 * fix(web): apply the sudo hint on the on-demand start_display path too start_display with a mode builds its own response and returns before the shared nonzero-result path, so a recognized sudo refusal there reported only "Failed to start display" and said nothing about the passwordless sudo that refused it -- the exact gap the rest of this PR closes everywhere else. Raised by CodeRabbit on #560 and verified against the code before fixing: the branch at api_v3.py:2058 does return early past the shared handler. Three regression cases: the on-demand branch reports the sudo cause, keeps its "Display started" message on success, and does not blame an unrelated failure on sudo. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- test/test_system_action_sudo_diagnosis.py | 131 ++++++++++++++++++++++ web_interface/blueprints/api_v3.py | 47 +++++++- 2 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 test/test_system_action_sudo_diagnosis.py diff --git a/test/test_system_action_sudo_diagnosis.py b/test/test_system_action_sudo_diagnosis.py new file mode 100644 index 00000000..88adca02 --- /dev/null +++ b/test/test_system_action_sudo_diagnosis.py @@ -0,0 +1,131 @@ +"""A privileged system action that cannot run must say why. + +The web interface runs unprivileged. Its systemctl/reboot/journalctl calls only +work once scripts/install/configure_web_sudo.sh has granted NOPASSWD, and +first_time_install.sh never invokes that script -- so on a fresh device every +one of those actions fails. What the user got back was: + + {"message": "Action failed; see logs for details", "status": "error"} + +...and the log viewer was broken for exactly the same reason, so "see logs" led +nowhere. This is the closed loop that src/web_interface/error_handler.py's +describe_exception() was written to break; /system/action's exception handler +was simply still discarding the cause. + +Observed live: POSTing reboot_system to a Pi returned that bare sentence, and +pinning down "no passwordless sudo" took reading the installer instead of +reading the response. +""" + +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest +from flask import Flask + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from web_interface.blueprints.api_v3 import api_v3 # noqa: E402 + + +@pytest.fixture +def client(): + app = Flask(__name__) + app.config['TESTING'] = True + app.register_blueprint(api_v3, url_prefix='/api/v3') + return app.test_client() + + +def _act(client, action='stop_display'): + return client.post('/api/v3/system/action', json={'action': action}) + + +def _result(args, rc=0, stderr=''): + return subprocess.CompletedProcess(args, rc, stdout='', stderr=stderr) + + +class TestSudoRefusalIsNamed: + @pytest.mark.parametrize("stderr", [ + "sudo: a password is required", + "sudo: no tty present and no askpass program specified", + "sudo: a terminal is required to read the password", + ]) + def test_every_sudo_refusal_wording_is_recognised(self, client, stderr): + with patch('subprocess.run', side_effect=lambda a, **k: _result(a, 1, stderr)): + body = _act(client).get_json() + + assert body['status'] == 'error' + # The point: the response tells the user what to do about it. + assert 'configure_web_sudo.sh' in body['message'] + + def test_the_raw_stderr_is_still_returned(self, client): + with patch('subprocess.run', + side_effect=lambda a, **k: _result(a, 1, "sudo: a password is required")): + body = _act(client).get_json() + assert 'a password is required' in body['stderr'] + + +class TestUnrelatedFailuresAreNotMisattributed: + def test_a_real_systemctl_error_keeps_the_generic_message(self, client): + with patch('subprocess.run', + side_effect=lambda a, **k: _result(a, 5, "Unit ledmatrix.service not found.")): + body = _act(client).get_json() + + assert 'configure_web_sudo.sh' not in body['message'], \ + "a missing unit was blamed on sudo" + assert 'Unit ledmatrix.service not found.' in body['stderr'] + assert body['returncode'] == 5 + + +class TestTheExceptionPathNamesTheCause: + def test_the_exception_type_and_message_are_returned(self, client): + with patch('subprocess.run', side_effect=FileNotFoundError(2, "sudo")): + body = _act(client).get_json() + + # The regression: 'details' was absent and the cause was discarded. + assert 'FileNotFoundError' in body['details'] + + def test_a_sudo_shaped_exception_also_gets_the_hint(self, client): + with patch('subprocess.run', + side_effect=PermissionError("sudo: a password is required")): + body = _act(client).get_json() + assert 'configure_web_sudo.sh' in body['message'] + + +class TestTheEarlyReturnPathsAlsoGetTheHint: + """start_display with a mode builds its own response and returns before the + shared nonzero-result path, so it needed the hint separately. Found by + CodeRabbit on #560; verified against the code before fixing.""" + + def test_on_demand_start_reports_the_sudo_cause(self, client): + with patch('subprocess.run', + side_effect=lambda a, **k: _result(a, 1, "sudo: a password is required")): + body = client.post('/api/v3/system/action', + json={'action': 'start_display', 'mode': 'nfl_live'}).get_json() + + assert 'configure_web_sudo.sh' in body['message'], "the on-demand branch still reported only 'Failed to start display'" + assert body['status'] == 'error' + + def test_on_demand_start_keeps_its_success_message(self, client): + with patch('subprocess.run', side_effect=lambda a, **k: _result(a, 0)): + body = client.post('/api/v3/system/action', + json={'action': 'start_display', 'mode': 'nfl_live'}).get_json() + assert body['status'] == 'success' + assert body['message'] == 'Display started' + + def test_an_unrelated_on_demand_failure_is_not_blamed_on_sudo(self, client): + with patch('subprocess.run', + side_effect=lambda a, **k: _result(a, 5, "Unit not found.")): + body = client.post('/api/v3/system/action', + json={'action': 'start_display', 'mode': 'nfl_live'}).get_json() + assert body['message'] == 'Failed to start display' + + +class TestSuccessIsUnchanged: + def test_a_working_action_still_reports_success(self, client): + with patch('subprocess.run', side_effect=lambda a, **k: _result(a, 0)): + body = _act(client).get_json() + assert body['status'] == 'success' + assert 'configure_web_sudo.sh' not in body.get('message', '') diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index a42d712f..10711053 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -1979,6 +1979,33 @@ def check_for_update(): return jsonify(_update_check_failed( "Could not check for updates; see logs for details.")) +#: sudo's own wording when it needs a password it cannot ask for. The web +#: interface runs unprivileged, so its systemctl/reboot/journalctl calls only +#: work once scripts/install/configure_web_sudo.sh has granted NOPASSWD -- +#: which first_time_install.sh does not do. That makes this the common case on +#: a fresh device, and "Action failed; see logs for details" named none of it, +#: while the log viewer was broken for the very same reason. +_SUDO_NEEDS_PASSWORD = ( + 'a password is required', + 'no tty present', + 'a terminal is required', +) + +_SUDO_HINT = ( + 'Passwordless sudo is not configured for the web interface user, so this ' + 'action cannot run. Run scripts/install/configure_web_sudo.sh as that user, ' + 'then retry.' +) + + +def _sudo_hint_for(text): + """An actionable hint when `text` is sudo refusing to prompt, else None.""" + lowered = (text or '').lower() + if any(marker in lowered for marker in _SUDO_NEEDS_PASSWORD): + return _SUDO_HINT + return None + + @api_v3.route('/system/action', methods=['POST']) def execute_system_action(): """Execute system actions (start/stop/reboot/etc)""" @@ -2014,7 +2041,14 @@ def execute_system_action(): logger.error("start_display (%s) stderr: %s", mode, result.stderr.strip()) resp = { 'status': 'success' if result.returncode == 0 else 'error', - 'message': 'Display started' if result.returncode == 0 else 'Failed to start display', + # This branch returns before the shared nonzero-result + # response below, so it needs the hint of its own or an + # on-demand start reports "Failed to start display" and + # says nothing about the sudo that actually refused it. + 'message': ( + 'Display started' if result.returncode == 0 + else _sudo_hint_for(result.stderr) or 'Failed to start display' + ), } if result.returncode != 0: resp['returncode'] = result.returncode @@ -2346,6 +2380,9 @@ def execute_system_action(): if result.returncode != 0: resp['returncode'] = result.returncode resp['stderr'] = result.stderr.strip() + hint = _sudo_hint_for(result.stderr) + if hint: + resp['message'] = hint return jsonify(resp) except subprocess.TimeoutExpired as e: @@ -2353,7 +2390,13 @@ def execute_system_action(): return jsonify({'status': 'error', 'message': 'Command timed out', 'returncode': -1, 'stderr': 'timeout'}) except Exception as e: logger.error("execute_system_action failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'Action failed; see logs for details'}), 500 + detail = describe_exception(e) + resp = { + 'status': 'error', + 'message': _sudo_hint_for(detail) or 'Action failed; see logs for details', + 'details': detail, + } + return jsonify(resp), 500 @api_v3.route('/system/git-info', methods=['GET']) def get_git_info(): From bdb9a94033f7cc05fd8d770a9cfc805d7f344364 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:07:38 -0400 Subject: [PATCH 12/16] refactor(api-v3): split the 10,469-line blueprint into a package (#553) * refactor(api-v3): split the 10,469-line blueprint into a package web_interface/blueprints/api_v3.py held 111 routes, 56 helpers and 181 functions in one module -- 9% of the core by line count and three times the next largest file. It becomes a package of nine route modules grouped by path segment, plus __init__.py for the shared imports, constants, Blueprint and helpers. Every route module decorates the SAME api_v3 Blueprint object, so endpoint names stay api_v3., the URL map is unchanged and app.py is untouched. Verified: 111 routes before, 111 after, byte-identical rules, endpoints and methods, and every endpoint still on the one blueprint. plugins 3,867 config 1,178 starlark 692 system 619 fonts 452 misc 398 wifi 361 display 326 backup 212 __init__ 1,787 (imports, constants, Blueprint, 56 helpers) Two things the URL-map check could not catch, both found by running the suite: 1. PROJECT_ROOT = Path(__file__).parent.parent.parent. Moving the code one directory deeper made that resolve to web_interface/ instead of the project root. Nothing failed at import; it surfaced as ~110 tests failing with 404s and "installation script not found", because every path built from it was one level too shallow. Now parents[3], and test_api_v3_url_map.py asserts PROJECT_ROOT/run.py exists so the next move cannot repeat it. 2. Module-attribute patching. Tests do monkeypatch.setattr(api_v3_module, "_BACKUP_EXPORT_DIR", ...) and a route module that binds such a name by value never sees the patch. The shared code therefore stays in __init__.py rather than moving to a _common submodule -- it has to live on the module the tests patch -- and the eleven names tests patch are read back through the package (_pkg.X) instead of bound by value. Those eleven were found by AST-scanning every setattr in the test tree, not by guessing; "time" is among them, used to drive a fake clock through the second-resolution credential-backup filenames. Test changes are confined to what genuinely moved: patch targets that now name the owning route module, imports of helpers, and six tests that scan the api_v3 source as a file and now read the package directory. Full suite: 4,278 passed, 68 skipped, 0 failed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 * fix(api-v3): address CodeRabbit findings from the blueprint-split review Fixes to the api_v3 package split (PR #553), one per finding verified against the actual code: - __init__.py: _redact_credentials only blanked scalar values under a credential-named key; a bare list of secrets under such a key (e.g. tokens: ["a", "b"]) passed through untouched, since the list branch recursed with no memory that its key looked like a credential. Nested dicts still walk normally (a documented, tested behaviour -- a container like secrets: {api_key: ..., note: ...} is a section name, not a value to blank outright), but any value reached under a credential-shaped key is now actually blanked. - __init__.py: the OAuth helper script's raw stderr/stdout went to logger.error unredacted (CWE-532) right next to a comment claiming this was deliberate; the HTTP response already used the existing redact_text helper. Routed the log line through the same helper. - __init__.py / starlark.py: the standalone Starlark manifest fallback (used when the plugin instance isn't loaded) read-modified-wrote manifest.json with no lock, unlike StarlarkAppsPlugin._update_manifest_safe (plugin-repos/starlark-apps/manager.py), which already holds an flock for the same file when the plugin is loaded. Added _starlark_manifest_lock, mirroring that pattern, and wrapped every standalone read-modify-write call site in it. The app-config update route also wrote config.json and the manifest as two separate, non-transactional writes (a second, distinct finding at the same call site); config.json is now rolled back if the manifest write that follows it fails. - backup.py: restore options used bare bool() on values from the request, so {"restore_secrets": "false"} restored secrets anyway (bool("false") is True). Switched to the existing _coerce_to_bool helper already used for this exact purpose elsewhere in the package. - config.py: an automated import-rewrite mangled four user-facing validation strings and their neighbouring comments -- "Invalid start time" had become "Invalid start _pkg.time" (and likewise for "end time") in both the schedule and dim-schedule per-day validation paths. - display.py: `import _pkg.time as time_module` -- _pkg is a local alias for the package, not a real importable module, so this raised ModuleNotFoundError whenever a caller restarted an already-running display service via /display/on-demand/start, after the on-demand request was already written to cache. Fixed to `import time`. Audited the rest of the package for the same `_pkg.` import mistake; every other `_pkg.` reference is a legitimate attribute read-through (`_pkg.time.time()`, `_pkg._get_starlark_plugin()`, ...), not a broken import statement. - fonts.py: validate_file_upload's max_size_mb parameter is silently unused by that helper (it only checks filename/extension) -- the font upload route saved arbitrarily large files as a result. Added the same seek-and-check pattern already used for the sibling .star upload. - wifi.py: two ad hoc, inconsistent bool coercions. POST /wifi/ap/auto-enable used bare bool(), so a JSON string "false" enabled it. POST /wifi/radio's enabled/force parsing recognized real bool and some strings but not int 1/0 (1 is True is False in Python). Factored one small _parse_bool_ish helper local to this file and used it at all three sites. Not changed: the "unknown/misspelled restore option keys default to True" half of the backup.py finding -- the file's own comment documents that a missing key deliberately means "restore everything," matching the already-existing JSON-parse-failure guard a few lines above it; only the bool-coercion defect was a real bug. Added or extended regression tests for every fix, following each area's existing test conventions. Full suite: 4328 passed, 62 skipped, 2 failed on both this branch and origin/main (missing tzdata package breaks two timezone-alias tests in test_onboarding_checklist.py, unrelated to this change) -- no new failures. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01S3bPMESe2TfrGvbs1ef9c5 * fix(api-v3): reject unknown restore option keys CodeRabbit's review of the blueprint split (#553) asked that POST /backup/restore reject option keys outside RestoreOptions' known set. The follow-up commit fixed the bool("false")-is-True bug with _coerce_to_bool but never added the key check: a typo'd or renamed key (e.g. "restoreSecrets") is silently ignored by opts_dict.get(key, True), so the flag stays at its True default and secrets get restored despite the caller's request saying otherwise -- with no indication anything was wrong. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Vmcwf5vMgYqdt8bJTZtiwb * fix(api-v3): address CodeRabbit findings on the blueprint split - _redact_credentials: blank scalar descendants of objects reached through a credential-owned list (e.g. tokens: [{"value": "secret"}]) regardless of field name -- the existing name-based walk only protected direct dict values under a credential key, not list items. - wifi.py: reject enabled/force/auto_enable_ap_mode values _parse_bool_ish can't recognize (400) instead of silently treating them as False, which could disable Wi-Fi or the radio itself. - Starlark manifest locking: lock a stable manifest.json.lock sidecar instead of manifest.json itself, in both the standalone route path (_starlark_manifest_lock) and the plugin path (StarlarkAppsPlugin._save_manifest / _update_manifest_safe). manifest.json is replaced by an atomic rename on every write, which swaps in a fresh inode; a lock held on the old inode does not exclude a second locker that opens the path afresh right after the rename and gets the new inode, so two writers could race despite each holding "a lock". A sidecar that no write ever touches always resolves to the same inode for every locker. Skipped as stale: the "serialize the complete manifest read-modify-write" finding at api_v3/__init__.py -- every standalone handler that calls _write_starlark_manifest is already wrapped in _starlark_manifest_lock() on this branch. Co-Authored-By: Claude Opus 5 * fix(api-v3): re-check reconciliation findings by the reconciler's own rules Both CodeRabbit findings on the merge commit, verified against the code first. Major, plugins.py: the stale-findings filter derived its own notion of "in config" and "on disk", and both were looser than the reconciliation module's. set(load_config()) also contains system keys, the secrets-file keys load_config() merges in, and non-dict values; and any directory holding a manifest.json counted as installed even when that manifest does not parse. Either looseness clears a finding that is still true -- and a secrets key read as a plugin is the precise bug the filter exists to stop reporting, so reintroducing that asymmetry while re-checking was the wrong way round. The two extractions now live in state_reconciliation.py as config_plugin_ids() and disk_plugin_ids(), with ignored_config_keys() and secrets_top_level_keys() alongside. _get_config_state() and _get_disk_state() use them too, so there is one definition rather than two that can drift. _get_disk_state() re-reads each manifest for version/name after taking membership from the shared extractor; that costs one extra small read per plugin on a path that runs once per boot. Minor, the new test: the fixture assigned api_v3.config_manager and api_v3.plugin_manager directly. Those live on a module-level blueprint singleton, so the mocks leaked into every later test that imports api_v3 -- pointing at a tmp_path already deleted. Both now go through monkeypatch.setattr, which restores them. This is the same pollution class that made an earlier test in this session break seven unrelated ones, so it is worth getting right. Five cases added for the parity itself: a secrets key, a system key and a non-dict value must not clear an "installed but missing from config" finding, and neither an unparseable manifest nor a .standalone-backup- directory may count as installed. All five fail against the looser version. Linux CI on the preceding commit: Core unit tests, plugin harness, CodeQL and CodeRabbit all pass. Codacy reads action_required on every commit of this branch including the first, so it is pre-existing and not from this work. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 (1M context) --- plugin-repos/starlark-apps/manager.py | 21 +- src/plugin_system/state_reconciliation.py | 117 +- test/fixtures/api_v3_url_map.json | 947 ++ test/test_api_v3_font_upload_size.py | 60 + test/test_api_v3_on_demand_restart.py | 93 + test/test_api_v3_optional_body.py | 11 +- test/test_api_v3_schedule_error_messages.py | 54 + test/test_api_v3_url_map.py | 136 + test/test_api_v3_wifi_endpoints.py | 80 +- test/test_config_main_redacts_secrets.py | 51 +- test/test_reconciliation_status_endpoint.py | 167 + test/test_starlark_display_contract.py | 45 + test/test_vegas_density.py | 5 +- test/test_web_api.py | 8 +- test/test_web_error_detail.py | 6 +- .../test_api_v3_backup_restore.py | 54 +- .../test_calendar_oauth_endpoints.py | 21 +- .../test_config_logging_omits_secrets.py | 6 +- .../test_secret_separation_parity.py | 18 +- .../test_starlark_pixlet_routes.py | 166 +- .../test_systemctl_sudoers_alignment.py | 11 +- web_interface/blueprints/api_v3.py | 10016 ---------------- web_interface/blueprints/api_v3/__init__.py | 1853 +++ web_interface/blueprints/api_v3/backup.py | 230 + web_interface/blueprints/api_v3/config.py | 1178 ++ web_interface/blueprints/api_v3/display.py | 326 + web_interface/blueprints/api_v3/fonts.py | 467 + web_interface/blueprints/api_v3/misc.py | 398 + web_interface/blueprints/api_v3/plugins.py | 3902 ++++++ web_interface/blueprints/api_v3/starlark.py | 712 ++ web_interface/blueprints/api_v3/system.py | 662 + web_interface/blueprints/api_v3/wifi.py | 413 + 32 files changed, 12151 insertions(+), 10083 deletions(-) create mode 100644 test/fixtures/api_v3_url_map.json create mode 100644 test/test_api_v3_font_upload_size.py create mode 100644 test/test_api_v3_on_demand_restart.py create mode 100644 test/test_api_v3_schedule_error_messages.py create mode 100644 test/test_api_v3_url_map.py create mode 100644 test/test_reconciliation_status_endpoint.py delete mode 100644 web_interface/blueprints/api_v3.py create mode 100644 web_interface/blueprints/api_v3/__init__.py create mode 100644 web_interface/blueprints/api_v3/backup.py create mode 100644 web_interface/blueprints/api_v3/config.py create mode 100644 web_interface/blueprints/api_v3/display.py create mode 100644 web_interface/blueprints/api_v3/fonts.py create mode 100644 web_interface/blueprints/api_v3/misc.py create mode 100644 web_interface/blueprints/api_v3/plugins.py create mode 100644 web_interface/blueprints/api_v3/starlark.py create mode 100644 web_interface/blueprints/api_v3/system.py create mode 100644 web_interface/blueprints/api_v3/wifi.py diff --git a/plugin-repos/starlark-apps/manager.py b/plugin-repos/starlark-apps/manager.py index 4f370a85..999b98a7 100644 --- a/plugin-repos/starlark-apps/manager.py +++ b/plugin-repos/starlark-apps/manager.py @@ -220,6 +220,8 @@ class StarlarkAppsPlugin(BasePlugin): # App storage self.apps_dir = self._get_apps_directory() self.manifest_file = self.apps_dir / "manifest.json" + # A dedicated, never-replaced file to flock -- see _update_manifest_safe. + self.manifest_lock_file = self.apps_dir / "manifest.json.lock" self.apps: Dict[str, StarlarkApp] = {} # Display state @@ -564,7 +566,8 @@ class StarlarkAppsPlugin(BasePlugin): def _save_manifest(self, manifest: Dict[str, Any]) -> bool: """ Save apps manifest to file with file locking to prevent race conditions. - Acquires exclusive lock on manifest file before writing to prevent concurrent modifications. + Acquires exclusive lock on the manifest lock sidecar before writing to + prevent concurrent modifications. """ temp_file = None lock_fd = None @@ -572,9 +575,14 @@ class StarlarkAppsPlugin(BasePlugin): # Create parent directory if needed self.manifest_file.parent.mkdir(parents=True, exist_ok=True) - # Open manifest file for locking (create if doesn't exist, don't truncate) - # Use os.open with O_CREAT | O_RDWR to create if missing, but don't truncate - lock_fd = os.open(str(self.manifest_file), os.O_CREAT | os.O_RDWR, 0o644) + # Lock the sidecar file, not manifest_file itself: manifest_file is + # replaced by an atomic rename below, which swaps in a fresh inode + # a second locker's fresh os.open() would pick up unguarded. The + # sidecar is never written to or renamed over, so it always + # resolves to the same inode for every locker (see + # _update_manifest_safe and web_interface's _starlark_manifest_lock, + # which must lock this same file for the guarantee to hold). + lock_fd = os.open(str(self.manifest_lock_file), os.O_CREAT | os.O_RDWR, 0o644) # Acquire exclusive lock on manifest file BEFORE creating temp file # This serializes all writers and prevents concurrent races @@ -630,8 +638,9 @@ class StarlarkAppsPlugin(BasePlugin): # Create parent directory if needed self.manifest_file.parent.mkdir(parents=True, exist_ok=True) - # Open manifest file for locking (create if doesn't exist, don't truncate) - lock_fd = os.open(str(self.manifest_file), os.O_CREAT | os.O_RDWR, 0o644) + # Lock the sidecar file, not manifest_file itself -- see the + # comment in _save_manifest for why. + lock_fd = os.open(str(self.manifest_lock_file), os.O_CREAT | os.O_RDWR, 0o644) # Acquire exclusive lock for entire read-modify-write cycle fcntl.flock(lock_fd, fcntl.LOCK_EX) diff --git a/src/plugin_system/state_reconciliation.py b/src/plugin_system/state_reconciliation.py index e967fbdf..f2b55100 100644 --- a/src/plugin_system/state_reconciliation.py +++ b/src/plugin_system/state_reconciliation.py @@ -56,6 +56,71 @@ class ReconciliationResult: message: str +def secrets_top_level_keys(config_manager) -> Set[str]: + """Top-level keys load_config() merges in from the secrets file. + + Deliberately fail-safe: an unreadable, absent, malformed or non-path + secrets location narrows this set rather than raising, because a failure + here must never break reconciliation. + """ + try: + path = config_manager.get_secrets_path() + with open(path, 'r') as f: + secrets = json.load(f) + except (AttributeError, OSError, TypeError, ValueError): + return set() + return set(secrets) if isinstance(secrets, dict) else set() + + +def ignored_config_keys(config_manager) -> Set[str]: + """Config keys that are not plugin ids: system keys plus secrets keys.""" + return set(StateReconciliation._SYSTEM_CONFIG_KEYS) | secrets_top_level_keys(config_manager) + + +def config_plugin_ids(config: Dict[str, Any], ignored_keys: Set[str]) -> Set[str]: + """Plugin ids a loaded config declares. + + Shared with the web interface so a stored verdict is re-checked against the + same definition that produced it. ``set(config)`` is NOT equivalent: it also + contains system keys, the secrets-file keys load_config() merges in, and + non-dict values. Counting any of those as a plugin is exactly what turned a + 'data' key in the secrets file into a phantom plugin, and using the loose + set to re-check findings would clear ones that are still true. + """ + return {k for k, v in (config or {}).items() + if isinstance(v, dict) and k not in ignored_keys} + + +def disk_plugin_ids(plugins_dir) -> Set[str]: + """Plugin ids actually installed on disk. + + A directory counts only when it is not a standalone backup and its + manifest.json parses. A corrupt manifest must not read as installed, or a + live "in config but not on disk" finding gets cleared on the strength of an + unreadable file. + """ + ids: Set[str] = set() + root = Path(plugins_dir) + try: + if not root.exists(): + return ids + for entry in root.iterdir(): + if not entry.is_dir() or '.standalone-backup-' in entry.name: + continue + manifest = entry / "manifest.json" + if not manifest.exists(): + continue + try: + with open(manifest, 'r') as f: + json.load(f) + except (OSError, ValueError): + continue + ids.add(entry.name) + except OSError: + return ids + return ids + + def still_unresolved(entries: List[Dict[str, Any]], config_keys: Set[str], installed_ids: Set[str]) -> List[Dict[str, Any]]: @@ -240,16 +305,7 @@ class StateReconciliation: reported as "in config but not on disk". Reading the file keeps this correct no matter what it holds. """ - try: - path = self.config_manager.get_secrets_path() - with open(path, 'r') as f: - secrets = json.load(f) - except (AttributeError, OSError, TypeError, ValueError): - # Deliberately broad: an unreadable, absent, malformed or - # non-path secrets location must narrow this set, never break - # reconciliation. ValueError covers json.JSONDecodeError. - return set() - return set(secrets) if isinstance(secrets, dict) else set() + return secrets_top_level_keys(self.config_manager) def _get_config_state(self) -> Dict[str, Dict[str, Any]]: """Get plugin state from config file.""" @@ -257,11 +313,8 @@ class StateReconciliation: try: config = self.config_manager.load_config() ignored = self._SYSTEM_CONFIG_KEYS | self._secrets_top_level_keys() - for plugin_id, plugin_config in config.items(): - if not isinstance(plugin_config, dict): - continue - if plugin_id in ignored: - continue + for plugin_id in config_plugin_ids(config, ignored): + plugin_config = config[plugin_id] state[plugin_id] = { 'enabled': plugin_config.get('enabled', True), 'version': plugin_config.get('version'), @@ -275,25 +328,21 @@ class StateReconciliation: """Get plugin state from disk (installed plugins).""" state = {} try: - if self.plugins_dir.exists(): - for plugin_dir in self.plugins_dir.iterdir(): - if plugin_dir.is_dir(): - plugin_id = plugin_dir.name - if '.standalone-backup-' in plugin_id: - continue - manifest_path = plugin_dir / "manifest.json" - if manifest_path.exists(): - import json - try: - with open(manifest_path, 'r') as f: - manifest = json.load(f) - state[plugin_id] = { - 'exists_on_disk': True, - 'version': manifest.get('version'), - 'name': manifest.get('name') - } - except Exception: # nosec B110 - corrupt/unreadable manifest; skip this plugin, outer except logs - pass + # Membership comes from the shared extractor so the web interface + # re-checks stored findings against this same definition; the + # manifest is then re-read here only for version/name. + for plugin_id in disk_plugin_ids(self.plugins_dir): + manifest_path = self.plugins_dir / plugin_id / "manifest.json" + try: + with open(manifest_path, 'r') as f: + manifest = json.load(f) + except (OSError, ValueError): # nosec B112 - raced or corrupt; skip + continue + state[plugin_id] = { + 'exists_on_disk': True, + 'version': manifest.get('version'), + 'name': manifest.get('name') + } except Exception as e: self.logger.warning(f"Error reading disk state: {e}") return state diff --git a/test/fixtures/api_v3_url_map.json b/test/fixtures/api_v3_url_map.json new file mode 100644 index 00000000..f2557fd9 --- /dev/null +++ b/test/fixtures/api_v3_url_map.json @@ -0,0 +1,947 @@ +[ + [ + "/api/v3/backup/", + "api_v3.backup_delete", + [ + "DELETE", + "OPTIONS" + ] + ], + [ + "/api/v3/backup/download/", + "api_v3.backup_download", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/backup/export", + "api_v3.backup_export", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/backup/list", + "api_v3.backup_list", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/backup/preview", + "api_v3.backup_preview", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/backup/restore", + "api_v3.backup_restore", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/backup/validate", + "api_v3.backup_validate", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/cache/delete", + "api_v3.delete_cache_file", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/cache/list", + "api_v3.list_cache_files", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/config/dim-schedule", + "api_v3.get_dim_schedule_config", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/config/dim-schedule", + "api_v3.save_dim_schedule_config", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/config/main", + "api_v3.get_main_config", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/config/main", + "api_v3.save_main_config", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/config/raw/main", + "api_v3.save_raw_main_config", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/config/raw/secrets", + "api_v3.save_raw_secrets_config", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/config/schedule", + "api_v3.get_schedule_config", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/config/schedule", + "api_v3.save_schedule_config", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/config/secrets", + "api_v3.get_secrets_config", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/display/current", + "api_v3.get_display_current", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/display/current-status", + "api_v3.get_current_display_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/display/modes", + "api_v3.get_display_modes", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/display/on-demand/start", + "api_v3.start_on_demand_display", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/display/on-demand/status", + "api_v3.get_on_demand_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/display/on-demand/stop", + "api_v3.stop_on_demand_display", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/errors/clear", + "api_v3.clear_old_errors", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/errors/plugin/", + "api_v3.get_plugin_errors", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/errors/summary", + "api_v3.get_error_summary", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/fonts/", + "api_v3.delete_font", + [ + "DELETE", + "OPTIONS" + ] + ], + [ + "/api/v3/fonts/catalog", + "api_v3.get_fonts_catalog", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/fonts/overrides", + "api_v3.get_fonts_overrides", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/fonts/overrides", + "api_v3.save_fonts_overrides", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/fonts/overrides/", + "api_v3.delete_font_override", + [ + "DELETE", + "OPTIONS" + ] + ], + [ + "/api/v3/fonts/preview", + "api_v3.get_font_preview", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/fonts/tokens", + "api_v3.get_font_tokens", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/fonts/upload", + "api_v3.upload_font", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/hardware/status", + "api_v3.get_hardware_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/health", + "api_v3.get_health", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/logs", + "api_v3.get_logs", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins//static/", + "api_v3.serve_plugin_static", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/action", + "api_v3.execute_plugin_action", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/assets/delete", + "api_v3.delete_plugin_asset", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/assets/list", + "api_v3.list_plugin_assets", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/assets/upload", + "api_v3.upload_plugin_asset", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/authenticate/spotify", + "api_v3.authenticate_spotify", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/authenticate/ytm", + "api_v3.authenticate_ytm", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/calendar/authenticate", + "api_v3.authenticate_calendar", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/calendar/list-calendars", + "api_v3.list_calendar_calendars", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/calendar/upload-credentials", + "api_v3.upload_calendar_credentials", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/config", + "api_v3.get_plugin_config", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/config", + "api_v3.save_plugin_config", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/config/reset", + "api_v3.reset_plugin_config", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/health", + "api_v3.get_plugin_health", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/health/", + "api_v3.get_plugin_health_single", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/health//reset", + "api_v3.reset_plugin_health", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/install", + "api_v3.install_plugin", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/install-from-url", + "api_v3.install_plugin_from_url", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/installed", + "api_v3.get_installed_plugins", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/limits/", + "api_v3.manage_plugin_limits", + [ + "GET", + "HEAD", + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/metrics", + "api_v3.get_plugin_metrics", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/metrics/", + "api_v3.get_plugin_metrics_single", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/metrics//reset", + "api_v3.reset_plugin_metrics", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/of-the-day/json/delete", + "api_v3.delete_of_the_day_json", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/of-the-day/json/upload", + "api_v3.upload_of_the_day_json", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/operation/", + "api_v3.get_operation_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/operation/history", + "api_v3.clear_operation_history", + [ + "DELETE", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/operation/history", + "api_v3.get_operation_history", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/reconciliation-status", + "api_v3.get_reconciliation_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/registry-from-url", + "api_v3.get_registry_from_url", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/saved-repositories", + "api_v3.add_saved_repository", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/saved-repositories", + "api_v3.get_saved_repositories", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/saved-repositories", + "api_v3.remove_saved_repository", + [ + "DELETE", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/schema", + "api_v3.get_plugin_schema", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/state", + "api_v3.get_plugin_state", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/state/reconcile", + "api_v3.reconcile_plugin_state", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/store/github-status", + "api_v3.get_github_auth_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/store/list", + "api_v3.list_plugin_store", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/plugins/store/refresh", + "api_v3.refresh_plugin_store", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/toggle", + "api_v3.toggle_plugin", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/uninstall", + "api_v3.uninstall_plugin", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/plugins/update", + "api_v3.update_plugin", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/skins", + "api_v3.list_skins", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/apps", + "api_v3.get_starlark_apps", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/apps/", + "api_v3.get_starlark_app", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/apps/", + "api_v3.uninstall_starlark_app", + [ + "DELETE", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/apps//config", + "api_v3.get_starlark_app_config", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/apps//config", + "api_v3.update_starlark_app_config", + [ + "OPTIONS", + "PUT" + ] + ], + [ + "/api/v3/starlark/apps//render", + "api_v3.render_starlark_app", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/starlark/apps//toggle", + "api_v3.toggle_starlark_app", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/starlark/install-pixlet", + "api_v3.install_pixlet", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/starlark/repository/browse", + "api_v3.browse_tronbyte_repository", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/repository/categories", + "api_v3.get_tronbyte_categories", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/repository/install", + "api_v3.install_from_tronbyte_repository", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/starlark/status", + "api_v3.get_starlark_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/upload", + "api_v3.upload_starlark_app", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/sync/status", + "api_v3.get_sync_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/system/action", + "api_v3.execute_system_action", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/system/check-update", + "api_v3.check_for_update", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/system/git-branches", + "api_v3.get_git_branches", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/system/git-info", + "api_v3.get_git_info", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/system/status", + "api_v3.get_system_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/system/version", + "api_v3.get_system_version", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/wifi/ap/auto-enable", + "api_v3.get_auto_enable_ap_mode", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/wifi/ap/auto-enable", + "api_v3.set_auto_enable_ap_mode", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/wifi/ap/disable", + "api_v3.disable_ap_mode", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/wifi/ap/enable", + "api_v3.enable_ap_mode", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/wifi/connect", + "api_v3.connect_wifi", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/wifi/disconnect", + "api_v3.disconnect_wifi", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/wifi/radio", + "api_v3.get_wifi_radio", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/wifi/radio", + "api_v3.set_wifi_radio", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/wifi/scan", + "api_v3.scan_wifi_networks", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/wifi/status", + "api_v3.get_wifi_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ] +] \ No newline at end of file diff --git a/test/test_api_v3_font_upload_size.py b/test/test_api_v3_font_upload_size.py new file mode 100644 index 00000000..cd1b731d --- /dev/null +++ b/test/test_api_v3_font_upload_size.py @@ -0,0 +1,60 @@ +"""Regression test: POST /fonts/upload must enforce its own stated size limit. + +validate_file_upload(filename, max_size_mb=10, allowed_extensions=[...]) reads +like it checks the upload's size, but it only ever validated the filename +(traversal characters, extension) -- max_size_mb was accepted and silently +ignored. Nothing else in the handler checked the actual upload size either, +so it saved whatever was posted to assets/fonts/ regardless of +size, unlike the sibling .star and plugin-asset upload routes, which check +`file.tell()` against a stated limit before saving. +""" + +import io +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402 + +URL = "/api/v3/fonts/upload" +TEN_MB = 10 * 1024 * 1024 + + +@pytest.fixture +def fonts_root(tmp_path): + # PROJECT_ROOT is bound by value in fonts.py, so it is patched there. + with patch("web_interface.blueprints.api_v3.fonts.PROJECT_ROOT", tmp_path): + yield tmp_path + + +def upload(client, content, filename="myfont.ttf", family="myfont"): + data = { + "font_file": (io.BytesIO(content), filename), + "font_family": family, + } + return client.post(URL, data=data, content_type="multipart/form-data") + + +class TestFontUploadSizeLimit: + def test_oversized_font_is_rejected(self, api_v3_client, fonts_root): + response = upload(api_v3_client, b"x" * (TEN_MB + 1)) + assert response.status_code == 400 + assert "too large" in response.get_json()["message"].lower() + assert not (fonts_root / "assets" / "fonts" / "myfont.ttf").exists(), ( + "an oversized font was saved to disk before being rejected") + + def test_a_font_right_at_the_limit_is_accepted(self, api_v3_client, fonts_root): + response = upload(api_v3_client, b"x" * TEN_MB, + filename="atlimit.ttf", family="atlimit") + assert response.status_code == 200, response.get_json() + assert (fonts_root / "assets" / "fonts" / "atlimit.ttf").exists() + + def test_an_ordinary_small_font_is_still_accepted(self, api_v3_client, fonts_root): + response = upload(api_v3_client, b"fake font bytes", + filename="small.ttf", family="small") + assert response.status_code == 200, response.get_json() + assert (fonts_root / "assets" / "fonts" / "small.ttf").read_bytes() == b"fake font bytes" diff --git a/test/test_api_v3_on_demand_restart.py b/test/test_api_v3_on_demand_restart.py new file mode 100644 index 00000000..c203fbe4 --- /dev/null +++ b/test/test_api_v3_on_demand_restart.py @@ -0,0 +1,93 @@ +"""Regression test: POST /display/on-demand/start restarting a running +service must not import a name that does not exist. + +display.py has `import web_interface.blueprints.api_v3 as _pkg` and reads +mutable, test-patched attributes back through it (`_pkg.time.time()`, +`_pkg._get_starlark_plugin()`, ...) rather than binding them by value, per +the package's own docstring. One spot went further and wrote a genuine +`import` *statement* against that alias -- + + import _pkg.time as time_module + +-- but `_pkg` is a local name bound by `import ... as _pkg` in this module, +not a real top-level package, so `import _pkg.time` is not something Python +can resolve; it raises ModuleNotFoundError. That line only runs when the +display service is already running and the caller also asked to (re)start +it, so this endpoint failed on exactly the restart path -- the one where a +cache write recording the new on-demand request had already happened. + +The route wraps its body in `except Exception`, so the failure reached the +caller as a handled 500 with a generic message, not an unhandled crash -- +but a 500 all the same on a request that should have restarted the service +and reported success. +""" + +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402 + +URL = "/api/v3/display/on-demand/start" + + +@pytest.fixture +def restart_path(api_v3_module): + """Force the `service_was_running and start_service` branch. + + plugin_manager and config_manager are set to None so the route takes + the simplest path to that branch rather than tripping over unrelated + MagicMock plumbing; _ensure_cache_manager, _get_display_service_status, + _stop_display_service and _ensure_display_service_running are bound by + value in display.py (see its own docstring), so they are patched on + that submodule rather than on the package. + """ + api_v3_module.api_v3.plugin_manager = None + api_v3_module.api_v3.config_manager = None + + with patch("web_interface.blueprints.api_v3.display._ensure_cache_manager") as ensure_cache, \ + patch("web_interface.blueprints.api_v3.display._get_display_service_status") as get_status, \ + patch("web_interface.blueprints.api_v3.display._stop_display_service") as stop_service, \ + patch("web_interface.blueprints.api_v3.display._ensure_display_service_running") as ensure_running: + ensure_cache.return_value = MagicMock() + # Active before the request: service_was_running becomes True. + get_status.return_value = {"active": True} + ensure_running.return_value = {"active": True} + yield { + "ensure_cache": ensure_cache, + "get_status": get_status, + "stop_service": stop_service, + "ensure_running": ensure_running, + } + + +class TestRestartingARunningService: + def test_it_does_not_500(self, api_v3_client, restart_path): + response = api_v3_client.post( + URL, json={"plugin_id": "weather", "start_service": True}) + body = response.get_json() + assert response.status_code == 200, body + assert body["status"] == "success", body + + def test_the_service_is_actually_stopped_and_restarted( + self, api_v3_client, restart_path): + api_v3_client.post( + URL, json={"plugin_id": "weather", "start_service": True}) + restart_path["stop_service"].assert_called_once() + restart_path["ensure_running"].assert_called_once() + + def test_a_service_that_was_not_running_is_not_stopped_first( + self, api_v3_client, restart_path): + # The buggy import sits inside `if service_was_running and + # start_service`, so it only ever fired on the restart path -- + # this is the other side of that branch, unaffected either way, + # kept here so the branch condition itself stays covered. + restart_path["get_status"].return_value = {"active": False} + response = api_v3_client.post( + URL, json={"plugin_id": "weather", "start_service": True}) + assert response.status_code == 200, response.get_json() + restart_path["stop_service"].assert_not_called() diff --git a/test/test_api_v3_optional_body.py b/test/test_api_v3_optional_body.py index 4b7eb133..fe1abc5e 100644 --- a/test/test_api_v3_optional_body.py +++ b/test/test_api_v3_optional_body.py @@ -106,7 +106,12 @@ class TestMissingBodyGivesTheDeclaredError: class TestNoBodyReadContradictsItsOwnGuard: - SOURCE = Path(__file__).parent.parent / "web_interface/blueprints/api_v3.py" + PKG = Path(__file__).parent.parent / "web_interface/blueprints/api_v3" + + @property + def _source(self) -> str: + """api_v3 is a package; read every module of it.""" + return "\n".join(p.read_text() for p in sorted(self.PKG.glob("*.py"))) def test_no_or_default_read_is_unguarded(self): """`get_json() or ` is a contradiction without silent=True. @@ -115,7 +120,7 @@ class TestNoBodyReadContradictsItsOwnGuard: means the call raises before the default can apply. """ offenders = [ - line.strip() for line in self.SOURCE.read_text().splitlines() + line.strip() for line in self._source.splitlines() if "request.get_json()" in line and " or " in line ] assert offenders == [], ( @@ -124,7 +129,7 @@ class TestNoBodyReadContradictsItsOwnGuard: def test_no_not_data_guard_is_unreachable(self): """A `if not data:` guard needs a read that can actually return None.""" - lines = self.SOURCE.read_text().splitlines() + lines = self._source.splitlines() offenders = [] for i, line in enumerate(lines): if re.search(r"=\s*request\.get_json\(\)\s*$", line): diff --git a/test/test_api_v3_schedule_error_messages.py b/test/test_api_v3_schedule_error_messages.py new file mode 100644 index 00000000..383ec172 --- /dev/null +++ b/test/test_api_v3_schedule_error_messages.py @@ -0,0 +1,54 @@ +"""Regression test: per-day schedule validation errors must say "time", not +"_pkg.time". + +The split into a package rewrote every bare `time` reference that needed to +read through the shared module as `_pkg.time` (see the package's own +docstring on why -- tests patch it, so it has to be read back live rather +than bound by value). That rewrite was mechanical and matched the substring +"time" inside string literals and comments too, so the user-facing message + + "Invalid start time for {day}: ..." + +came out as + + "Invalid start _pkg.time for {day}: ..." + +in both POST /config/schedule and POST /config/dim-schedule, for both the +start and end time of a per-day entry. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402 + + +@pytest.mark.parametrize("url", [ + "/api/v3/config/schedule", + "/api/v3/config/dim-schedule", +]) +class TestPerDayTimeErrorsAreNotCorrupted: + def test_invalid_start_time_message(self, api_v3_client, api_v3_module, url): + response = api_v3_client.post(url, json={ + "mode": "per_day", + "monday_start": "not-a-time", + }) + assert response.status_code == 400 + message = response.get_json()["message"] + assert "_pkg" not in message, message + assert message.startswith("Invalid start time for monday:"), message + + def test_invalid_end_time_message(self, api_v3_client, api_v3_module, url): + response = api_v3_client.post(url, json={ + "mode": "per_day", + "monday_start": "07:00", + "monday_end": "not-a-time", + }) + assert response.status_code == 400 + message = response.get_json()["message"] + assert "_pkg" not in message, message + assert message.startswith("Invalid end time for monday:"), message diff --git a/test/test_api_v3_url_map.py b/test/test_api_v3_url_map.py new file mode 100644 index 00000000..157ab8a1 --- /dev/null +++ b/test/test_api_v3_url_map.py @@ -0,0 +1,136 @@ +"""The /api/v3 URL map is a contract, and the split must not have moved it. + +api_v3.py was one 10,469-line module and is now a package. Every route module +in it decorates the *same* Blueprint object, so this refactor was supposed to +be invisible from outside: same URLs, same endpoint names, same methods. + +"Supposed to be" is the problem. A route function silently dropped during a +move -- a module that never gets imported, a decorator left behind -- costs +nothing at import time and fails only when someone hits the URL. So the map is +pinned here. + +The snapshot is intentionally the *whole* map rather than a count. A count +passes when one route is deleted and another added, which is exactly the shape +a careless move produces. + +If you are adding a route, this test is meant to fail: add the entry to +EXPECTED. If you are moving one between modules, it is meant to pass unchanged +-- endpoint names are `api_v3.` regardless of which module the +function lives in, and that is the property that makes the package safe. +""" +import json +import os + +import pytest +from flask import Flask + +from web_interface.blueprints.api_v3 import api_v3 + +SNAPSHOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "fixtures", "api_v3_url_map.json") + + +def _current_map(): + app = Flask(__name__) + app.register_blueprint(api_v3, url_prefix="/api/v3") + return sorted( + [r.rule, r.endpoint, sorted(r.methods)] + for r in app.url_map.iter_rules() + if r.endpoint != "static" + ) + + +def test_the_url_map_matches_the_snapshot(): + current = _current_map() + with open(SNAPSHOT, encoding="utf-8") as fh: + expected = json.load(fh) + + cur = {(r, e) for r, e, _ in current} + exp = {(r, e) for r, e, _ in expected} + + lost = sorted(exp - cur) + added = sorted(cur - exp) + assert not lost, ( + f"{len(lost)} route(s) disappeared from /api/v3: {lost[:5]}. " + "A route lost in a module move costs nothing at import time and fails " + "only when someone hits the URL.") + assert not added, ( + f"{len(added)} new route(s): {added[:5]}. If that is intended, " + f"regenerate {os.path.relpath(SNAPSHOT)}.") + + # Methods too: a route that quietly loses POST is still a broken route. + cur_methods = {(r, e): m for r, e, m in current} + for rule, endpoint, methods in expected: + assert cur_methods[(rule, endpoint)] == methods, ( + f"{rule} ({endpoint}) methods changed: " + f"{methods} -> {cur_methods[(rule, endpoint)]}") + + +def test_every_endpoint_is_on_the_one_blueprint(): + """The package must not fragment into several blueprints. + + Splitting into per-domain *blueprints* would rename every endpoint from + `api_v3.foo` to `api_v3_plugins.foo` and break any url_for() that names + one. Keeping a single Blueprint object across the modules is what makes + the split a pure code move, so assert it rather than trusting it. + """ + for rule, endpoint, _ in _current_map(): + assert endpoint.startswith("api_v3."), ( + f"{rule} is registered as {endpoint}, not on the api_v3 blueprint") + + +def test_the_snapshot_is_not_empty(): + """Guards the failure mode this file exists to prevent. + + An empty or truncated snapshot would make every assertion above pass + vacuously -- the same trap as a route map that imports no modules. + """ + with open(SNAPSHOT, encoding="utf-8") as fh: + expected = json.load(fh) + assert len(expected) > 100, ( + f"snapshot has only {len(expected)} routes; it should have the whole " + "/api/v3 surface") + + +@pytest.mark.parametrize("module", [ + "backup", "config", "display", "fonts", "misc", + "plugins", "starlark", "system", "wifi", +]) +def test_every_route_module_contributes(module): + """Each module must actually register something. + + A module that fails to import, or that is left out of __init__, takes its + routes with it silently -- the package still imports and the app still + starts. + """ + import importlib + mod = importlib.import_module(f"web_interface.blueprints.api_v3.{module}") + routes = [n for n in dir(mod) + if callable(getattr(mod, n, None)) + and getattr(getattr(mod, n), "__module__", "") == mod.__name__] + assert routes, f"{module}.py defines no view functions" + + +def test_project_root_points_at_the_project(): + """PROJECT_ROOT is derived from __file__, so moving the file breaks it. + + The split moved this code from web_interface/blueprints/api_v3.py to + web_interface/blueprints/api_v3/_common.py -- one directory deeper -- and + `Path(__file__).parent.parent.parent` quietly began resolving to + web_interface/ instead of the project root. Nothing failed at import. It + surfaced as routes returning 404 and "installation script not found", + because every path built from it pointed one level too shallow. + + A URL-map check cannot catch that: the routes were all registered, they + just could not find anything. + """ + from pathlib import Path + + from web_interface.blueprints.api_v3 import PROJECT_ROOT + + # The project root is the directory holding run.py and web_interface/. + assert (PROJECT_ROOT / "run.py").is_file(), ( + f"PROJECT_ROOT is {PROJECT_ROOT}, which has no run.py; it is not the " + "project root") + assert (PROJECT_ROOT / "web_interface").is_dir() + assert PROJECT_ROOT == Path(__file__).resolve().parents[1] diff --git a/test/test_api_v3_wifi_endpoints.py b/test/test_api_v3_wifi_endpoints.py index 0c713ae2..344aef8b 100644 --- a/test/test_api_v3_wifi_endpoints.py +++ b/test/test_api_v3_wifi_endpoints.py @@ -195,7 +195,7 @@ class TestRadio: @pytest.mark.parametrize("raw,expected", [ (True, True), ("true", True), ("1", True), ("yes", True), - (False, False), ("false", False), ("off", False), (0, False), + (False, False), ("false", False), ("no", False), (0, False), ]) def test_enabled_coercion_is_string_aware( self, api_v3_client, wifi_manager, raw, expected): @@ -228,6 +228,84 @@ class TestRadio: wifi_manager.set_wifi_radio.side_effect = RuntimeError("boom") assert api_v3_client.post(self.URL, json={"enabled": True}).status_code == 500 + def test_unrecognized_enabled_value_is_rejected_not_treated_as_false( + self, api_v3_client, wifi_manager): + # An invalid `enabled` used to silently fall back to False, which + # can disconnect Wi-Fi (or, with force=true, drop the caller's own + # connection to this interface) even though nothing asked for that. + response = api_v3_client.post(self.URL, json={"enabled": "typo"}) + assert response.status_code == 400 + wifi_manager.set_wifi_radio.assert_not_called() + + def test_unrecognized_force_value_is_rejected_not_treated_as_false( + self, api_v3_client, wifi_manager): + response = api_v3_client.post( + self.URL, json={"enabled": False, "force": "typo"}) + assert response.status_code == 400 + wifi_manager.set_wifi_radio.assert_not_called() + + +class TestAutoEnableApMode: + URL = "/api/v3/wifi/ap/auto-enable" + + def test_requires_the_field(self, api_v3_client, wifi_manager): + response = api_v3_client.post(self.URL, json={}) + assert response.status_code == 400 + + @pytest.mark.parametrize("raw,expected", [ + (True, True), (False, False), + ("true", True), ("True", True), ("1", True), ("yes", True), + ("false", False), ("False", False), ("0", False), ("no", False), + (1, True), (0, False), + ]) + def test_value_is_coerced_not_just_truthy( + self, api_v3_client, wifi_manager, raw, expected): + # Regression: bool(data['auto_enable_ap_mode']) meant a caller who + # sent the JSON string "false" got it stored as True — bool("false") + # is True, since any non-empty string is truthy. + wifi_manager.config = {} + response = api_v3_client.post(self.URL, json={"auto_enable_ap_mode": raw}) + assert response.status_code == 200, response.get_json() + assert response.get_json()["data"]["auto_enable_ap_mode"] is expected + assert wifi_manager.config["auto_enable_ap_mode"] is expected + + def test_a_string_false_does_not_enable_it(self, api_v3_client, wifi_manager): + # The exact shape of the bug. + wifi_manager.config = {} + api_v3_client.post(self.URL, json={"auto_enable_ap_mode": "false"}) + assert wifi_manager.config["auto_enable_ap_mode"] is False + + def test_unrecognized_value_is_rejected_not_treated_as_false( + self, api_v3_client, wifi_manager): + wifi_manager.config = {} + response = api_v3_client.post(self.URL, json={"auto_enable_ap_mode": "typo"}) + assert response.status_code == 400 + assert "auto_enable_ap_mode" not in wifi_manager.config + + +class TestRadioEnabledAndForceAcceptIntegers: + """`{"enabled": 1}` / `{"enabled": 0}` used to be mishandled: the old + coercion was `raw is True or (isinstance(raw, str) and ...)`, and + `1 is True` is False in Python -- an int is never the `True` singleton + even though it equals it -- so a plain integer fell through to False + regardless of its value. + """ + URL = "/api/v3/wifi/radio" + + @pytest.mark.parametrize("raw,expected", [(1, True), (0, False)]) + def test_enabled_as_an_integer(self, api_v3_client, wifi_manager, raw, expected): + wifi_manager.set_wifi_radio.return_value = (True, "ok", None) + wifi_manager.get_wifi_radio_state.return_value = {} + api_v3_client.post(self.URL, json={"enabled": raw}) + wifi_manager.set_wifi_radio.assert_called_once_with(expected, force=False) + + @pytest.mark.parametrize("raw,expected", [(1, True), (0, False)]) + def test_force_as_an_integer(self, api_v3_client, wifi_manager, raw, expected): + wifi_manager.set_wifi_radio.return_value = (True, "ok", None) + wifi_manager.get_wifi_radio_state.return_value = {} + api_v3_client.post(self.URL, json={"enabled": True, "force": raw}) + wifi_manager.set_wifi_radio.assert_called_once_with(True, force=expected) + class TestNoRealNetworking: def test_wifi_manager_is_never_constructed_for_real(self, api_v3_client): diff --git a/test/test_config_main_redacts_secrets.py b/test/test_config_main_redacts_secrets.py index 15172df2..d8f94993 100644 --- a/test/test_config_main_redacts_secrets.py +++ b/test/test_config_main_redacts_secrets.py @@ -103,6 +103,54 @@ def test_non_dict_input_passes_through(): assert _redact_credentials(None) is None +def test_a_list_of_bare_credentials_is_redacted(): + """A credential-named key whose value is a list of scalars, not a list + of named-field dicts, used to pass through untouched: the dict branch + recursed into `v` on its own value only, so a bare string list item had + no field name to test and fell through the base case unredacted. + """ + config = {"tokens": ["abc123", "def456"], "timezone": "America/New_York"} + out = _redact_credentials(config) + assert out["tokens"] == ["", ""] + assert out["timezone"] == "America/New_York" + + +def test_a_list_nested_inside_a_credential_named_container_is_also_blanked(): + """The list fix applies at any depth under a credential-named key, not + only when the list is the key's direct value. + """ + config = {"auth": {"backup_tokens": ["a", "b"], "note": "keep"}} + out = _redact_credentials(config) + assert out["auth"]["backup_tokens"] == ["", ""] + assert out["auth"]["note"] == "keep" + + +def test_objects_inside_a_credential_named_list_are_blanked_regardless_of_field_names(): + """A list item has no field name of its own to test against + `_CREDENTIAL_NAME_PARTS`, so an object reached through a credential-owned + list must be blanked outright rather than walked by field name -- unlike + the dict-directly-under-a-credential-key case in the test above. + """ + config = {"tokens": [{"value": "secret", "kind": "bearer"}]} + out = _redact_credentials(config) + assert out["tokens"] == [{"value": "", "kind": ""}] + + +def test_credential_shaped_container_with_named_fields_still_only_blanks_those(): + """Unchanged behaviour for the case the container test above already + covers: a dict whose sub-keys are semantically named fields is walked + normally, not blanket-blanked, so an ordinary field next to a + credential-named one survives. + """ + config = {"secrets": {"api_key": "k", "note": "keep", "list": ["a", "b"]}} + out = _redact_credentials(config) + assert out["secrets"]["api_key"] == "" + assert out["secrets"]["note"] == "keep" + # "list" isn't itself credential-named, so it is walked, not blanked -- + # but it holds no credential-named field either, so it survives whole. + assert out["secrets"]["list"] == ["a", "b"] + + def test_the_endpoint_itself_redacts(): """Through the view function, not the helper. @@ -130,7 +178,8 @@ def test_the_endpoint_itself_redacts(): app = flask.Flask(__name__) try: with app.test_request_context("/config/main"): - response = mod.get_main_config() + # get_main_config is a route; it lives in the config module now. + response = mod.config.get_main_config() payload = response.get_json() if hasattr(response, "get_json") else _json.loads(response[0].data) finally: mod.api_v3.config_manager = previous diff --git a/test/test_reconciliation_status_endpoint.py b/test/test_reconciliation_status_endpoint.py new file mode 100644 index 00000000..f6b77897 --- /dev/null +++ b/test/test_reconciliation_status_endpoint.py @@ -0,0 +1,167 @@ +"""The reconciliation-status endpoint must re-check its stored verdict. + +#557 added _drop_stale_reconciliation_findings() so a resolved condition stops +being reported: the verdict is a snapshot written once per run, and a run that +fails to apply a fix also declines to retry, so a device whose plugins were all +present in config kept being told for hours that four of them were missing. + +That wiring had no test. Only the pure still_unresolved() helper in +src/plugin_system/ was covered, which lives outside web_interface and therefore +survived the api_v3 blueprint split untouched -- so when #553 moved this +endpoint into api_v3/plugins.py, losing the filter would have been completely +silent. This test exists so that cannot happen again. +""" + +import json +import sys +import tempfile +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from flask import Flask + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from web_interface.blueprints.api_v3 import api_v3 # noqa: E402 + +IN_CONFIG = "plugin_missing_in_config" +ON_DISK = "plugin_missing_on_disk" + + +@pytest.fixture +def client(tmp_path, monkeypatch): + """App whose status file, config and plugins dir are all under our control.""" + monkeypatch.setattr(tempfile, "gettempdir", lambda: str(tmp_path)) + + plugins_dir = tmp_path / "plugin-repos" + plugins_dir.mkdir() + + def _install(plugin_id): + d = plugins_dir / plugin_id + d.mkdir() + (d / "manifest.json").write_text(json.dumps({"id": plugin_id}), encoding="utf-8") + + def _setup(verdict, config=None, installed=(), secrets=None, corrupt=()): + (tmp_path / "ledmatrix_reconciliation.json").write_text( + json.dumps(verdict), encoding="utf-8") + for pid in installed: + _install(pid) + for pid in corrupt: + d = plugins_dir / pid + d.mkdir(exist_ok=True) + (d / "manifest.json").write_text("{ this is not json", encoding="utf-8") + + secrets_path = tmp_path / "config_secrets.json" + secrets_path.write_text(json.dumps(secrets or {}), encoding="utf-8") + + cm = MagicMock() + cm.load_config.return_value = dict(config or {}) + cm.get_secrets_path.return_value = str(secrets_path) + pm = MagicMock() + pm.plugins_dir = str(plugins_dir) + # setattr via monkeypatch: these live on a module-level blueprint + # singleton, so assigning them directly leaks mocks -- pointing at a + # deleted tmp_path -- into every later test that imports api_v3. + monkeypatch.setattr(api_v3, "config_manager", cm, raising=False) + monkeypatch.setattr(api_v3, "plugin_manager", pm, raising=False) + + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(api_v3, url_prefix="/api/v3") + return app.test_client() + + return _setup + + +def _get(c): + return c.get("/api/v3/plugins/reconciliation-status").get_json()["data"] + + +class TestStaleFindingsAreDroppedByTheEndpoint: + def test_a_plugin_now_in_config_is_no_longer_reported(self, client): + c = client({"done": True, "unresolved": [ + {"plugin_id": "football-scoreboard", "type": IN_CONFIG}]}, + config={"football-scoreboard": {"enabled": True}}) + # The regression: this kept being reported for hours after it resolved. + assert _get(c)["unresolved"] == [] + + def test_a_plugin_now_installed_is_no_longer_reported(self, client): + c = client({"done": True, "unresolved": [ + {"plugin_id": "odds-ticker", "type": ON_DISK}]}, + installed=["odds-ticker"]) + assert _get(c)["unresolved"] == [] + + def test_a_finding_that_still_holds_is_kept(self, client): + c = client({"done": True, "unresolved": [ + {"plugin_id": "ghost-plugin", "type": ON_DISK}]}) + assert [e["plugin_id"] for e in _get(c)["unresolved"]] == ["ghost-plugin"] + + def test_the_reported_device_verdict_clears(self, client): + """The five findings the live device served, against its real state.""" + installed = ["football-scoreboard", "ledmatrix-weather", + "odds-ticker", "starlark-apps"] + c = client({"done": True, "unresolved": [ + {"plugin_id": p, "type": IN_CONFIG} for p in installed]}, + config={p: {"enabled": True} for p in installed}, + installed=installed) + assert _get(c)["unresolved"] == [] + + +class TestTheEndpointStaysRobust: + def test_an_empty_verdict_is_passed_through(self, client): + c = client({"done": True, "unresolved": []}) + assert _get(c) == {"done": True, "unresolved": []} + + def test_a_broken_config_manager_leaves_findings_untouched(self, client): + """Best-effort: a stale warning beats a failed endpoint.""" + c = client({"done": True, "unresolved": [ + {"plugin_id": "football-scoreboard", "type": IN_CONFIG}]}) + api_v3.config_manager.load_config.side_effect = OSError("config unreadable") + body = _get(c) + assert [e["plugin_id"] for e in body["unresolved"]] == ["football-scoreboard"] + + def test_a_run_still_in_progress_is_reported_as_such(self, client): + c = client({"done": False, "unresolved": []}) + assert _get(c)["done"] is False + + +class TestTheFilterUsesTheReconcilersOwnRules: + """A looser definition of "in config" or "on disk" clears findings that are + still true. Both cases raised by CodeRabbit on #553.""" + + def test_a_secrets_key_does_not_clear_an_in_config_finding(self, client): + # load_config() merges the secrets file in, so 'data' appears in the + # config dict -- but it is not a plugin. A plain set(config) would treat + # it as one and clear this finding. + c = client({"done": True, "unresolved": [ + {"plugin_id": "data", "type": IN_CONFIG}]}, + config={"data": {"mode": "nfl_recent"}}, + secrets={"data": {"mode": "nfl_recent"}}) + assert [e["plugin_id"] for e in _get(c)["unresolved"]] == ["data"] + + def test_a_system_key_does_not_clear_an_in_config_finding(self, client): + c = client({"done": True, "unresolved": [ + {"plugin_id": "display", "type": IN_CONFIG}]}, + config={"display": {"hardware": {}}}) + assert [e["plugin_id"] for e in _get(c)["unresolved"]] == ["display"] + + def test_a_non_dict_value_does_not_clear_an_in_config_finding(self, client): + c = client({"done": True, "unresolved": [ + {"plugin_id": "timezone", "type": IN_CONFIG}]}, + config={"timezone": "America/Chicago"}) + assert [e["plugin_id"] for e in _get(c)["unresolved"]] == ["timezone"] + + def test_an_unparseable_manifest_does_not_count_as_installed(self, client): + # Otherwise a corrupt file clears a live "in config but not on disk" + # finding on the strength of something nothing can read. + c = client({"done": True, "unresolved": [ + {"plugin_id": "broken-plugin", "type": ON_DISK}]}, + corrupt=["broken-plugin"]) + assert [e["plugin_id"] for e in _get(c)["unresolved"]] == ["broken-plugin"] + + def test_a_standalone_backup_dir_does_not_count_as_installed(self, client): + c = client({"done": True, "unresolved": [ + {"plugin_id": "weather.standalone-backup-20260101", "type": ON_DISK}]}, + installed=["weather.standalone-backup-20260101"]) + assert len(_get(c)["unresolved"]) == 1 diff --git a/test/test_starlark_display_contract.py b/test/test_starlark_display_contract.py index be05393d..b8713c24 100644 --- a/test/test_starlark_display_contract.py +++ b/test/test_starlark_display_contract.py @@ -6,6 +6,7 @@ frame for the whole display_duration instead of rotating on -- the same class of defect as a display() that returns None. """ +import json import sys import types from pathlib import Path @@ -197,6 +198,50 @@ class TestInstalledAppsTakeTurns: assert p.display(force_clear=True) is False +class TestManifestLockSidecarIsStable: + """_save_manifest and _update_manifest_safe must lock a sidecar file that + a manifest write never replaces. manifest.json itself is swapped for a + fresh inode by every atomic write (temp file + rename); a lock taken on + manifest.json directly would not exclude a second locker whose fresh + os.open() lands on that new inode right after a write, so two writers + could still race each other despite each believing it held the lock. + web_interface._starlark_manifest_lock (web_interface/blueprints/api_v3) + must lock this same sidecar name for that guarantee to hold across both + the plugin-loaded and standalone paths. + """ + + def _plugin_with_manifest_dir(self, manager_module, tmp_path): + p = _plugin(manager_module) + p.manifest_file = tmp_path / "manifest.json" + p.manifest_lock_file = tmp_path / "manifest.json.lock" + return p + + def test_the_lock_sidecar_is_not_the_manifest_file(self, manager_module, tmp_path): + p = self._plugin_with_manifest_dir(manager_module, tmp_path) + assert p.manifest_lock_file != p.manifest_file + assert p.manifest_lock_file.name == "manifest.json.lock" + + def test_the_locked_files_inode_survives_repeated_writes(self, manager_module, tmp_path): + p = self._plugin_with_manifest_dir(manager_module, tmp_path) + + assert p._save_manifest({"apps": {}}) + lock_ino_before = p.manifest_lock_file.stat().st_ino + + for app_id in ("one", "two", "three"): + def _update(manifest, app_id=app_id): + manifest.setdefault("apps", {})[app_id] = {"enabled": True} + assert p._update_manifest_safe(_update) + + lock_ino_after = p.manifest_lock_file.stat().st_ino + assert lock_ino_after == lock_ino_before, ( + "the locked file's inode changed across writes -- a locker that " + "opened it before this write and one that opens it after would " + "no longer contend for the same lock") + + manifest = json.loads(p.manifest_file.read_text()) + assert set(manifest["apps"]) == {"one", "two", "three"} + + class TestAnimationsRunAtFrameRate: def test_the_plugin_asks_for_the_high_fps_loop(self, manager_module): """The controller reads this attribute; a multi-frame app is otherwise diff --git a/test/test_vegas_density.py b/test/test_vegas_density.py index cd6db2bc..2338421f 100644 --- a/test/test_vegas_density.py +++ b/test/test_vegas_density.py @@ -470,7 +470,10 @@ class TestApiBoundsMatchValidate: """Extract the numeric_fields map from api_v3 without importing Flask.""" import ast import pathlib - src = pathlib.Path('web_interface/blueprints/api_v3.py').read_text() + # api_v3 is a package; the routes are spread across its modules. + src = '\n'.join( + p.read_text() for p in + sorted(pathlib.Path('web_interface/blueprints/api_v3').glob('*.py'))) tree = ast.parse(src) for node in ast.walk(tree): if not isinstance(node, ast.Assign): diff --git a/test/test_web_api.py b/test/test_web_api.py index 8cadf922..679bfd84 100644 --- a/test/test_web_api.py +++ b/test/test_web_api.py @@ -390,7 +390,7 @@ class TestConfigAPI: class TestSystemAPI: """Test system API endpoints.""" - @patch('web_interface.blueprints.api_v3.subprocess') + @patch('web_interface.blueprints.api_v3.system.subprocess') def test_get_system_status(self, mock_subprocess, client): """Test getting system status.""" # The endpoint returns 503 without psutil, which is an optional @@ -407,7 +407,7 @@ class TestSystemAPI: data = json.loads(response.data) assert 'service' in data or 'status' in data or 'active' in data - @patch('web_interface.blueprints.api_v3.subprocess') + @patch('web_interface.blueprints.api_v3.system.subprocess') def test_get_system_version(self, mock_subprocess, client): """Test getting system version.""" mock_result = MagicMock() @@ -421,7 +421,7 @@ class TestSystemAPI: data = json.loads(response.data) assert 'version' in data.get('data', {}) or 'version' in data - @patch('web_interface.blueprints.api_v3.subprocess') + @patch('web_interface.blueprints.api_v3.system.subprocess') def test_execute_system_action(self, mock_subprocess, client): """Test executing system action.""" mock_result = MagicMock() @@ -502,7 +502,7 @@ class TestDisplayAPI: if response.status_code in [200, 201]: assert api_v3.cache_manager.set.called - @patch('web_interface.blueprints.api_v3._ensure_cache_manager') + @patch('web_interface.blueprints.api_v3.display._ensure_cache_manager') def test_stop_on_demand_display(self, mock_ensure_cache, client): """Test stopping on-demand display.""" diff --git a/test/test_web_error_detail.py b/test/test_web_error_detail.py index 55f81db8..6324bc01 100644 --- a/test/test_web_error_detail.py +++ b/test/test_web_error_detail.py @@ -114,7 +114,11 @@ class TestHandlersCarryDetail: """ import ast - src = open("web_interface/blueprints/api_v3.py").read() + # api_v3 is a package; the routes are spread across its modules. + import pathlib + src = "\n".join( + p.read_text() for p in + sorted(pathlib.Path("web_interface/blueprints/api_v3").glob("*.py"))) tree = ast.parse(src) # This used to match one exact message string, so a handler that wrote diff --git a/test/web_interface/test_api_v3_backup_restore.py b/test/web_interface/test_api_v3_backup_restore.py index 7b67ca16..4c4a8304 100644 --- a/test/web_interface/test_api_v3_backup_restore.py +++ b/test/web_interface/test_api_v3_backup_restore.py @@ -139,6 +139,58 @@ class TestRequestValidation: assert post(client, options="{}").status_code == 200 assert restore.call_args[0][2].restore_config is True + def test_unknown_option_key_is_refused(self, client, restore): + # Regression: opts_dict.get('restore_secrets', True) silently + # ignores a typo'd/renamed key like "restoreSecrets" and keeps the + # True default, restoring secrets a caller's request clearly meant + # to exclude -- with no indication anything was wrong. + response = post(client, options=json.dumps({"restoreSecrets": False})) + assert response.status_code == 400 + assert "Unknown restore option" in response.get_json()["message"] + assert "restoreSecrets" in response.get_json()["message"] + restore.assert_not_called() + + def test_known_and_unknown_keys_together_are_refused(self, client, restore): + response = post(client, options=json.dumps({ + "restore_secrets": False, "restore_everything": True})) + assert response.status_code == 400 + restore.assert_not_called() + + +class TestOptionsAreBooleanAware: + """Regression: bool("false") is True in Python. + + Every restore flag used bare bool() coercion, so a caller that sends its + options as JSON strings rather than real booleans -- a form field, a + hand-built request -- had `{"restore_secrets": "false"}` restore secrets + anyway, the opposite of what was asked. Fixed with the same + string-aware `_coerce_to_bool` already used for checkbox-style config + fields elsewhere in this package (config.py, plugins.py). + """ + + @pytest.mark.parametrize("raw,expected", [ + ("false", False), ("False", False), ("FALSE", False), + ("0", False), + ("true", True), ("True", True), ("1", True), + ]) + def test_string_valued_flags_are_parsed_not_just_truthy( + self, client, restore, raw, expected): + post(client, options=json.dumps({"restore_secrets": raw})) + assert restore.call_args[0][2].restore_secrets is expected + + def test_a_string_false_does_not_restore_secrets(self, client, restore): + # The exact shape of the bug: a truthy non-empty string coerced by + # bare bool() to True regardless of its contents. + post(client, options=json.dumps({"restore_secrets": "false"})) + assert restore.call_args[0][2].restore_secrets is False + + def test_real_json_booleans_still_work(self, client, restore): + post(client, options=json.dumps({"restore_secrets": False, + "restore_config": True})) + options = restore.call_args[0][2] + assert options.restore_secrets is False + assert options.restore_config is True + class TestSuccess: def test_success_returns_the_result(self, client, restore): @@ -230,7 +282,7 @@ class TestPluginReinstall: def test_missing_store_manager_is_reported_per_plugin(self, client, restore): restore.return_value = FakeResult(plugins_to_install=[{"plugin_id": "clock"}]) api_v3.plugin_store_manager = None - with patch("web_interface.blueprints.api_v3.plugin_store_manager", None): + with patch("web_interface.blueprints.api_v3.backup.plugin_store_manager", None): body = post(client).get_json() assert body["data"]["plugins_failed"][0]["error"] == "Store manager unavailable" diff --git a/test/web_interface/test_calendar_oauth_endpoints.py b/test/web_interface/test_calendar_oauth_endpoints.py index 9520be5f..ef18b413 100644 --- a/test/web_interface/test_calendar_oauth_endpoints.py +++ b/test/web_interface/test_calendar_oauth_endpoints.py @@ -77,7 +77,10 @@ class TestTheRoutesExistAtAll: oauth = Path(project_root) / 'web_interface/static/v3/js/widgets/google-oauth.js' assert '/api/v3/plugins/calendar/list-calendars' in picker.read_text(encoding='utf-8') assert '/api/v3/plugins/calendar/authenticate' in oauth.read_text(encoding='utf-8') - source = (Path(project_root) / 'web_interface/blueprints/api_v3.py').read_text(encoding='utf-8') + # api_v3 is a package now, so the route strings are spread across its + # modules; read the whole directory rather than one file. + pkg = Path(project_root) / 'web_interface/blueprints/api_v3' + source = "\n".join(f.read_text(encoding='utf-8') for f in sorted(pkg.glob('*.py'))) assert "'/plugins/calendar/list-calendars'" in source assert "'/plugins/calendar/authenticate'" in source @@ -360,6 +363,22 @@ class TestDiagnosticsAreRedacted: assert 'hunter2' not in error, error assert '' in error, error + def test_script_stderr_is_redacted_in_the_log_too(self, tmp_path, caplog): + # Regression: the return value went through redact_text (asserted + # above), but the logger.error call right next to it logged `raw` + # verbatim -- a script that handles OAuth client secrets and can + # quote them in its stderr, landing unredacted in the log (CWE-532). + script = tmp_path / 'calendar_registration.py' + script.write_text( + 'import sys\n' + 'sys.stderr.write("boom client_secret=hunter2 more\\n")\n', + encoding='utf-8') + with caplog.at_level('ERROR', logger=mod.logger.name): + mod._run_calendar_registration(tmp_path, '') + logged = '\n'.join(r.getMessage() for r in caplog.records) + assert 'hunter2' not in logged, logged + assert '' in logged, logged + def test_a_failing_script_payload_is_redacted(self, client): (client.plugin_dir / 'credentials.json').write_text('{}', encoding='utf-8') (client.plugin_dir / 'calendar_registration.py').write_text( diff --git a/test/web_interface/test_config_logging_omits_secrets.py b/test/web_interface/test_config_logging_omits_secrets.py index 031eb4f1..250b215c 100644 --- a/test/web_interface/test_config_logging_omits_secrets.py +++ b/test/web_interface/test_config_logging_omits_secrets.py @@ -10,8 +10,8 @@ from pathlib import Path import pytest -SOURCE = (Path(__file__).resolve().parents[2] - / "web_interface" / "blueprints" / "api_v3.py") +API_V3_PKG = (Path(__file__).resolve().parents[2] + / "web_interface" / "blueprints" / "api_v3") #: Objects that still hold submitted secret values at the point these log #: calls run. Interpolating one whole into a log message leaks credentials. @@ -19,7 +19,7 @@ UNREDACTED = ("plugin_config", "secrets_config", "current_secrets") def _logging_lines(): - for number, line in enumerate(SOURCE.read_text(encoding="utf-8").splitlines(), 1): + for number, line in enumerate("\n".join(p.read_text(encoding="utf-8") for p in sorted(API_V3_PKG.glob("*.py"))).splitlines(), 1): stripped = line.strip() if stripped.startswith("#"): continue diff --git a/test/web_interface/test_secret_separation_parity.py b/test/web_interface/test_secret_separation_parity.py index 91367234..9486715e 100644 --- a/test/web_interface/test_secret_separation_parity.py +++ b/test/web_interface/test_secret_separation_parity.py @@ -15,8 +15,18 @@ from pathlib import Path from src.web_interface.secret_helpers import find_secret_fields, separate_secrets -API_V3_PATH = (Path(__file__).resolve().parents[2] - / "web_interface" / "blueprints" / "api_v3.py") +API_V3_PKG = (Path(__file__).resolve().parents[2] + / "web_interface" / "blueprints" / "api_v3") + + +def _api_v3_source() -> str: + """Every module of the api_v3 package as one string. + + It used to be a single file; the inline copies this guards against could + now reappear in any module of the package. + """ + return "\n".join(p.read_text(encoding="utf-8") + for p in sorted(API_V3_PKG.glob("*.py"))) # The migration is complete: any inline reimplementation is a regression. EXPECTED_INLINE_COPIES = 0 @@ -24,7 +34,7 @@ EXPECTED_INLINE_COPIES = 0 class TestNoInlineCopies: def _count(self, name: str) -> int: - source = API_V3_PATH.read_text(encoding="utf-8") + source = _api_v3_source() return len(re.findall(rf"^\s*def {name}\(", source, flags=re.MULTILINE)) def test_no_inline_find_secret_fields(self): @@ -46,7 +56,7 @@ class TestNoInlineCopies: def test_canonical_import_present(self): # Tripwire: the endpoints still need the helpers, so removing the # import means either dead secret handling or a new local copy. - source = API_V3_PATH.read_text(encoding="utf-8") + source = _api_v3_source() assert re.search( r"from src\.web_interface\.secret_helpers import .*find_secret_fields", source, diff --git a/test/web_interface/test_starlark_pixlet_routes.py b/test/web_interface/test_starlark_pixlet_routes.py index 24fab51e..2d42734b 100644 --- a/test/web_interface/test_starlark_pixlet_routes.py +++ b/test/web_interface/test_starlark_pixlet_routes.py @@ -63,14 +63,14 @@ class TestRoutesAreRegistered: class TestInstallPixlet: def test_it_does_not_404(self, client): - with patch('web_interface.blueprints.api_v3.subprocess.run') as run: + with patch('web_interface.blueprints.api_v3.starlark.subprocess.run') as run: run.return_value = MagicMock(returncode=0, stdout="ok", stderr="") resp = client.post('/api/v3/starlark/install-pixlet') assert resp.status_code != 404, "the route is still missing" assert resp.get_json().get('message') != 'Resource not found' def test_success_is_reported_in_the_shape_the_button_reads(self, client): - with patch('web_interface.blueprints.api_v3.subprocess.run') as run: + with patch('web_interface.blueprints.api_v3.starlark.subprocess.run') as run: run.return_value = MagicMock(returncode=0, stdout="done", stderr="") resp = client.post('/api/v3/starlark/install-pixlet') body = resp.get_json() @@ -78,7 +78,7 @@ class TestInstallPixlet: assert 'message' in body, "the JS shows data.message on success" def test_a_failed_download_says_why(self, client): - with patch('web_interface.blueprints.api_v3.subprocess.run') as run: + with patch('web_interface.blueprints.api_v3.starlark.subprocess.run') as run: run.return_value = MagicMock(returncode=1, stdout="", stderr="no such release") resp = client.post('/api/v3/starlark/install-pixlet') body = resp.get_json() @@ -88,7 +88,7 @@ class TestInstallPixlet: def test_a_timeout_is_reported_rather_than_hanging(self, client): import subprocess as sp - with patch('web_interface.blueprints.api_v3.subprocess.run', + with patch('web_interface.blueprints.api_v3.starlark.subprocess.run', side_effect=sp.TimeoutExpired(cmd='x', timeout=300)): resp = client.post('/api/v3/starlark/install-pixlet') assert resp.get_json()['status'] == 'error' @@ -395,6 +395,164 @@ class TestTheManifestStaysRelocatable: assert self._install(tmp_path)['star_file'] == 'demo.star' +class TestManifestLockPreventsLostUpdates: + """The standalone manifest fallback (no plugin instance loaded) reads, + mutates and writes manifest.json with no coordination across requests. + Each write is atomic on its own (temp file + rename), but two concurrent + read-modify-write cycles can still race: both read the same starting + manifest, and the second write silently discards whatever the first one + added. _starlark_manifest_lock closes that window -- mirrors + StarlarkAppsPlugin._update_manifest_safe, which already does this when + the plugin instance is loaded. + """ + + @pytest.fixture + def starlark_dir(self, tmp_path, monkeypatch): + from web_interface.blueprints import api_v3 as module + apps_dir = tmp_path / "starlark-apps" + apps_dir.mkdir() + monkeypatch.setattr(module, '_STARLARK_APPS_DIR', apps_dir) + monkeypatch.setattr(module, '_STARLARK_MANIFEST_FILE', apps_dir / 'manifest.json') + monkeypatch.setattr(module, '_STARLARK_MANIFEST_LOCK_FILE', apps_dir / 'manifest.json.lock') + module._write_starlark_manifest({'apps': {}}) + return apps_dir + + def test_the_locked_file_survives_a_manifest_write(self, starlark_dir): + """_write_starlark_manifest replaces manifest.json with a fresh inode + on every write (temp file + rename). If the lock were taken on that + same file, a second locker's fresh os.open() right after the rename + would land on the new inode -- unguarded, because only the old, + now-orphaned inode was ever locked -- and two writers could race + despite each believing it "held the lock" (see the docstring on + _starlark_manifest_lock). Locking a sidecar path that no write ever + touches or renames over closes that: the inode identity of what gets + locked must not change across writes. + """ + import os + + from web_interface.blueprints import api_v3 as module + + with module._starlark_manifest_lock(): + manifest = module._read_starlark_manifest() + lock_ino_before = os.stat(module._STARLARK_MANIFEST_LOCK_FILE).st_ino + + for app_id in ('one', 'two', 'three'): + with module._starlark_manifest_lock(): + manifest = module._read_starlark_manifest() + manifest.setdefault('apps', {})[app_id] = {'enabled': True} + assert module._write_starlark_manifest(manifest) + + lock_ino_after = os.stat(module._STARLARK_MANIFEST_LOCK_FILE).st_ino + assert lock_ino_after == lock_ino_before, ( + "the locked file's inode changed across writes -- a locker that " + "opened it before this write and one that opens it after would " + "no longer contend for the same lock") + + def test_two_concurrent_updates_are_both_kept(self, starlark_dir): + import threading + import time as _time + + from web_interface.blueprints import api_v3 as module + + def add_app(app_id): + with module._starlark_manifest_lock(): + manifest = module._read_starlark_manifest() + # Widen the window between read and write. Without the lock + # both threads read here before either writes, and whichever + # writes second overwrites the other's addition; with the + # lock, the second thread cannot even start its read until + # the first has written and released. + _time.sleep(0.05) + manifest.setdefault('apps', {})[app_id] = {'enabled': True} + module._write_starlark_manifest(manifest) + + threads = [threading.Thread(target=add_app, args=(app_id,)) + for app_id in ('a', 'b')] + for t in threads: + t.start() + for t in threads: + t.join(timeout=5) + + manifest = module._read_starlark_manifest() + assert set(manifest['apps']) == {'a', 'b'}, ( + "a concurrent update was lost: %r" % (manifest,)) + + def test_the_lock_is_reentrant_safe_across_sequential_calls(self, starlark_dir): + """Not reentrant within one thread -- just that using it twice in a + row (the ordinary case: one request, then the next) works cleanly + and does not leak the lock file descriptor or leave it locked.""" + from web_interface.blueprints import api_v3 as module + + for app_id in ('first', 'second'): + with module._starlark_manifest_lock(): + manifest = module._read_starlark_manifest() + manifest.setdefault('apps', {})[app_id] = {'enabled': True} + module._write_starlark_manifest(manifest) + + manifest = module._read_starlark_manifest() + assert set(manifest['apps']) == {'first', 'second'} + + +class TestConfigAndManifestStayInSync: + """Standalone-mode PUT /starlark/apps//config (no plugin instance + loaded) writes config.json and then the manifest. If the manifest write + fails after config.json was already written, the two disagree about + what was saved unless config.json is rolled back. + """ + + @pytest.fixture + def app_dir(self, tmp_path, monkeypatch): + from web_interface.blueprints import api_v3 as module + apps_dir = tmp_path / "starlark-apps" + apps_dir.mkdir() + monkeypatch.setattr(module, '_STARLARK_APPS_DIR', apps_dir) + monkeypatch.setattr(module, '_STARLARK_MANIFEST_FILE', apps_dir / 'manifest.json') + one_app_dir = apps_dir / 'demo' + one_app_dir.mkdir() + module._write_starlark_manifest({'apps': {'demo': {'name': 'Demo', 'enabled': True}}}) + return one_app_dir + + def test_manifest_write_failure_rolls_back_an_existing_config_json(self, client, app_dir): + config_file = app_dir / 'config.json' + config_file.write_text(json.dumps({'existing': 'value'})) + + with patch('web_interface.blueprints.api_v3._get_starlark_plugin', return_value=None), \ + patch('web_interface.blueprints.api_v3._write_starlark_manifest', return_value=False): + resp = client.put('/api/v3/starlark/apps/demo/config', + json={'new_field': 'x'}) + + assert resp.status_code == 500 + assert json.loads(config_file.read_text()) == {'existing': 'value'}, ( + "config.json kept the new value even though the manifest write " + "that was supposed to follow it failed") + + def test_manifest_write_failure_removes_a_freshly_created_config_json(self, client, app_dir): + config_file = app_dir / 'config.json' + assert not config_file.exists() + + with patch('web_interface.blueprints.api_v3._get_starlark_plugin', return_value=None), \ + patch('web_interface.blueprints.api_v3._write_starlark_manifest', return_value=False): + resp = client.put('/api/v3/starlark/apps/demo/config', + json={'new_field': 'x'}) + + assert resp.status_code == 500 + assert not config_file.exists(), ( + "config.json was left behind even though the manifest write " + "that was supposed to follow it failed") + + def test_success_updates_both_config_and_manifest(self, client, app_dir): + from web_interface.blueprints import api_v3 as module + + with patch('web_interface.blueprints.api_v3._get_starlark_plugin', return_value=None): + resp = client.put('/api/v3/starlark/apps/demo/config', + json={'new_field': 'x'}) + + assert resp.status_code == 200, resp.get_json() + assert json.loads((app_dir / 'config.json').read_text())['new_field'] == 'x' + manifest = json.loads(module._STARLARK_MANIFEST_FILE.read_text()) + assert manifest['apps']['demo']['config']['new_field'] == 'x' + + # --------------------------------------------------------------------------- # The store loaded, then stopped loading, and nothing anywhere said why. # diff --git a/test/web_interface/test_systemctl_sudoers_alignment.py b/test/web_interface/test_systemctl_sudoers_alignment.py index f29691f0..0013eccf 100644 --- a/test/web_interface/test_systemctl_sudoers_alignment.py +++ b/test/web_interface/test_systemctl_sudoers_alignment.py @@ -19,7 +19,12 @@ import re from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parents[2] -API_V3 = PROJECT_ROOT / "web_interface" / "blueprints" / "api_v3.py" +# api_v3 is a package; a sudo systemctl call can live in any of its modules. +API_V3_PKG = PROJECT_ROOT / "web_interface" / "blueprints" / "api_v3" + + +def _api_v3_source() -> str: + return "\n".join(p.read_text() for p in sorted(API_V3_PKG.glob("*.py"))) SUDOERS_SCRIPT = PROJECT_ROOT / "scripts" / "install" / "configure_web_sudo.sh" @@ -51,7 +56,7 @@ def _granted_systemctl_rules(script: str) -> set[tuple[str, str]]: def test_every_sudo_systemctl_call_is_granted() -> None: - calls = _sudo_systemctl_calls(API_V3.read_text()) + calls = _sudo_systemctl_calls(_api_v3_source()) rules = _granted_systemctl_rules(SUDOERS_SCRIPT.read_text()) assert calls, "expected to find sudo systemctl calls in api_v3.py" @@ -68,7 +73,7 @@ def test_every_sudo_systemctl_call_is_granted() -> None: def test_units_are_fully_qualified() -> None: """Privileged systemctl calls must name the unit as .service so they match the sudoers grants, which use the fully-qualified unit name.""" - calls = _sudo_systemctl_calls(API_V3.read_text()) + calls = _sudo_systemctl_calls(_api_v3_source()) unqualified = {(v, u) for v, u in calls if not u.endswith(".service")} assert not unqualified, ( "sudo systemctl calls must use fully-qualified .service unit names: " diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py deleted file mode 100644 index 10711053..00000000 --- a/web_interface/blueprints/api_v3.py +++ /dev/null @@ -1,10016 +0,0 @@ -from flask import Blueprint, request, jsonify, Response -import json -import os -import re -import stat -import sys -import shutil -import subprocess -import tempfile -import time -import hashlib -import uuid -import logging -from datetime import datetime -from pathlib import Path -from typing import Dict, Any, Optional, Tuple, Type -from urllib.parse import urlparse, urlunparse - -logger = logging.getLogger(__name__) - -# Import new infrastructure -from src.web_interface.api_helpers import success_response, error_response, validate_request_json -from src.web_interface.errors import ErrorCode -from src.web_interface.secret_helpers import (find_secret_fields, mask_all_secret_values, - merge_secrets, remove_empty_secrets, - separate_secrets, - strip_masked_values) -from src.web_interface.error_handler import describe_exception, redact_text -from src.plugin_system.operation_types import OperationType -from src.web_interface.validators import ( - validate_file_upload -) -from src.error_aggregator import get_error_aggregator -from src.common.permission_utils import install_requirements_file - -_SUDO = shutil.which('sudo') -_JOURNALCTL = shutil.which('journalctl') -_GIT = shutil.which('git') - -# Cap subprocess output returned to the browser — pip can produce MBs on build failures. -_MAX_OUTPUT_BYTES = 51_200 # 50 KB - - -def _truncate_output(stdout: str, stderr: str) -> str: - """Combine stdout+stderr and truncate to _MAX_OUTPUT_BYTES (keeping the tail).""" - combined = (stdout + stderr).strip() - if len(combined) > _MAX_OUTPUT_BYTES: - combined = '[...output truncated...]\n' + combined[-_MAX_OUTPUT_BYTES:] - return combined - - -def _pip_install_requirements(req_file: Path, timeout: int) -> subprocess.CompletedProcess: - """Install a requirements.txt file, preferring the vetted sudo wrapper so - the packages are visible to root-run ledmatrix.service — not just to - whichever non-root user runs this web process. Falls back to installing - for the current process only if the wrapper isn't set up yet (i.e. the - admin hasn't run scripts/install/configure_web_sudo.sh since upgrading), - so the button still does *something* useful rather than hard-failing. - - Thin wrapper around the shared implementation in permission_utils so the - Plugin Store's own dependency installation (store_manager.py) follows the - exact same root-visible install path instead of a divergent one. - """ - return install_requirements_file(req_file, timeout=timeout) - - -def _scrub_git_remote_url(url: str) -> str: - """Strip embedded username/password from an HTTPS remote URL before returning it to the UI.""" - try: - p = urlparse(url) - if p.scheme in ('http', 'https') and (p.username or p.password): - netloc = p.hostname or '' - if p.port: - netloc += f':{p.port}' - return urlunparse(p._replace(netloc=netloc)) - except Exception: - pass - return url - -# Will be initialized when blueprint is registered -# NOTE: the managers live on the blueprint object (app.py sets -# api_v3.config_manager / api_v3.plugin_manager). Deliberately not -# mirrored as module globals: a bare `config_manager` used to resolve to -# a None that was never assigned, which silently disabled the /health -# checks and made /display/current fall back to a hardcoded 128x64. -plugin_store_manager = None -saved_repositories_manager = None -cache_manager = None -schema_manager = None -operation_queue = None -plugin_state_manager = None -operation_history = None -sync_manager = None # Optional DisplaySyncManager instance (set by app.py if available) - -# Get project root directory (web_interface/../..) -PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent - -# System fonts that cannot be deleted (used by catalog API and delete endpoint) -SYSTEM_FONTS = frozenset([ - 'pressstart2p-regular', 'pressstart2p', - '4x6-font', '4x6', - '5by7.regular', '5by7', '5x7', - '5x8', '6x9', '6x10', '6x12', '6x13', '6x13b', '6x13o', - '7x13', '7x13b', '7x13o', '7x14', '7x14b', - '8x13', '8x13b', '8x13o', - '9x15', '9x15b', '9x18', '9x18b', - '10x20', - 'matrixchunky8', 'matrixlight6', 'tom-thumb', - 'clr6x12', 'helvr12', 'texgyre-27' -]) - -api_v3 = Blueprint('api_v3', __name__) - -def _get_plugin_version(plugin_id: str) -> str: - """Read the installed version from a plugin's manifest.json. - - Returns the version string on success, or '' if the manifest - cannot be read (missing, corrupt, permission denied, etc.). - """ - manifest_path = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id / "manifest.json" - try: - with open(manifest_path, 'r', encoding='utf-8') as f: - manifest = json.load(f) - return manifest.get('version', '') - except (FileNotFoundError, PermissionError, OSError) as e: - logger.warning("[PluginVersion] Could not read manifest for %s at %s: %s", plugin_id, manifest_path, e) - except json.JSONDecodeError as e: - logger.warning("[PluginVersion] Invalid JSON in manifest for %s at %s: %s", plugin_id, manifest_path, e) - return '' - -def _is_plugin_update_available(installed_version: str, latest_version: str) -> bool: - """Return True when the registry's ``latest_version`` is strictly newer - than the installed version. - - Thin alias for the shared comparator in - `src.plugin_system.compatibility.is_update_available` — the store's - `update_plugin` uses the same function, so the UI badge and the actual - reinstall decision can never disagree. - """ - from src.plugin_system.compatibility import is_update_available - return is_update_available(installed_version, latest_version) - -def _ensure_cache_manager(): - """Ensure cache manager is initialized.""" - global cache_manager - if cache_manager is None: - from src.cache_manager import CacheManager - cache_manager = CacheManager() - return cache_manager - -def _save_config_atomic(config_manager, config_data, create_backup=True): - """ - Save configuration using atomic save if available, fallback to regular save. - - Returns: - tuple: (success: bool, error_message: str or None) - """ - if hasattr(config_manager, 'save_config_atomic'): - result = config_manager.save_config_atomic(config_data, create_backup=create_backup) - if result.status.value != 'success': - return False, result.message - return True, None - else: - try: - config_manager.save_config(config_data) - return True, None - except Exception as e: - return False, str(e) - -def _coerce_to_bool(value): - """ - Coerce a form value to a proper Python boolean. - - HTML checkboxes send string values like "true", "on", "1" when checked. - This ensures we store actual booleans in config JSON, not strings. - - Args: - value: The form value (string, bool, int, or None) - - Returns: - bool: True if value represents a truthy checkbox state, False otherwise - """ - if value is None: - return False - if isinstance(value, bool): - return value - if isinstance(value, int): - return value == 1 - if isinstance(value, str): - return value.lower() in ('true', 'on', '1', 'yes') - return False - -def _get_display_service_status(): - """Return status information about the ledmatrix service.""" - try: - result = subprocess.run( - ['systemctl', 'is-active', 'ledmatrix'], - capture_output=True, - text=True, - timeout=3 - ) - return { - 'active': result.stdout.strip() == 'active', - 'returncode': result.returncode, - 'stdout': result.stdout.strip(), - 'stderr': result.stderr.strip() - } - except subprocess.TimeoutExpired: - return { - 'active': False, - 'returncode': -1, - 'stdout': '', - 'stderr': 'timeout' - } - except Exception as err: - return { - 'active': False, - 'returncode': -1, - 'stdout': '', - 'stderr': str(err) - } - -def _run_systemctl_command(args): - """Run a systemctl command safely.""" - try: - result = subprocess.run( - args, - capture_output=True, - text=True, - timeout=15 - ) - return { - 'returncode': result.returncode, - 'stdout': result.stdout, - 'stderr': result.stderr - } - except subprocess.TimeoutExpired: - return { - 'returncode': -1, - 'stdout': '', - 'stderr': 'timeout' - } - except Exception as err: - return { - 'returncode': -1, - 'stdout': '', - 'stderr': str(err) - } - -def _ensure_display_service_running(): - """Ensure the ledmatrix display service is running.""" - status = _get_display_service_status() - if status.get('active'): - status['started'] = False - return status - result = _run_systemctl_command(['sudo', 'systemctl', 'start', 'ledmatrix.service']) - service_status = _get_display_service_status() - result['started'] = result.get('returncode') == 0 - result['active'] = service_status.get('active') - result['status'] = service_status - return result - -def _stop_display_service(): - """Stop the ledmatrix display service.""" - result = _run_systemctl_command(['sudo', 'systemctl', 'stop', 'ledmatrix.service']) - status = _get_display_service_status() - result['active'] = status.get('active') - result['status'] = status - return result - -#: Field names whose value is a credential. Matched by name because this -#: endpoint returns the whole config, core keys included, and core config has -#: no schema to carry x-secret markers. -_CREDENTIAL_NAME_PARTS = ("password", "passwd", "secret", "token", "api_key", - "apikey", "access_key", "private_key", "client_secret") - - -def _looks_like_a_credential(name: str) -> bool: - lowered = name.lower() - return any(part in lowered for part in _CREDENTIAL_NAME_PARTS) - - -def _redact_credentials(value): - """A copy of `value` with credential-named fields blanked. - - /config/main returned the raw config to anyone who could reach the port, - and this interface has no authentication. On one rig that meant a 40-char - GitHub token, a 183-char Home Assistant token and five API keys were - readable by anything on the LAN. - - The x-secret masking used by the plugin config endpoints does not help - here: this endpoint never consults a schema, and core keys such as - github.api_token have no schema to mark. Matching on the field name is - blunt, but for a whole-config dump the right default is that anything - named like a credential does not leave the process. - - Blanked rather than removed, and safe to blank: POST /config/main merges - into the loaded config and only writes the keys it was given, so a client - that round-trips this response cannot erase a secret it never saw. - """ - if isinstance(value, dict): - return {k: ("" if _looks_like_a_credential(k) and not isinstance(v, (dict, list)) - else _redact_credentials(v)) - for k, v in value.items()} - if isinstance(value, list): - return [_redact_credentials(item) for item in value] - return value - - -@api_v3.route('/config/main', methods=['GET']) -def get_main_config(): - """Get main configuration, with credentials redacted.""" - try: - if not api_v3.config_manager: - return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - - config = api_v3.config_manager.load_config() - return jsonify({'status': 'success', 'data': _redact_credentials(config)}) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/config/schedule', methods=['GET']) -def get_schedule_config(): - """Get current schedule configuration""" - try: - if not api_v3.config_manager: - return error_response( - ErrorCode.CONFIG_LOAD_FAILED, - 'Config manager not initialized', - status_code=500 - ) - - config = api_v3.config_manager.load_config() - schedule_config = config.get('schedule', {}) - - return success_response(data=schedule_config) - except Exception as e: - logger.error("%s failed", request.path, exc_info=True) - return error_response( - ErrorCode.CONFIG_LOAD_FAILED, - "An error occurred; see logs for details", - details=describe_exception(e), - status_code=500 - ) - -def _validate_time_format(time_str): - """Validate time format is HH:MM""" - try: - datetime.strptime(time_str, '%H:%M') - return True, None - except (ValueError, TypeError): - return False, f"Invalid time format: {time_str}. Expected HH:MM format." - -@api_v3.route('/config/schedule', methods=['POST']) -def save_schedule_config(): - """Save schedule configuration""" - try: - if not api_v3.config_manager: - return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - - data = request.get_json(silent=True) - if not data: - return jsonify({'status': 'error', 'message': 'No data provided'}), 400 - - # Load current config - current_config = api_v3.config_manager.load_config() - - # Build schedule configuration - # Handle enabled checkbox - can be True, False, or 'on' - enabled_value = data.get('enabled', False) - if isinstance(enabled_value, str): - enabled_value = enabled_value.lower() in ('true', 'on', '1') - schedule_config = { - 'enabled': enabled_value - } - - mode = data.get('mode', 'global') - schedule_config['mode'] = mode - - if mode == 'global': - # Simple global schedule - start_time = data.get('start_time', '07:00') - end_time = data.get('end_time', '23:00') - - # Validate time formats - is_valid, error_msg = _validate_time_format(start_time) - if not is_valid: - return error_response( - ErrorCode.VALIDATION_ERROR, - error_msg, - status_code=400 - ) - - is_valid, error_msg = _validate_time_format(end_time) - if not is_valid: - return error_response( - ErrorCode.VALIDATION_ERROR, - error_msg, - status_code=400 - ) - - schedule_config['start_time'] = start_time - schedule_config['end_time'] = end_time - # Remove days config when switching to global mode - schedule_config.pop('days', None) - else: - # Per-day schedule - schedule_config['days'] = {} - # Remove global times when switching to per-day mode - schedule_config.pop('start_time', None) - schedule_config.pop('end_time', None) - days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] - enabled_days_count = 0 - - for day in days: - day_config = {} - enabled_key = f'{day}_enabled' - start_key = f'{day}_start' - end_key = f'{day}_end' - - # Check if day is enabled - if enabled_key in data: - enabled_val = data[enabled_key] - # Handle checkbox values that may come as 'on', True, or False - if isinstance(enabled_val, str): - day_config['enabled'] = enabled_val.lower() in ('true', 'on', '1') - else: - day_config['enabled'] = bool(enabled_val) - else: - # Default to enabled if not specified - day_config['enabled'] = True - - # Only add times if day is enabled - if day_config.get('enabled', True): - enabled_days_count += 1 - start_time = None - end_time = None - - if start_key in data and data[start_key]: - start_time = data[start_key] - else: - start_time = '07:00' - - if end_key in data and data[end_key]: - end_time = data[end_key] - else: - end_time = '23:00' - - # Validate time formats - is_valid, error_msg = _validate_time_format(start_time) - if not is_valid: - return error_response( - ErrorCode.VALIDATION_ERROR, - f"Invalid start time for {day}: {error_msg}", - status_code=400 - ) - - is_valid, error_msg = _validate_time_format(end_time) - if not is_valid: - return error_response( - ErrorCode.VALIDATION_ERROR, - f"Invalid end time for {day}: {error_msg}", - status_code=400 - ) - - day_config['start_time'] = start_time - day_config['end_time'] = end_time - - schedule_config['days'][day] = day_config - - # Validate that at least one day is enabled in per-day mode - if enabled_days_count == 0: - return error_response( - ErrorCode.VALIDATION_ERROR, - "At least one day must be enabled in per-day schedule mode", - status_code=400 - ) - - # Update and save config using atomic save - current_config['schedule'] = schedule_config - success, error_msg = _save_config_atomic(api_v3.config_manager, current_config, create_backup=True) - if not success: - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - f"Failed to save schedule configuration: {error_msg}", - status_code=500 - ) - - # Invalidate cache on config change - try: - from web_interface.cache import invalidate_cache - invalidate_cache() - except ImportError: - pass - - return success_response(message='Schedule configuration saved successfully') - except Exception as e: - import logging - logger.error("Error saving schedule config", exc_info=True) - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - "An error occurred; see logs for details", - - status_code=500, details=describe_exception(e) - ) - -@api_v3.route('/config/dim-schedule', methods=['GET']) -def get_dim_schedule_config(): - """Get current dim schedule configuration""" - import logging - import json - - if not api_v3.config_manager: - logging.error("[DIM SCHEDULE] Config manager not initialized") - return error_response( - ErrorCode.CONFIG_LOAD_FAILED, - 'Config manager not initialized', - status_code=500 - ) - - try: - config = api_v3.config_manager.load_config() - dim_schedule_config = config.get('dim_schedule', { - 'enabled': False, - 'dim_brightness': 30, - 'mode': 'global', - 'start_time': '20:00', - 'end_time': '07:00', - 'days': {} - }) - - return success_response(data=dim_schedule_config) - except FileNotFoundError as e: - logging.error(f"[DIM SCHEDULE] Config file not found: {e}", exc_info=True) - return error_response( - ErrorCode.CONFIG_LOAD_FAILED, - "Configuration file not found", - status_code=500 - ) - except json.JSONDecodeError as e: - logging.error(f"[DIM SCHEDULE] Invalid JSON in config file: {e}", exc_info=True) - return error_response( - ErrorCode.CONFIG_LOAD_FAILED, - "Configuration file contains invalid JSON", - status_code=500 - ) - except (IOError, OSError) as e: - logging.error(f"[DIM SCHEDULE] Error reading config file: {e}", exc_info=True) - return error_response( - ErrorCode.CONFIG_LOAD_FAILED, - "An error occurred; see logs for details", - status_code=500, details=describe_exception(e) - ) - except Exception as e: - logging.error(f"[DIM SCHEDULE] Unexpected error loading config: {e}", exc_info=True) - return error_response( - ErrorCode.CONFIG_LOAD_FAILED, - "An error occurred; see logs for details", - status_code=500, details=describe_exception(e) - ) - -@api_v3.route('/config/dim-schedule', methods=['POST']) -def save_dim_schedule_config(): - """Save dim schedule configuration""" - try: - if not api_v3.config_manager: - return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - - data = request.get_json(silent=True) - if not data: - return jsonify({'status': 'error', 'message': 'No data provided'}), 400 - - # Load current config - current_config = api_v3.config_manager.load_config() - - # Build dim schedule configuration - enabled_value = data.get('enabled', False) - if isinstance(enabled_value, str): - enabled_value = enabled_value.lower() in ('true', 'on', '1') - - # Validate and get dim_brightness - dim_brightness_raw = data.get('dim_brightness', 30) - try: - # Handle empty string or None - if dim_brightness_raw is None or dim_brightness_raw == '': - dim_brightness = 30 - else: - dim_brightness = int(dim_brightness_raw) - except (ValueError, TypeError, OverflowError): - return error_response( - ErrorCode.VALIDATION_ERROR, - "dim_brightness must be an integer between 0 and 100", - status_code=400 - ) - - if not 0 <= dim_brightness <= 100: - return error_response( - ErrorCode.VALIDATION_ERROR, - "dim_brightness must be between 0 and 100", - status_code=400 - ) - - dim_schedule_config = { - 'enabled': enabled_value, - 'dim_brightness': dim_brightness - } - - mode = data.get('mode', 'global') - dim_schedule_config['mode'] = mode - - if mode == 'global': - # Simple global schedule - start_time = data.get('start_time', '20:00') - end_time = data.get('end_time', '07:00') - - # Validate time formats - is_valid, error_msg = _validate_time_format(start_time) - if not is_valid: - return error_response( - ErrorCode.VALIDATION_ERROR, - error_msg, - status_code=400 - ) - - is_valid, error_msg = _validate_time_format(end_time) - if not is_valid: - return error_response( - ErrorCode.VALIDATION_ERROR, - error_msg, - status_code=400 - ) - - dim_schedule_config['start_time'] = start_time - dim_schedule_config['end_time'] = end_time - # Remove days config when switching to global mode - dim_schedule_config.pop('days', None) - else: - # Per-day schedule - dim_schedule_config['days'] = {} - # Remove global times when switching to per-day mode - dim_schedule_config.pop('start_time', None) - dim_schedule_config.pop('end_time', None) - days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] - enabled_days_count = 0 - - for day in days: - day_config = {} - enabled_key = f'{day}_enabled' - start_key = f'{day}_start' - end_key = f'{day}_end' - - # Check if day is enabled - if enabled_key in data: - enabled_val = data[enabled_key] - if isinstance(enabled_val, str): - day_config['enabled'] = enabled_val.lower() in ('true', 'on', '1') - else: - day_config['enabled'] = bool(enabled_val) - else: - day_config['enabled'] = True - - # Only add times if day is enabled - if day_config.get('enabled', True): - enabled_days_count += 1 - start_time = data.get(start_key) or '20:00' - end_time = data.get(end_key) or '07:00' - - # Validate time formats - is_valid, error_msg = _validate_time_format(start_time) - if not is_valid: - return error_response( - ErrorCode.VALIDATION_ERROR, - f"Invalid start time for {day}: {error_msg}", - status_code=400 - ) - - is_valid, error_msg = _validate_time_format(end_time) - if not is_valid: - return error_response( - ErrorCode.VALIDATION_ERROR, - f"Invalid end time for {day}: {error_msg}", - status_code=400 - ) - - day_config['start_time'] = start_time - day_config['end_time'] = end_time - - dim_schedule_config['days'][day] = day_config - - # Validate that at least one day is enabled in per-day mode - if enabled_days_count == 0: - return error_response( - ErrorCode.VALIDATION_ERROR, - "At least one day must be enabled in per-day dim schedule mode", - status_code=400 - ) - - # Update and save config using atomic save - current_config['dim_schedule'] = dim_schedule_config - success, error_msg = _save_config_atomic(api_v3.config_manager, current_config, create_backup=True) - if not success: - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - f"Failed to save dim schedule configuration: {error_msg}", - status_code=500 - ) - - # Invalidate cache on config change - try: - from web_interface.cache import invalidate_cache - invalidate_cache() - except ImportError: - pass - - return success_response(message='Dim schedule configuration saved successfully') - except Exception as e: - import logging - logger.error("Error saving dim schedule config", exc_info=True) - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - "An error occurred; see logs for details", - - status_code=500, details=describe_exception(e) - ) - -@api_v3.route('/config/main', methods=['POST']) -def save_main_config(): - """Save main configuration""" - try: - if not api_v3.config_manager: - return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - - # Try to get JSON data first, fallback to form data - data = None - if request.content_type == 'application/json': - data = request.get_json() - else: - # Handle form data - data = request.form.to_dict() - # Convert checkbox values - for key in ['web_display_autostart']: - if key in data: - data[key] = data[key] == 'on' - - if not data: - return jsonify({'status': 'error', 'message': 'No data provided'}), 400 - - # What arrives here is the config itself, and the headers carry the - # session cookie -- neither belongs in the journal, least of all at - # ERROR on every save. The shape of the request is the part with - # diagnostic value, so log that, at the level it deserves. - logger.debug("save_main_config: %s, %d top-level key(s)", - request.content_type or 'no content-type', len(data)) - - # Merge with existing config (similar to original implementation) - current_config = api_v3.config_manager.load_config() - - # Handle general settings - # Note: Checkboxes don't send data when unchecked, so we need to check if we're updating general settings - # If any general setting is present, we're updating the general tab - is_general_update = any(k in data for k in ['timezone', 'city', 'state', 'country', 'web_display_autostart', - 'auto_discover', 'auto_load_enabled', 'development_mode', 'plugins_directory']) - - if is_general_update: - # For checkbox: if not present in data during general update, it means unchecked - current_config['web_display_autostart'] = _coerce_to_bool(data.get('web_display_autostart')) - - if 'timezone' in data: - current_config['timezone'] = data['timezone'] - - # Device-wide scroll frame rate, read by plugins via - # BasePlugin.global_config. Bounds match ScrollHelper.set_target_fps, - # which clamps silently -- rejecting here instead means a value that - # would have been quietly altered is reported rather than appearing to - # save and then behaving differently. - if 'target_fps' in data and data['target_fps'] not in ('', None): - raw_target_fps = data['target_fps'] - # A JSON body can carry real floats and bools, where int() would - # silently truncate: 90.5 would save as 90, and true as 1. Reject - # them rather than storing a value the user did not ask for. Form - # posts arrive as strings, so '90.5' still fails in int() below. - if isinstance(raw_target_fps, (bool, float)): - return jsonify({ - 'status': 'error', - 'message': "Invalid value for target_fps: must be an integer" - }), 400 - try: - target_fps = int(raw_target_fps) - except (ValueError, TypeError, OverflowError): - return jsonify({ - 'status': 'error', - 'message': "Invalid value for target_fps: must be an integer" - }), 400 - if not (30 <= target_fps <= 200): - return jsonify({ - 'status': 'error', - 'message': "Invalid value for target_fps: must be between 30 and 200" - }), 400 - current_config['target_fps'] = target_fps - - # Handle location settings - if 'city' in data or 'state' in data or 'country' in data: - if 'location' not in current_config: - current_config['location'] = {} - if 'city' in data: - current_config['location']['city'] = data['city'] - if 'state' in data: - current_config['location']['state'] = data['state'] - if 'country' in data: - current_config['location']['country'] = data['country'] - - # Handle plugin system settings - if 'auto_discover' in data or 'auto_load_enabled' in data or 'development_mode' in data or 'plugins_directory' in data: - if 'plugin_system' not in current_config: - current_config['plugin_system'] = {} - - # Handle plugin system checkboxes - always set to handle unchecked state - # HTML checkboxes omit the key when unchecked, so missing key = unchecked = False - for checkbox in ['auto_discover', 'auto_load_enabled', 'development_mode']: - current_config['plugin_system'][checkbox] = _coerce_to_bool(data.get(checkbox)) - - # Handle plugins_directory - if 'plugins_directory' in data: - current_config['plugin_system']['plugins_directory'] = data['plugins_directory'] - - # Handle display settings - display_fields = ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'hardware_mapping', - 'gpio_slowdown', 'rp1_rio', 'scan_mode', 'disable_hardware_pulsing', 'inverse_colors', 'show_refresh_rate', - 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', 'use_short_date_format', - 'max_dynamic_duration_seconds', 'led_rgb_sequence', 'multiplexing', 'panel_type', - 'row_address_type', 'pixel_mapper_config', 'orientation'] - - if any(k in data for k in display_fields): - if 'display' not in current_config: - current_config['display'] = {} - if 'hardware' not in current_config['display']: - current_config['display']['hardware'] = {} - if 'runtime' not in current_config['display']: - current_config['display']['runtime'] = {} - - # Allowed values for validated string fields - LED_RGB_ALLOWED = {'RGB', 'RBG', 'GRB', 'GBR', 'BRG', 'BGR'} - PANEL_TYPE_ALLOWED = {'', 'FM6126A', 'FM6127'} - - # Validate led_rgb_sequence - if 'led_rgb_sequence' in data and data['led_rgb_sequence'] not in LED_RGB_ALLOWED: - return jsonify({'status': 'error', 'message': f"Invalid LED RGB sequence '{data['led_rgb_sequence']}'. Allowed values: {', '.join(sorted(LED_RGB_ALLOWED))}"}), 400 - - # Validate panel_type - if 'panel_type' in data and data['panel_type'] not in PANEL_TYPE_ALLOWED: - return jsonify({'status': 'error', 'message': f"Invalid panel type '{data['panel_type']}'. Allowed values: Standard (empty), FM6126A, FM6127"}), 400 - - # Validate multiplexing - if 'multiplexing' in data: - try: - mux_val = int(data['multiplexing']) - if mux_val < 0 or mux_val > 22: - return jsonify({'status': 'error', 'message': f"Invalid multiplexing value '{data['multiplexing']}'. Must be an integer from 0 to 22."}), 400 - except (ValueError, TypeError, OverflowError): - return jsonify({'status': 'error', 'message': f"Invalid multiplexing value '{data['multiplexing']}'. Must be an integer from 0 to 22."}), 400 - - # Validate pixel_mapper_config (free-form mapper string, e.g. "U-mapper;Rotate:90") - if 'pixel_mapper_config' in data and not isinstance(data['pixel_mapper_config'], str): - return jsonify({'status': 'error', 'message': 'pixel_mapper_config must be a string (e.g. "U-mapper;Rotate:90" or empty)'}), 400 - - # Validate orientation (physical mounting rotation; composed onto pixel_mapper_config at runtime) - ORIENTATION_ALLOWED = {'normal', '180'} - if 'orientation' in data and data['orientation'] not in ORIENTATION_ALLOWED: - return jsonify({'status': 'error', 'message': f"Invalid orientation '{data['orientation']}'. Allowed values: {', '.join(sorted(ORIENTATION_ALLOWED))}"}), 400 - - # Validate row_address_type - if 'row_address_type' in data: - try: - rat_val = int(data['row_address_type']) - if rat_val < 0 or rat_val > 4: - return jsonify({'status': 'error', 'message': f"Invalid row_address_type '{data['row_address_type']}'. Must be an integer from 0 to 4."}), 400 - except (ValueError, TypeError, OverflowError): - return jsonify({'status': 'error', 'message': f"Invalid row_address_type '{data['row_address_type']}'. Must be an integer from 0 to 4."}), 400 - - # Handle hardware settings - for field in ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'hardware_mapping', 'scan_mode', - 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', - 'led_rgb_sequence', 'multiplexing', 'panel_type', 'row_address_type', - 'pixel_mapper_config', 'orientation']: - if field in data: - if field in ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'scan_mode', - 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', - 'multiplexing', 'row_address_type']: - current_config['display']['hardware'][field] = int(data[field]) - else: - current_config['display']['hardware'][field] = data[field] - - # Handle runtime settings - if 'gpio_slowdown' in data: - current_config['display']['runtime']['gpio_slowdown'] = int(data['gpio_slowdown']) - if 'rp1_rio' in data: - try: - rp1_val = int(data['rp1_rio']) - if rp1_val not in (0, 1): - return jsonify({'status': 'error', 'message': "rp1_rio must be 0 (PIO) or 1 (RIO)"}), 400 - current_config['display']['runtime']['rp1_rio'] = rp1_val - except (ValueError, TypeError, OverflowError): - return jsonify({'status': 'error', 'message': "rp1_rio must be 0 or 1"}), 400 - - # Handle checkboxes - coerce to bool to ensure proper JSON types - for checkbox in ['disable_hardware_pulsing', 'inverse_colors', 'show_refresh_rate']: - current_config['display']['hardware'][checkbox] = _coerce_to_bool(data.get(checkbox)) - - # Handle display-level checkboxes (always set to handle unchecked state) - current_config['display']['use_short_date_format'] = _coerce_to_bool(data.get('use_short_date_format')) - - # Handle dynamic duration settings - if 'max_dynamic_duration_seconds' in data: - if 'dynamic_duration' not in current_config['display']: - current_config['display']['dynamic_duration'] = {} - current_config['display']['dynamic_duration']['max_duration_seconds'] = int(data['max_dynamic_duration_seconds']) - - # Handle double-sided display settings - double_sided_fields = ['double_sided_enabled', 'double_sided_copies', 'double_sided_axis'] - if any(k in data for k in double_sided_fields): - if 'display' not in current_config: - current_config['display'] = {} - if 'double_sided' not in current_config['display']: - current_config['display']['double_sided'] = {} - ds_config = current_config['display']['double_sided'] - - # Enabled checkbox: omitted from the form when unchecked. - # The Display form posts copies/axis on every save regardless of this - # checkbox, so when the feature is off we accept the values without - # rejecting the whole save — otherwise a stale copies/chain_length - # mismatch locks the user out of every other display setting. - enabled = _coerce_to_bool(data.get('double_sided_enabled')) - ds_config['enabled'] = enabled - - def _copies_fits_hardware(copies: int) -> Optional[str]: - """Error message if copies doesn't divide the panel evenly, else None.""" - # Use axis from this request if provided, else from stored config. - hw = current_config.get('display', {}).get('hardware', {}) - effective_axis = (data.get('double_sided_axis') - or current_config.get('display', {}).get('double_sided', {}).get('axis', 'horizontal')) - if effective_axis == 'horizontal': - chain_length = int(hw.get('chain_length', 2) or 2) - if chain_length % copies != 0: - return f"Double-sided copies ({copies}) must divide chain length ({chain_length}) evenly" - elif effective_axis == 'vertical': - parallel = int(hw.get('parallel', 1) or 1) - if parallel % copies != 0: - return f"Double-sided copies ({copies}) must divide parallel ({parallel}) evenly" - return None - - if 'double_sided_copies' in data and data['double_sided_copies'] not in ('', None): - copies = None - try: - copies = int(data['double_sided_copies']) - except (ValueError, TypeError, OverflowError): - if enabled: - return jsonify({'status': 'error', 'message': "Double-sided copies must be an integer"}), 400 - if copies is not None and not (2 <= copies <= 8): - if enabled: - return jsonify({'status': 'error', 'message': "Double-sided copies must be between 2 and 8"}), 400 - # Disabled: leave the stored value alone rather than writing junk. - copies = None - if copies is not None: - # Divisibility is a hardware-relational check — only meaningful - # when the feature is actually on. - if enabled: - fit_error = _copies_fits_hardware(copies) - if fit_error: - return jsonify({'status': 'error', 'message': fit_error}), 400 - ds_config['copies'] = copies - - if 'double_sided_axis' in data: - axis = data['double_sided_axis'] - if axis not in ('horizontal', 'vertical'): - if enabled: - return jsonify({'status': 'error', 'message': "Double-sided axis must be 'horizontal' or 'vertical'"}), 400 - else: - ds_config['axis'] = axis - - # Handle Vegas scroll mode settings - vegas_fields = ['vegas_scroll_enabled', 'vegas_scroll_speed', 'vegas_separator_width', - 'vegas_target_fps', 'vegas_buffer_ahead', 'vegas_plugin_order', 'vegas_excluded_plugins', - 'vegas_auto_trim', 'vegas_trim_threshold', 'vegas_content_padding', - 'vegas_min_plugin_width', 'vegas_lead_in_width', 'vegas_plugins_per_cycle', - 'vegas_max_plugin_width_ratio', 'vegas_dynamic_duration_enabled', - 'vegas_min_cycle_duration', 'vegas_max_cycle_duration', - 'vegas_intra_plugin_gap', 'vegas_render_width_pct', - 'vegas_min_content_separation', 'vegas_min_cut_gap', - 'vegas_continuous_scroll', 'vegas_extend_threshold_screens', - 'vegas_smooth_scroll', 'vegas_overflow_mode'] - - if any(k in data for k in vegas_fields): - if 'display' not in current_config: - current_config['display'] = {} - if 'vegas_scroll' not in current_config['display']: - current_config['display']['vegas_scroll'] = {} - - vegas_config = current_config['display']['vegas_scroll'] - - # Handle enabled checkbox - # HTML checkboxes omit the key entirely when unchecked, so if the form - # was submitted (any vegas field present) but enabled key is missing, - # the checkbox was unchecked and we should set enabled=False - vegas_config['enabled'] = _coerce_to_bool(data.get('vegas_scroll_enabled')) - vegas_config['auto_trim'] = _coerce_to_bool(data.get('vegas_auto_trim')) - vegas_config['dynamic_duration_enabled'] = _coerce_to_bool( - data.get('vegas_dynamic_duration_enabled')) - vegas_config['continuous_scroll'] = _coerce_to_bool( - data.get('vegas_continuous_scroll')) - vegas_config['smooth_scroll'] = _coerce_to_bool( - data.get('vegas_smooth_scroll')) - - # max_plugin_width_ratio is the one fractional setting, so it is - # handled outside the integer loop below. - if data.get('vegas_overflow_mode') not in ('', None): - mode = str(data['vegas_overflow_mode']).strip().lower() - if mode not in ('rotate', 'truncate'): - return jsonify({ - 'status': 'error', - 'message': "Invalid value for vegas_overflow_mode: " - "must be 'rotate' or 'truncate'" - }), 400 - vegas_config['overflow_mode'] = mode - - if data.get('vegas_extend_threshold_screens') not in ('', None): - try: - screens = float(data['vegas_extend_threshold_screens']) - except (ValueError, TypeError, OverflowError): - return jsonify({ - 'status': 'error', - 'message': "Invalid value for vegas_extend_threshold_screens: " - "must be a number" - }), 400 - if not (1.0 <= screens <= 10.0): - return jsonify({ - 'status': 'error', - 'message': "Invalid value for vegas_extend_threshold_screens: " - "must be between 1.0 and 10.0" - }), 400 - vegas_config['extend_threshold_screens'] = screens - - if data.get('vegas_max_plugin_width_ratio') not in ('', None): - try: - ratio = float(data['vegas_max_plugin_width_ratio']) - except (ValueError, TypeError, OverflowError): - return jsonify({ - 'status': 'error', - 'message': "Invalid value for vegas_max_plugin_width_ratio: " - "must be a number" - }), 400 - if not (0 <= ratio <= 20): - return jsonify({ - 'status': 'error', - 'message': "Invalid value for vegas_max_plugin_width_ratio: " - "must be between 0 and 20 (0 disables the cap)" - }), 400 - vegas_config['max_plugin_width_ratio'] = ratio - - # Handle numeric settings with validation. - # - # These bounds must match VegasModeConfig.validate(), which is what - # actually gates Vegas starting. Where they were looser, a value - # saved with a 200 and then made VegasModeCoordinator.start() bail - # out with only a log line, so the ticker silently never ran. - # Where they were tighter (scroll_speed capped at 100 against a - # slider that goes to 200), a legitimate value was rejected with a - # 400. See test_vegas_api_bounds_match_validate. - numeric_fields = { - 'vegas_scroll_speed': ('scroll_speed', 1, 200), - 'vegas_separator_width': ('separator_width', 0, 128), - 'vegas_intra_plugin_gap': ('intra_plugin_gap', 0, 128), - 'vegas_render_width_pct': ('render_width_pct', 10, 100), - 'vegas_min_content_separation': ('min_content_separation', 0, 256), - 'vegas_min_cut_gap': ('min_cut_gap', 1, 128), - 'vegas_target_fps': ('target_fps', 30, 200), - 'vegas_buffer_ahead': ('buffer_ahead', 1, 5), - 'vegas_trim_threshold': ('trim_threshold', 0, 254), - 'vegas_content_padding': ('content_padding', 0, 128), - 'vegas_min_plugin_width': ('min_plugin_width', 0, 512), - 'vegas_lead_in_width': ('lead_in_width', 0, 2048), - 'vegas_plugins_per_cycle': ('plugins_per_cycle', 1, 50), - 'vegas_min_cycle_duration': ('min_cycle_duration', 5, 3600), - 'vegas_max_cycle_duration': ('max_cycle_duration', 10, 3600), - } - for field_name, (config_key, min_val, max_val) in numeric_fields.items(): - if field_name in data: - raw_value = data[field_name] - # Skip empty strings (treat as "not provided") - if raw_value == '' or raw_value is None: - continue - try: - int_value = int(raw_value) - except (ValueError, TypeError, OverflowError): - return jsonify({ - 'status': 'error', - 'message': f"Invalid value for {field_name}: must be an integer" - }), 400 - if not (min_val <= int_value <= max_val): - return jsonify({ - 'status': 'error', - 'message': f"Invalid value for {field_name}: must be between {min_val} and {max_val}" - }), 400 - vegas_config[config_key] = int_value - - # Handle plugin order and exclusions (JSON arrays) - if 'vegas_plugin_order' in data: - try: - if isinstance(data['vegas_plugin_order'], str): - parsed = json.loads(data['vegas_plugin_order']) - else: - parsed = data['vegas_plugin_order'] - # Ensure result is a list - vegas_config['plugin_order'] = list(parsed) if isinstance(parsed, (list, tuple)) else [] - except (json.JSONDecodeError, TypeError, ValueError): - vegas_config['plugin_order'] = [] - - if 'vegas_excluded_plugins' in data: - try: - if isinstance(data['vegas_excluded_plugins'], str): - parsed = json.loads(data['vegas_excluded_plugins']) - else: - parsed = data['vegas_excluded_plugins'] - # Ensure result is a list - vegas_config['excluded_plugins'] = list(parsed) if isinstance(parsed, (list, tuple)) else [] - except (json.JSONDecodeError, TypeError, ValueError): - vegas_config['excluded_plugins'] = [] - - # Handle multi-display sync settings - sync_fields = ["sync_role", "sync_port", "sync_follower_position"] - if any(k in data for k in sync_fields): - if 'sync' not in current_config: - current_config['sync'] = {} - SYNC_ROLE_ALLOWED = {'standalone', 'leader', 'follower'} - if 'sync_role' in data: - role_val = str(data['sync_role']).lower() - if role_val not in SYNC_ROLE_ALLOWED: - return jsonify({'status': 'error', 'message': f"Invalid sync role '{role_val}'. Must be one of: standalone, leader, follower"}), 400 - current_config['sync']['role'] = role_val - if 'sync_port' in data: - try: - port_val = int(data['sync_port']) - if not (1024 <= port_val <= 65535): - return jsonify({'status': 'error', 'message': "sync_port must be between 1024 and 65535"}), 400 - current_config['sync']['port'] = port_val - except (ValueError, TypeError, OverflowError): - return jsonify({'status': 'error', 'message': "sync_port must be an integer"}), 400 - - if "sync_follower_position" in data: - pos_val = str(data["sync_follower_position"]).lower() - if pos_val not in {"left", "right"}: - return jsonify({"status": "error", "message": "sync_follower_position must be left or right"}), 400 - current_config["sync"]["follower_position"] = pos_val - - # Handle primary rotation order: must be a JSON array of plugin-id - # strings. Reject anything else with a 400 rather than silently - # coercing, so a buggy client can't clear or corrupt the saved order. - if 'plugin_rotation_order' in data: - raw_order = data.pop('plugin_rotation_order') - try: - parsed = json.loads(raw_order) if isinstance(raw_order, str) else raw_order - except (json.JSONDecodeError, TypeError, ValueError): - return jsonify({'status': 'error', - 'message': 'plugin_rotation_order must be valid JSON'}), 400 - if not isinstance(parsed, list) or not all(isinstance(p, str) for p in parsed): - return jsonify({'status': 'error', - 'message': 'plugin_rotation_order must be a list of plugin-id strings'}), 400 - if 'display' not in current_config: - current_config['display'] = {} - current_config['display']['plugin_rotation_order'] = parsed - - # Handle display durations. Popped from `data` (not just read) so - # they can never also fall through to the generic "remaining keys" - # merge near the end of this function, which would otherwise write - # them AGAIN as bogus top-level config keys (e.g. "clock_duration": 30 - # sitting at config root alongside the correct - # display.display_durations.clock_duration). - duration_fields = [k for k in list(data.keys()) - if k.endswith('_duration') or k in ('default_duration', 'transition_duration')] - if duration_fields: - if 'display' not in current_config: - current_config['display'] = {} - if 'display_durations' not in current_config['display']: - current_config['display']['display_durations'] = {} - - for field in duration_fields: - raw_value = data.pop(field) - try: - int_value = int(raw_value) - except (ValueError, TypeError, OverflowError): - return jsonify({'status': 'error', - 'message': f"Invalid duration for {field}: must be an integer"}), 400 - current_config['display']['display_durations'][field] = int_value - - # Per-mode durations from the Rotation & Durations page, posted as - # duration__ (mode keys are arbitrary plugin mode names, so - # they can't use the suffix convention above). Same pop-and-validate - # treatment, for the same reason. - mode_duration_fields = [k for k in list(data.keys()) if k.startswith('duration__')] - if mode_duration_fields: - if 'display' not in current_config: - current_config['display'] = {} - if 'display_durations' not in current_config['display']: - current_config['display']['display_durations'] = {} - - for field in mode_duration_fields: - raw_value = data.pop(field) - mode_key = field[len('duration__'):] - if not mode_key: - continue - try: - int_value = int(raw_value) - except (ValueError, TypeError, OverflowError): - return jsonify({'status': 'error', - 'message': f"Invalid duration for mode '{mode_key}': must be an integer"}), 400 - current_config['display']['display_durations'][mode_key] = int_value - - # Handle plugin configurations dynamically - # Any key that matches a plugin ID should be saved as plugin config - # This includes proper secret field handling from schema - plugin_keys_to_remove = [] - for key in data: - # Check if this key is a plugin ID - if api_v3.plugin_manager and key in api_v3.plugin_manager.plugin_manifests: - plugin_id = key - plugin_config = data[key] - - # Load plugin schema to identify secret fields (same logic as save_plugin_config) - secret_fields = set() - if api_v3.plugin_manager: - plugins_dir = api_v3.plugin_manager.plugins_dir - else: - plugin_system_config = current_config.get('plugin_system', {}) - plugins_dir_name = plugin_system_config.get('plugins_directory', 'plugin-repos') - if os.path.isabs(plugins_dir_name): - plugins_dir = Path(plugins_dir_name) - else: - plugins_dir = PROJECT_ROOT / plugins_dir_name - schema_path = plugins_dir / plugin_id / 'config_schema.json' - - if schema_path.exists(): - try: - with open(schema_path, 'r', encoding='utf-8') as f: - schema = json.load(f) - if 'properties' in schema: - secret_fields = find_secret_fields(schema['properties']) - except Exception as e: - logger.debug("Error reading schema for secret detection: %s", e) - - # Separate secrets from regular config (same logic as save_plugin_config) - regular_config, secrets_config = separate_secrets(plugin_config, secret_fields) - # The config form renders secrets masked, so every save posts - # them back blank. Without this the blank is merged over the - # stored value and the credential is destroyed by the act of - # changing an unrelated setting. A blank means "unchanged". - secrets_config = remove_empty_secrets(secrets_config) - - # PRE-PROCESSING: Preserve 'enabled' state if not in regular_config - # This prevents overwriting the enabled state when saving config from a form that doesn't include the toggle - if 'enabled' not in regular_config: - try: - if plugin_id in current_config and 'enabled' in current_config[plugin_id]: - regular_config['enabled'] = current_config[plugin_id]['enabled'] - elif api_v3.plugin_manager: - # Fallback to plugin instance if config doesn't have it - plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) - if plugin_instance: - regular_config['enabled'] = plugin_instance.enabled - # Final fallback: default to True if plugin is loaded (matches BasePlugin default) - if 'enabled' not in regular_config: - regular_config['enabled'] = True - except Exception as e: - logger.debug("Error preserving enabled state: %s", e) - # Default to True on error to avoid disabling plugins - regular_config['enabled'] = True - - # Get current secrets config - current_secrets = api_v3.config_manager.get_raw_file_content('secrets') - - # Deep merge regular config into main config - if plugin_id not in current_config: - current_config[plugin_id] = {} - current_config[plugin_id] = deep_merge(current_config[plugin_id], regular_config) - - # Deep merge secrets into secrets config - if secrets_config: - if plugin_id not in current_secrets: - current_secrets[plugin_id] = {} - # Lists merge by replacement, so deep_merge here wrote a - # blanked array straight over the stored credentials. - current_secrets[plugin_id] = merge_secrets( - current_secrets[plugin_id], secrets_config) - # Save secrets file - api_v3.config_manager.save_raw_file_content('secrets', current_secrets) - - # Mark for removal from data dict (already processed) - plugin_keys_to_remove.append(key) - - # Notify plugin of config change if loaded (with merged config including secrets) - try: - if api_v3.plugin_manager: - plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) - if plugin_instance: - # Reload merged config (includes secrets) and pass the plugin-specific section - merged_config = api_v3.config_manager.load_config() - plugin_full_config = merged_config.get(plugin_id, {}) - if hasattr(plugin_instance, 'on_config_change'): - plugin_instance.on_config_change(plugin_full_config) - except Exception as hook_err: - # Don't fail the save if hook fails - logger.warning("on_config_change failed: %s", hook_err) - - # Remove processed plugin keys from data (they're already in current_config) - for key in plugin_keys_to_remove: - del data[key] - - # Handle any remaining config keys - # System settings (timezone, city, etc.) are already handled above - # Plugin configs should use /api/v3/plugins/config endpoint, but we'll handle them here too for flexibility - for key in data: - # Skip system settings that are already handled above - if key in ['timezone', 'city', 'state', 'country', - 'web_display_autostart', 'auto_discover', - 'auto_load_enabled', 'development_mode', - 'plugins_directory', 'target_fps']: - continue - # Skip fields that are already handled above in their own named sections. - # Without this, every form field name lands as a top-level config key too. - if key in display_fields: - continue - if key in sync_fields: - continue - if key in vegas_fields: - continue - if key in double_sided_fields: - continue - # For any remaining keys (including plugin keys), use deep merge to preserve existing settings - if key in current_config and isinstance(current_config[key], dict) and isinstance(data[key], dict): - # Deep merge to preserve existing settings - current_config[key] = deep_merge(current_config[key], data[key]) - else: - current_config[key] = data[key] - - # Save the merged config using atomic save - success, error_msg = _save_config_atomic(api_v3.config_manager, current_config, create_backup=True) - if not success: - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - f"Failed to save configuration: {error_msg}", - status_code=500 - ) - - # Invalidate cache on config change - try: - from web_interface.cache import invalidate_cache - invalidate_cache() - except ImportError: - pass - - return success_response(message='Configuration saved successfully') - except Exception as e: - logger.error("Error saving config", exc_info=True) - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - "An error occurred; see logs for details", - status_code=500, details=describe_exception(e) - ) - -@api_v3.route('/config/secrets', methods=['GET']) -def get_secrets_config(): - """Get secrets configuration""" - try: - if not api_v3.config_manager: - return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - - config = api_v3.config_manager.get_raw_file_content('secrets') - # This interface has no authentication, and this file is nothing but - # credentials. It was handing all of them to anyone who could reach - # the port. Values are masked; empty and YOUR_* placeholders are left - # alone so a client can still tell "set" from "not set". - return jsonify({'status': 'success', - 'data': mask_all_secret_values(config)}) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/config/raw/main', methods=['POST']) -def save_raw_main_config(): - """Save raw main configuration JSON""" - try: - if not api_v3.config_manager: - return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - - # silent=True so a malformed body returns None instead of raising - # Werkzeug's own BadRequest, which would answer in a different - # shape than this API's. Distinguish the two causes: a body that - # was sent but does not parse is a different mistake from no body. - data = request.get_json(silent=True) - if data is None and request.get_data(): - return jsonify({'status': 'error', 'message': 'Invalid JSON in request body'}), 400 - if not data: - return jsonify({'status': 'error', 'message': 'No data provided'}), 400 - - # Save the raw config file - api_v3.config_manager.save_raw_file_content('main', data) - - return jsonify({'status': 'success', 'message': 'Main configuration saved successfully'}) - except Exception as e: - from src.exceptions import ConfigError - logger.error("Error saving raw main config", exc_info=True) - - # Extract more specific error message if it's a ConfigError - if isinstance(e, ConfigError): - error_message = 'An error occurred; see logs for details' - if hasattr(e, 'config_path') and e.config_path: - error_message = f"{error_message} (config_path: {e.config_path})" - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - error_message, - details=describe_exception(e), - - context={'config_path': e.config_path} if hasattr(e, 'config_path') and e.config_path else None, - status_code=500 - ) - else: - error_message = 'An error occurred; see logs for details' - return error_response( - ErrorCode.UNKNOWN_ERROR, - error_message, - details=describe_exception(e), - - status_code=500 - ) - -@api_v3.route('/config/raw/secrets', methods=['POST']) -def save_raw_secrets_config(): - """Save raw secrets configuration JSON""" - try: - if not api_v3.config_manager: - return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - - # See save_raw_main_config: silent parsing, with a sent-but-broken - # body reported separately from a missing one. - data = request.get_json(silent=True) - if data is None and request.get_data(): - return jsonify({'status': 'error', 'message': 'Invalid JSON in request body'}), 400 - if not data: - return jsonify({'status': 'error', 'message': 'No data provided'}), 400 - - # The GET above masks what it returns, and this endpoint's only client - # reads the whole file, edits one field and posts all of it back. So - # most of what arrives here is the mask, echoed rather than changed -- - # storing it verbatim would replace every untouched credential with - # eight bullets. Strip those, then merge onto what is already stored, - # which makes "unchanged" mean unchanged. - # - # The cost is that a secret can no longer be cleared by blanking it. - # That needs its own affordance; a control that erases credentials as - # a side effect of saving an unrelated one is not it. - current = api_v3.config_manager.get_raw_file_content('secrets') or {} - merged = deep_merge(current, strip_masked_values(data)) - api_v3.config_manager.save_raw_file_content('secrets', merged) - - # Reload GitHub token in plugin store manager if it exists - if api_v3.plugin_store_manager: - api_v3.plugin_store_manager.github_token = api_v3.plugin_store_manager._load_github_token() - - return jsonify({'status': 'success', 'message': 'Secrets configuration saved successfully'}) - except Exception as e: - from src.exceptions import ConfigError - logger.error("Error saving raw secrets config", exc_info=True) - - # Extract more specific error message if it's a ConfigError - if isinstance(e, ConfigError): - # ConfigError has a message attribute and may have context - error_message = 'An error occurred; see logs for details' - if hasattr(e, 'config_path') and e.config_path: - error_message = f"{error_message} (config_path: {e.config_path})" - else: - error_message = 'An error occurred; see logs for details' - - return jsonify({'status': 'error', 'message': error_message, - 'details': describe_exception(e)}), 500 - -@api_v3.route('/system/status', methods=['GET']) -def get_system_status(): - """Get system status""" - try: - # Check cache first (10 second TTL for system status) - try: - from web_interface.cache import get_cached, set_cached - cached_result = get_cached('system_status', ttl_seconds=10) - if cached_result is not None: - return jsonify({'status': 'success', 'data': cached_result}) - except ImportError: - # Cache not available, continue without caching - get_cached = None - set_cached = None - - # Import psutil for system monitoring - try: - import psutil - except ImportError: - # Fallback if psutil not available - return jsonify({ - 'status': 'error', - 'message': 'psutil not available for system monitoring' - }), 503 - - # Get system metrics using psutil - cpu_percent = psutil.cpu_percent(interval=0.1) # Short interval for responsiveness - memory = psutil.virtual_memory() - memory_percent = memory.percent - disk = psutil.disk_usage('/') - disk_percent = disk.percent - - # Calculate uptime - boot_time = psutil.boot_time() - uptime_seconds = time.time() - boot_time - uptime_hours = uptime_seconds / 3600 - uptime_days = uptime_hours / 24 - - # Format uptime string - if uptime_days >= 1: - uptime_str = f"{int(uptime_days)}d {int(uptime_hours % 24)}h" - elif uptime_hours >= 1: - uptime_str = f"{int(uptime_hours)}h {int((uptime_seconds % 3600) / 60)}m" - else: - uptime_str = f"{int(uptime_seconds / 60)}m" - - # Get CPU temperature (Raspberry Pi) - cpu_temp = None - try: - temp_file = '/sys/class/thermal/thermal_zone0/temp' - if os.path.exists(temp_file): - with open(temp_file, 'r') as f: - temp_millidegrees = int(f.read().strip()) - cpu_temp = temp_millidegrees / 1000.0 # Convert to Celsius - except (IOError, ValueError, OSError): - # Temperature sensor not available or error reading - cpu_temp = None - - # Get display service status - service_status = _get_display_service_status() - - status = { - 'timestamp': time.time(), - 'uptime': uptime_str, - 'uptime_seconds': int(uptime_seconds), - 'service_active': service_status.get('active', False), - 'cpu_percent': round(cpu_percent, 1), - 'memory_used_percent': round(memory_percent, 1), - 'memory_total_mb': round(memory.total / (1024 * 1024), 1), - 'memory_used_mb': round(memory.used / (1024 * 1024), 1), - # MemAvailable, not total-minus-used: it accounts for reclaimable - # page cache, so it is what actually predicts memory trouble. A - # board can read 70% "used" and be fine, or read the same and be - # about to fail fork(), and only this number tells them apart. - 'memory_available_mb': round(memory.available / (1024 * 1024), 1), - 'cpu_temp': round(cpu_temp, 1) if cpu_temp is not None else None, - 'disk_used_percent': round(disk_percent, 1), - 'disk_total_gb': round(disk.total / (1024 * 1024 * 1024), 1), - 'disk_used_gb': round(disk.used / (1024 * 1024 * 1024), 1) - } - - # Cache the result if available - if set_cached: - try: - set_cached('system_status', status, ttl_seconds=10) - except Exception: - pass # Cache write failed, but continue - - return jsonify({'status': 'success', 'data': status}) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/health', methods=['GET']) -def get_health(): - """Get system health status""" - try: - health_status = { - 'status': 'healthy', - 'timestamp': time.time(), - 'services': {}, - 'checks': {} - } - - # Check web interface service - # Stamp the start time before measuring against it -- reading it with a - # fallback of time.time() and only assigning afterwards made the very - # first call subtract two separate clock reads, reporting a small - # negative uptime. - if not hasattr(get_health, '_start_time'): - get_health._start_time = time.time() - health_status['services']['web_interface'] = { - 'status': 'running', - 'uptime_seconds': time.time() - get_health._start_time - } - - # Check display service - display_service_status = _get_display_service_status() - health_status['services']['display_service'] = { - 'status': 'active' if display_service_status.get('active') else 'inactive', - 'details': display_service_status - } - - # Check config file accessibility - try: - if api_v3.config_manager: - test_config = api_v3.config_manager.load_config() - health_status['checks']['config_file'] = { - 'status': 'accessible', - 'readable': True - } - else: - health_status['checks']['config_file'] = { - 'status': 'unknown', - 'readable': False - } - except Exception as e: - health_status['checks']['config_file'] = { - 'status': 'error', - 'readable': False, - 'error': 'see logs for details' - } - - # Check plugin system - try: - if api_v3.plugin_manager: - # Try to discover plugins (lightweight check) - plugin_count = len(api_v3.plugin_manager.get_available_plugins()) if hasattr(api_v3.plugin_manager, 'get_available_plugins') else 0 - health_status['checks']['plugin_system'] = { - 'status': 'operational', - 'plugin_count': plugin_count - } - else: - health_status['checks']['plugin_system'] = { - 'status': 'not_initialized' - } - except Exception as e: - health_status['checks']['plugin_system'] = { - 'status': 'error', - 'error': 'see logs for details' - } - - # Check hardware connectivity (if display manager available) - try: - snapshot_path = "/tmp/led_matrix_preview.png" - if os.path.exists(snapshot_path): - # Check if snapshot is recent (updated in last 60 seconds) - mtime = os.path.getmtime(snapshot_path) - age_seconds = time.time() - mtime - health_status['checks']['hardware'] = { - 'status': 'connected' if age_seconds < 60 else 'stale', - 'snapshot_age_seconds': round(age_seconds, 1) - } - else: - health_status['checks']['hardware'] = { - 'status': 'no_snapshot', - 'note': 'Display service may not be running' - } - except Exception as e: - health_status['checks']['hardware'] = { - 'status': 'unknown', - 'error': 'see logs for details' - } - - # Determine overall health - all_healthy = all( - check.get('status') in ['accessible', 'operational', 'connected', 'running', 'active'] - for check in health_status['checks'].values() - ) - - if not all_healthy: - health_status['status'] = 'degraded' - - return jsonify({'status': 'success', 'data': health_status}) - except Exception as e: - logger.error("%s failed", request.path, exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e), - 'data': {'status': 'unhealthy'} - }), 500 - -def _git_current_branch(project_dir): - """Current branch name, or '' when detached or git fails.""" - try: - r = subprocess.run(['git', 'branch', '--show-current'], - capture_output=True, text=True, timeout=10, cwd=str(project_dir)) - return r.stdout.strip() if r.returncode == 0 else '' - except (subprocess.TimeoutExpired, OSError): - return '' - - -def _git_upstream(project_dir): - """Configured upstream for the current branch (e.g. 'origin/main'), or ''.""" - try: - r = subprocess.run(['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], - capture_output=True, text=True, timeout=10, cwd=str(project_dir)) - return r.stdout.strip() if r.returncode == 0 else '' - except (subprocess.TimeoutExpired, OSError): - return '' - - -def _git_remote_branch_exists(project_dir, branch): - """True when origin/ exists locally as a remote-tracking ref.""" - if not branch: - return False - try: - r = subprocess.run( - ['git', 'show-ref', '--verify', '--quiet', f'refs/remotes/origin/{branch}'], - capture_output=True, text=True, timeout=10, cwd=str(project_dir)) - return r.returncode == 0 - except (subprocess.TimeoutExpired, OSError): - return False - - -def resolve_pull_command(project_dir): - """Work out how to pull, for branches with and without an upstream. - - A plain ``git pull --rebase`` fails outright on a branch that has no - upstream ("There is no tracking information for the current branch"), - which is easy to end up on: checking out a branch by name, restoring a - backup, or following an install guide that names one. The update button - then reports a failure the user cannot act on. - - ``--autostash`` is passed for the same reason. Rebase refuses to start - when any tracked file is modified, and on these installs something always - is: first_time_install.sh chmods five scripts that git tracked as 644, so - every machine that ran the installer carries five permanent mode changes - and the update button reports "cannot pull with rebase: You have unstaged - changes". Those modes are corrected in this commit, but a user cannot pull - the correction while the pull is what is blocked, and any other local edit - would reproduce it anyway. Autostash reapplies the changes afterwards. - - Returns ``(args, note, error)``. When ``origin/`` exists the pull - is made explicit against it, so the update proceeds and the branch is - given tracking information afterwards. - """ - upstream = _git_upstream(project_dir) - if upstream: - return ['git', 'pull', '--rebase', '--autostash'], '', None - - branch = _git_current_branch(project_dir) - if not branch: - return None, '', ( - "This checkout is in a detached HEAD state, so there is no branch " - "to update. Switch to a branch first (Tools -> Switch branch)." - ) - if _git_remote_branch_exists(project_dir, branch): - return ( - ['git', 'pull', '--rebase', '--autostash', 'origin', branch], - f"Branch '{branch}' had no upstream; pulled from origin/{branch} and set it as the upstream.", - None, - ) - return None, '', ( - f"Branch '{branch}' has no upstream and there is no origin/{branch} to " - f"pull from. Use Switch branch to move to a branch that exists on the " - f"remote, or push this one first." - ) - - -_BRANCH_NAME_RE = re.compile(r'[A-Za-z0-9._/-]{1,200}') - - -def is_valid_branch_name(name): - """Accept only plain branch names. - - This value becomes a subprocess argument, so anything exotic is refused - rather than escaped. '..' is excluded because it is range syntax to git. - """ - if not name or not _BRANCH_NAME_RE.fullmatch(name): - return False - return '..' not in name and not name.startswith('-') - - -def checkout_branch(project_dir, target, stash=False): - """Switch the checkout to `target`, returning (payload, http_status). - - Split out of the route so it can be tested against real repositories. - Attaches tracking when the branch exists on origin, so the next - Pull Latest is a plain `git pull` rather than the no-upstream fallback. - """ - target = (target or '').strip() - if not target: - return {'status': 'error', 'message': 'Branch name required'}, 400 - if not is_valid_branch_name(target): - return {'status': 'error', 'message': 'Invalid branch name'}, 400 - - try: - subprocess.run(['git', 'fetch', 'origin', '--prune'], - capture_output=True, text=True, timeout=60, cwd=project_dir) - - local_exists = subprocess.run( - ['git', 'show-ref', '--verify', '--quiet', f'refs/heads/{target}'], - capture_output=True, text=True, timeout=10, cwd=project_dir).returncode == 0 - remote_exists = _git_remote_branch_exists(project_dir, target) - if not local_exists and not remote_exists: - return {'status': 'error', - 'message': f"No branch '{target}' locally or on origin"}, 404 - - # Local edits block a checkout. Pull Latest already stashes for the - # same reason, so offer it here too -- but only when asked, never - # silently: putting someone's edits away unasked is worse than - # refusing the switch. - stash_note = '' - if stash: - stashed = subprocess.run(['git', 'stash', 'push', '-m', f'switch to {target}'], - capture_output=True, text=True, timeout=60, cwd=project_dir) - if stashed.returncode == 0 and 'No local changes' not in stashed.stdout: - stash_note = ' Local changes were stashed (recover them with git stash list).' - - if local_exists: - co = subprocess.run(['git', 'checkout', target], - capture_output=True, text=True, timeout=60, cwd=project_dir) - else: - # -B so a stale local ref does not block the checkout. - co = subprocess.run(['git', 'checkout', '-B', target, f'origin/{target}'], - capture_output=True, text=True, timeout=60, cwd=project_dir) - - if co.returncode != 0: - logger.warning("git checkout %s failed: %s", target, co.stderr) - return { - 'status': 'error', - 'message': f"Could not switch to '{target}'.", - # Keep git's full list of blocking files: naming them is the - # difference between an error the user can act on and one they - # cannot. - 'detail': (co.stderr or '').strip(), - 'can_retry_with_stash': 'would be overwritten by checkout' in (co.stderr or ''), - }, 200 - - if remote_exists: - subprocess.run(['git', 'branch', f'--set-upstream-to=origin/{target}', target], - capture_output=True, text=True, timeout=10, cwd=project_dir) - - logger.info("Switched checkout to branch %s", target) - return { - 'status': 'success', - 'message': f"Now on '{target}'.{stash_note} Use Pull Latest to fetch its newest code.", - }, 200 - except subprocess.TimeoutExpired: - return {'status': 'error', 'message': 'Timed out talking to git'}, 504 - except OSError as exc: - logger.error("checkout_branch failed: %s", exc, exc_info=True) - return {'status': 'error', 'message': 'Could not switch branch'}, 500 - - -def get_git_version(project_dir=None): - """Get git version information from the repository""" - if project_dir is None: - project_dir = PROJECT_ROOT - - try: - # Try to get tag description (e.g., v2.4-10-g123456) - result = subprocess.run( - ['git', 'describe', '--tags', '--dirty'], - capture_output=True, - text=True, - timeout=5, - cwd=str(project_dir) - ) - - if result.returncode == 0: - version_str = result.stdout.strip() - if re.match(r'^[a-zA-Z0-9._\-]+$', version_str): - return version_str - - # Fallback to short commit hash - result = subprocess.run( - ['git', 'rev-parse', '--short', 'HEAD'], - capture_output=True, - text=True, - timeout=5, - cwd=str(project_dir) - ) - - if result.returncode == 0: - version_str = result.stdout.strip() - if re.match(r'^[a-zA-Z0-9._\-]+$', version_str): - return version_str - - return 'Unknown' - except Exception: - return 'Unknown' - -@api_v3.route('/system/version', methods=['GET']) -def get_system_version(): - """Get LEDMatrix repository version""" - try: - version = get_git_version() - return jsonify({'status': 'success', 'data': {'version': version}}) - except Exception as e: - logger.error("get_system_version failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'Unable to retrieve version'}), 500 - -_update_check_cache: Dict[str, Any] = {'result': None, 'ts': 0.0} -_UPDATE_CHECK_TTL = 300 # 5 minutes — avoids a git fetch on every page load - -def _update_check_failed(detail: str) -> Dict[str, Any]: - """A check that could not run is not the same as being up to date. - - Reporting update_available=False on a git failure hides the banner, and - the banner is the only route to the update button -- so a checkout git - refuses to touch looks exactly like a current one, permanently. The most - common cause is an install performed as root: git then reports "dubious - ownership" and every command fails, including the fetch here. - """ - return {'update_available': False, 'remote_sha': 'unknown', - 'commits_behind': 0, 'check_failed': True, 'error': detail} - - -def _describe_git_failure(stderr: str) -> str: - """Turn git's stderr into something the user can act on.""" - text = (stderr or '').strip() - if 'dubious ownership' in text or 'detected dubious ownership' in text: - return ("This checkout is owned by a different user than the one " - "running the web interface, so git refuses to use it. It is " - "usually the result of installing as root. Fix the ownership " - "and the update will work: sudo chown -R $USER:$USER " - + str(PROJECT_ROOT)) - if 'could not resolve host' in text.lower() or 'network is unreachable' in text.lower(): - return "Could not reach GitHub to check for updates." - return "Could not check for updates: " + (text.splitlines()[0] if text else "git failed") - - -@api_v3.route('/system/check-update', methods=['GET']) -def check_for_update(): - """Check whether a newer LEDMatrix commit is available on origin/main.""" - now = time.time() - if _update_check_cache['result'] and now - _update_check_cache['ts'] < _UPDATE_CHECK_TTL: - return jsonify(_update_check_cache['result']) - - _safe: Dict[str, Any] = {'update_available': False, 'remote_sha': 'unknown', 'commits_behind': 0} - try: - cwd = str(PROJECT_ROOT) - fetch_result = subprocess.run( - ['git', 'fetch', 'origin', 'main', '--quiet'], - capture_output=True, timeout=10, cwd=cwd, - ) - if fetch_result.returncode != 0: - stderr = fetch_result.stderr.decode(errors='replace').strip() - logger.warning("check-update: git fetch failed (rc=%d): %s", - fetch_result.returncode, stderr) - failed = _update_check_failed(_describe_git_failure(stderr)) - _update_check_cache['result'] = failed - _update_check_cache['ts'] = now - return jsonify(failed) - local = subprocess.run( - ['git', 'rev-parse', 'HEAD'], - capture_output=True, text=True, timeout=5, cwd=cwd, - ).stdout.strip() - remote = subprocess.run( - ['git', 'rev-parse', 'origin/main'], - capture_output=True, text=True, timeout=5, cwd=cwd, - ).stdout.strip() - - if not local or not remote: - return jsonify(_safe) - - if local == remote: - result: Dict[str, Any] = {'update_available': False, 'remote_sha': remote, 'commits_behind': 0} - else: - count_str = subprocess.run( - ['git', 'rev-list', 'HEAD..origin/main', '--count'], - capture_output=True, text=True, timeout=5, cwd=cwd, - ).stdout.strip() - count = int(count_str) if count_str.isdigit() else 0 - result = {'update_available': count > 0, 'remote_sha': remote, 'commits_behind': count} - - _update_check_cache['result'] = result - _update_check_cache['ts'] = now - return jsonify(result) - except Exception as e: - logger.warning("check-update failed: %s", e) - return jsonify(_update_check_failed( - "Could not check for updates; see logs for details.")) - -#: sudo's own wording when it needs a password it cannot ask for. The web -#: interface runs unprivileged, so its systemctl/reboot/journalctl calls only -#: work once scripts/install/configure_web_sudo.sh has granted NOPASSWD -- -#: which first_time_install.sh does not do. That makes this the common case on -#: a fresh device, and "Action failed; see logs for details" named none of it, -#: while the log viewer was broken for the very same reason. -_SUDO_NEEDS_PASSWORD = ( - 'a password is required', - 'no tty present', - 'a terminal is required', -) - -_SUDO_HINT = ( - 'Passwordless sudo is not configured for the web interface user, so this ' - 'action cannot run. Run scripts/install/configure_web_sudo.sh as that user, ' - 'then retry.' -) - - -def _sudo_hint_for(text): - """An actionable hint when `text` is sudo refusing to prompt, else None.""" - lowered = (text or '').lower() - if any(marker in lowered for marker in _SUDO_NEEDS_PASSWORD): - return _SUDO_HINT - return None - - -@api_v3.route('/system/action', methods=['POST']) -def execute_system_action(): - """Execute system actions (start/stop/reboot/etc)""" - try: - # HTMX sends data as form data, not JSON - data = request.get_json(silent=True) or {} - if not data: - # Try to get from form data if JSON fails - data = { - 'action': request.form.get('action'), - 'mode': request.form.get('mode') - } - - if not data or 'action' not in data: - return jsonify({'status': 'error', 'message': 'Action required'}), 400 - - action = data['action'] - mode = data.get('mode') # For on-demand modes - - # Map actions to subprocess calls (similar to original implementation) - if action == 'start_display': - if mode: - # For on-demand modes, we would need to integrate with the display controller - # For now, just start the display service - try: - result = subprocess.run(['sudo', 'systemctl', 'start', 'ledmatrix.service'], - capture_output=True, text=True, timeout=10) - except subprocess.TimeoutExpired as e: - logger.error("start_display (%s) timed out: %s", mode, e) - return jsonify({'status': 'error', 'message': 'Command timed out', 'returncode': -1, 'stderr': 'timeout'}) - logger.info("start_display (%s) returned code %d", mode, result.returncode) - if result.returncode != 0 and result.stderr: - logger.error("start_display (%s) stderr: %s", mode, result.stderr.strip()) - resp = { - 'status': 'success' if result.returncode == 0 else 'error', - # This branch returns before the shared nonzero-result - # response below, so it needs the hint of its own or an - # on-demand start reports "Failed to start display" and - # says nothing about the sudo that actually refused it. - 'message': ( - 'Display started' if result.returncode == 0 - else _sudo_hint_for(result.stderr) or 'Failed to start display' - ), - } - if result.returncode != 0: - resp['returncode'] = result.returncode - resp['stderr'] = result.stderr.strip() - return jsonify(resp) - else: - result = subprocess.run(['sudo', 'systemctl', 'start', 'ledmatrix.service'], - capture_output=True, text=True, timeout=10) - elif action == 'stop_display': - result = subprocess.run(['sudo', 'systemctl', 'stop', 'ledmatrix.service'], - capture_output=True, text=True, timeout=10) - elif action == 'enable_autostart': - result = subprocess.run(['sudo', 'systemctl', 'enable', 'ledmatrix.service'], - capture_output=True, text=True, timeout=10) - elif action == 'disable_autostart': - result = subprocess.run(['sudo', 'systemctl', 'disable', 'ledmatrix.service'], - capture_output=True, text=True, timeout=10) - elif action == 'reboot_system': - result = subprocess.run(['sudo', 'reboot'], - capture_output=True, text=True, timeout=10) - elif action == 'shutdown_system': - result = subprocess.run(['sudo', 'poweroff'], - capture_output=True, text=True, timeout=10) - elif action == 'git_pull': - # Use PROJECT_ROOT instead of hardcoded path - project_dir = str(PROJECT_ROOT) - - # Decide how to pull BEFORE stashing. If this checkout cannot be - # updated at all, stashing first would put the user's local changes - # away for an update that was never going to run. - pull_args, upstream_note, pull_error = resolve_pull_command(project_dir) - if pull_error: - logger.warning("git pull not attempted: %s", pull_error) - return jsonify({'status': 'error', 'message': pull_error}) - - # Check if there are local changes that need to be stashed - # Exclude plugins directory - plugins are separate repos and shouldn't be stashed with base project - # Use --untracked-files=no to skip untracked files check (much faster with symlinked plugins) - try: - status_result = subprocess.run( - ['git', 'status', '--porcelain', '--untracked-files=no'], - capture_output=True, - text=True, - timeout=30, - cwd=project_dir - ) - # Filter out any changes in plugins directory - plugins are separate repositories - # Git status format: XY filename (where X is status of index, Y is status of work tree) - status_lines = [line for line in status_result.stdout.strip().split('\n') - if line.strip() and 'plugins/' not in line] - has_changes = bool('\n'.join(status_lines).strip()) - except subprocess.TimeoutExpired: - # If status check times out, assume there might be changes and proceed - # This is safer than failing the update - has_changes = True - status_result = type('obj', (object,), {'stdout': '', 'stderr': 'Status check timed out'})() - - stash_info = "" - - # Stash local changes if they exist (excluding plugins) - # Plugins are separate repositories and shouldn't be stashed with base project updates - if has_changes: - try: - # Use pathspec to exclude plugins directory from stash - stash_result = subprocess.run( - ['git', 'stash', 'push', '-m', 'LEDMatrix auto-stash before update', '--', ':!plugins'], - capture_output=True, - text=True, - timeout=30, - cwd=project_dir - ) - if stash_result.returncode == 0: - logger.debug("git stash: stashed local changes before pull") - stash_info = " Local changes were stashed." - else: - logger.warning("git stash failed before pull (returncode=%d)", stash_result.returncode) - except subprocess.TimeoutExpired: - logger.warning("git stash timed out, proceeding with pull") - - # Record HEAD before the pull so dependency changes can be detected - old_head = None - try: - _pre = subprocess.run(['git', 'rev-parse', 'HEAD'], - capture_output=True, text=True, timeout=10, cwd=project_dir) - if _pre.returncode == 0: - old_head = _pre.stdout.strip() - except subprocess.TimeoutExpired: - logger.warning("git rev-parse timed out before pull") - - # Whether the pull actually brought new code in. "Already up to - # date" is a success too, and prompting for a restart then would - # train users to ignore the prompt. - code_changed = False - - # Perform the git pull. Branches without an upstream were given - # an explicit "origin " above so the update still works. - result = subprocess.run( - pull_args, - capture_output=True, - text=True, - timeout=60, - cwd=project_dir - ) - - # Give the branch tracking information so the next pull is a plain - # `git pull` — otherwise every update repeats the fallback. - if result.returncode == 0 and upstream_note: - branch = _git_current_branch(project_dir) - if branch: - try: - subprocess.run( - ['git', 'branch', f'--set-upstream-to=origin/{branch}', branch], - capture_output=True, text=True, timeout=10, cwd=project_dir) - except (subprocess.TimeoutExpired, OSError) as exc: - logger.debug("could not set upstream for %s: %s", branch, exc) - - # Return custom response for git_pull - if result.returncode == 0: - pull_message = "Code updated successfully." - if has_changes: - pull_message = f"Code updated successfully. Local changes were automatically stashed.{stash_info}" - if result.stdout and "Already up to date" not in result.stdout: - pull_message = f"Code updated successfully.{stash_info}" - if upstream_note: - pull_message = f"{pull_message} {upstream_note}" - - # Keep Python dependencies in sync automatically: if the pull - # changed a requirements file, install it now — users updating - # from the web UI (most of them) never SSH in to pip install. - # Installs go through the same root-visible path as the - # Tools-tab buttons (_pip_install_requirements). - dep_notes = [] - try: - _post = subprocess.run(['git', 'rev-parse', 'HEAD'], - capture_output=True, text=True, timeout=10, cwd=project_dir) - new_head = _post.stdout.strip() if _post.returncode == 0 else None - if old_head and new_head and old_head != new_head: - code_changed = True - diff = subprocess.run( - ['git', 'diff', '--name-only', f'{old_head}..{new_head}'], - capture_output=True, text=True, timeout=15, cwd=project_dir) - changed = set(diff.stdout.split()) if diff.returncode == 0 else set() - for rel in ('requirements.txt', 'web_interface/requirements.txt'): - req_path = PROJECT_ROOT / rel - if rel not in changed or not req_path.exists(): - continue - # Each file's install is isolated: a timeout or - # OSError (e.g. the sudo wrapper/interpreter - # missing) on one file must not abort the other. - try: - r = _pip_install_requirements(req_path, timeout=180) - if r.returncode == 0: - dep_notes.append(f"Dependencies from {rel} updated.") - else: - dep_notes.append( - f"Dependency install from {rel} failed — " - "run Install Base Requirements from the Tools tab.") - logger.warning("post-update pip install failed for %s: %s", - rel, _truncate_output(r.stdout, r.stderr)) - except subprocess.TimeoutExpired: - dep_notes.append( - f"Dependency install from {rel} timed out — " - "run Install Base Requirements from the Tools tab.") - logger.warning("post-update pip install timed out for %s", rel) - except OSError as install_err: - dep_notes.append( - f"Dependency install from {rel} failed — " - "run Install Base Requirements from the Tools tab.") - logger.warning("post-update pip install errored for %s: %s", - rel, install_err) - except subprocess.TimeoutExpired: - logger.warning("post-update dependency sync timed out") - if dep_notes: - pull_message += " " + " ".join(dep_notes) - # A `git pull` restores built-in plugins (committed under - # plugin-repos/) even if the user uninstalled them. Re-remove - # any the user previously uninstalled so the update doesn't - # resurrect them. - if api_v3.plugin_store_manager: - try: - purged = api_v3.plugin_store_manager.purge_uninstalled_plugins() - if purged: - logger.info( - "Re-removed %d uninstalled plugin(s) restored by update: %s", - len(purged), ", ".join(purged), - ) - except (OSError, RuntimeError) as purge_err: - logger.warning("Post-update plugin purge failed: %s", purge_err) - else: - logger.warning("git pull failed (returncode=%d): %s", result.returncode, result.stderr) - # Show git's own first line: "check logs" leaves the user with - # nothing to act on, and these failures are usually actionable - # (conflicting local commits, no upstream, network). - detail = next((ln.strip() for ln in (result.stderr or '').splitlines() - if ln.strip()), '') - pull_message = f"Update failed: {detail}" if detail else "Update failed; check logs for details" - - # Nothing here restarts anything: the pull replaces files on - # disk while the display and web services keep running the code - # they loaded at boot. Without this the user is told the update - # succeeded and sees no change until they happen to reboot. - return jsonify({ - 'status': 'success' if result.returncode == 0 else 'error', - 'message': pull_message, - 'restart_required': bool(result.returncode == 0 and code_changed), - }) - elif action == 'checkout_branch': - # Switch branches from the Tools tab. Needed because a checkout - # that predates tracking (or a restored backup) can leave the pi - # on a branch the update button cannot pull. - result_payload, http_status = checkout_branch( - str(PROJECT_ROOT), data.get('branch') or '', stash=bool(data.get('stash'))) - return jsonify(result_payload), http_status - - elif action == 'restart_display_service': - result = subprocess.run(['sudo', 'systemctl', 'restart', 'ledmatrix.service'], - capture_output=True, text=True, timeout=10) - elif action == 'restart_web_service': - # Try to restart the web service (assuming it's ledmatrix-web.service) - result = subprocess.run(['sudo', 'systemctl', 'restart', 'ledmatrix-web.service'], - capture_output=True, text=True, timeout=10) - elif action == 'install_base_requirements': - # Base + web interface requirements: flask-compress and friends - # live in web_interface/requirements.txt, not the root file. - req_files = [f for f in (PROJECT_ROOT / 'requirements.txt', - PROJECT_ROOT / 'web_interface' / 'requirements.txt') - if f.exists()] - if not req_files: - return jsonify({'status': 'error', 'message': 'No requirements.txt found at project root'}) - outputs = [] - all_ok = True - for req_file in req_files: - label = req_file.relative_to(PROJECT_ROOT) - # Isolate each file's install: a timeout or OSError on one - # (e.g. requirements.txt) must not abort the rest of the - # loop (e.g. web_interface/requirements.txt never attempted). - try: - result = _pip_install_requirements(req_file, timeout=120) - all_ok = all_ok and result.returncode == 0 - outputs.append(f"== {label} ==\n" + _truncate_output(result.stdout, result.stderr)) - except subprocess.TimeoutExpired: - all_ok = False - outputs.append(f"== {label} ==\nTimed out after 120s") - logger.warning("install_base_requirements timed out for %s", label) - except OSError as install_err: - all_ok = False - outputs.append(f"== {label} ==\nFailed: {install_err}") - logger.warning("install_base_requirements errored for %s: %s", label, install_err) - return jsonify({ - 'status': 'success' if all_ok else 'error', - 'message': 'Base requirements installed successfully' if all_ok else 'pip install failed', - 'output': "\n".join(outputs) - }) - elif action == 'install_plugin_requirements': - active_pm = getattr(api_v3, 'plugin_manager', None) - if active_pm: - plugins_dir = Path(active_pm.plugins_dir) - else: - _cm = getattr(api_v3, 'config_manager', None) - _cfg = _cm.load_config() if _cm else {} - _dir_name = _cfg.get('plugin_system', {}).get('plugins_directory', 'plugin-repos') - plugins_dir = Path(_dir_name) if os.path.isabs(_dir_name) else PROJECT_ROOT / _dir_name - results = [] - if plugins_dir.exists(): - for p in sorted(plugins_dir.iterdir()): - req = p / 'requirements.txt' - if p.is_dir() and req.exists(): - try: - r = _pip_install_requirements(req, timeout=60) - results.append({ - 'plugin': p.name, - 'ok': r.returncode == 0, - 'output': _truncate_output(r.stdout, r.stderr) - }) - except subprocess.TimeoutExpired: - results.append({'plugin': p.name, 'ok': False, 'output': 'pip install timed out'}) - except OSError as exc: - results.append({'plugin': p.name, 'ok': False, 'output': exc.strerror or 'OS error'}) - ok_count = sum(1 for r in results if r['ok']) - all_ok = all(r['ok'] for r in results) if results else True - return jsonify({ - 'status': 'success' if all_ok else 'error', - 'message': f'Processed {len(results)} plugin(s) — {ok_count} succeeded' if results else 'No plugin requirements.txt files found', - 'details': results - }) - elif action == 'force_git_reset': - if not _GIT: - return jsonify({'status': 'error', 'message': 'git not found on this system'}), 503 - project_dir = str(PROJECT_ROOT) - fetch = subprocess.run( - [_GIT, 'fetch', 'origin'], - capture_output=True, text=True, timeout=30, cwd=project_dir - ) - if fetch.returncode != 0: - return jsonify({'status': 'error', 'message': 'git fetch failed', 'output': fetch.stderr.strip()}) - reset = subprocess.run( - [_GIT, 'reset', '--hard', 'origin/main'], - capture_output=True, text=True, timeout=30, cwd=project_dir - ) - return jsonify({ - 'status': 'success' if reset.returncode == 0 else 'error', - 'message': 'Reset to origin/main successfully' if reset.returncode == 0 else 'git reset failed', - 'output': (reset.stdout + reset.stderr).strip() - }) - elif action == 'clear_pycache': - cleared = 0 - failed = 0 - for d in PROJECT_ROOT.rglob('__pycache__'): - if d.is_dir(): - try: - shutil.rmtree(d) - cleared += 1 - except OSError: - failed += 1 - msg = f'Cleared {cleared} __pycache__ directories' - if failed: - msg += f' ({failed} could not be removed)' - return jsonify({'status': 'success', 'message': msg}) - else: - return jsonify({'status': 'error', 'message': 'Unknown action'}), 400 - - logger.info("system action '%s' returncode=%d", action, result.returncode) - if result.returncode != 0 and result.stderr: - logger.error("system action '%s' stderr: %s", action, result.stderr.strip()) - resp = { - 'status': 'success' if result.returncode == 0 else 'error', - 'message': 'Action completed' if result.returncode == 0 else 'Action failed; check logs for details', - } - if result.returncode != 0: - resp['returncode'] = result.returncode - resp['stderr'] = result.stderr.strip() - hint = _sudo_hint_for(result.stderr) - if hint: - resp['message'] = hint - return jsonify(resp) - - except subprocess.TimeoutExpired as e: - logger.error("system action '%s' timed out: %s", action, e) - return jsonify({'status': 'error', 'message': 'Command timed out', 'returncode': -1, 'stderr': 'timeout'}) - except Exception as e: - logger.error("execute_system_action failed: %s", e, exc_info=True) - detail = describe_exception(e) - resp = { - 'status': 'error', - 'message': _sudo_hint_for(detail) or 'Action failed; see logs for details', - 'details': detail, - } - return jsonify(resp), 500 - -@api_v3.route('/system/git-info', methods=['GET']) -def get_git_info(): - """Return branch, dirty state, recent commits and remote URL for the Tools tab.""" - if not _GIT: - return jsonify({'status': 'error', 'message': 'git not found on this system'}), 503 - d = str(PROJECT_ROOT) - try: - branch = subprocess.run([_GIT, 'branch', '--show-current'], capture_output=True, text=True, timeout=10, cwd=d) - if branch.returncode != 0: - return jsonify({'status': 'error', 'message': f'git branch failed: {branch.stderr.strip()}'}), 500 - - status = subprocess.run([_GIT, 'status', '--short', '--untracked-files=no'], capture_output=True, text=True, timeout=15, cwd=d) - if status.returncode != 0: - return jsonify({'status': 'error', 'message': f'git status failed: {status.stderr.strip()}'}), 500 - - log = subprocess.run([_GIT, 'log', '--oneline', '-5'], capture_output=True, text=True, timeout=10, cwd=d) - remote = subprocess.run([_GIT, 'remote', 'get-url', 'origin'], capture_output=True, text=True, timeout=10, cwd=d) - branch_name = branch.stdout.strip() - upstream = _git_upstream(d) - return jsonify({ - 'branch': branch_name, - 'dirty': bool(status.stdout.strip()), - 'status': status.stdout.strip(), - 'recent_commits': log.stdout.strip() if log.returncode == 0 else '', - 'remote_url': _scrub_git_remote_url(remote.stdout.strip()) if remote.returncode == 0 else '', - # Surfaced so the Tools tab can warn before the user clicks Pull - # Latest, rather than after it fails. - 'upstream': upstream, - 'can_pull': bool(upstream) or _git_remote_branch_exists(d, branch_name), - }) - except Exception as e: - logger.error("get_git_info failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'Failed to get git info'}), 500 - - -@api_v3.route('/system/git-branches', methods=['GET']) -def get_git_branches(): - """List branches available to switch to, for the Tools tab picker.""" - if not _GIT: - return jsonify({'status': 'error', 'message': 'git not found on this system'}), 503 - d = str(PROJECT_ROOT) - try: - # Refresh remote refs so a branch created since the last fetch shows up. - subprocess.run([_GIT, 'fetch', 'origin', '--prune'], - capture_output=True, text=True, timeout=60, cwd=d) - - local = subprocess.run([_GIT, 'for-each-ref', '--format=%(refname:short)', 'refs/heads'], - capture_output=True, text=True, timeout=15, cwd=d) - remote = subprocess.run([_GIT, 'for-each-ref', '--format=%(refname:short)', 'refs/remotes/origin'], - capture_output=True, text=True, timeout=15, cwd=d) - if local.returncode != 0: - return jsonify({'status': 'error', 'message': 'Could not list branches'}), 500 - - local_names = [b for b in local.stdout.split() if b] - remote_names = [] - for ref in remote.stdout.split() if remote.returncode == 0 else []: - name = ref.split('origin/', 1)[-1] - # origin/HEAD is a symbolic alias, not a branch a user can pick. - if name and name != 'HEAD' and name not in local_names: - remote_names.append(name) - - return jsonify({ - 'status': 'success', - 'current': _git_current_branch(d), - 'upstream': _git_upstream(d), - 'local': sorted(local_names), - 'remote_only': sorted(remote_names), - }) - except subprocess.TimeoutExpired: - return jsonify({'status': 'error', 'message': 'Timed out talking to the remote'}), 504 - except OSError as e: - logger.error("get_git_branches failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'Failed to list branches'}), 500 - - -@api_v3.route('/hardware/status', methods=['GET']) -def get_hardware_status(): - """Return LED matrix hardware initialization status written by display_manager at startup.""" - status_path = "/tmp/led_matrix_hw_status.json" # nosec B108 - try: - with open(status_path) as f: - hw_data = json.load(f) - return jsonify({"status": "success", "data": hw_data}) - except FileNotFoundError: - return jsonify({"status": "success", "data": {"ok": None, "error": "Display service not yet started"}}) - except PermissionError: - logger.warning("Permission denied reading hardware status file; display service may be running as a different user") - return jsonify({"status": "success", "data": {"ok": False, "error": "Hardware status temporarily unavailable"}}) - except json.JSONDecodeError: - logger.error("Failed to parse hardware status file", exc_info=True) - return jsonify({"status": "success", "data": {"ok": False, "error": "Hardware status file corrupted"}}) - except Exception: - logger.error("Unexpected error reading hardware status", exc_info=True) - return jsonify({"status": "error", "message": "Unable to read hardware status"}), 500 - -@api_v3.route('/display/current', methods=['GET']) -def get_display_current(): - """Get current display state""" - try: - import base64 - from PIL import Image - import io - - snapshot_path = "/tmp/led_matrix_preview.png" - - # Get display dimensions from config - try: - if api_v3.config_manager: - main_config = api_v3.config_manager.load_config() - hardware_config = main_config.get('display', {}).get('hardware', {}) - cols = hardware_config.get('cols', 64) - chain_length = hardware_config.get('chain_length', 2) - rows = hardware_config.get('rows', 32) - parallel = hardware_config.get('parallel', 1) - width = cols * chain_length - height = rows * parallel - else: - width = 128 - height = 64 - except Exception: - width = 128 - height = 64 - - # Try to read snapshot file - image_data = None - if os.path.exists(snapshot_path): - try: - with Image.open(snapshot_path) as img: - # Convert to PNG and encode as base64 - buffer = io.BytesIO() - img.save(buffer, format='PNG') - image_data = base64.b64encode(buffer.getvalue()).decode('utf-8') - except Exception as img_err: - # File might be being written or corrupted, return None - pass - - display_data = { - 'timestamp': time.time(), - 'width': width, - 'height': height, - 'image': image_data # Base64 encoded image data or None if unavailable - } - return jsonify({'status': 'success', 'data': display_data}) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/display/modes', methods=['GET']) -def get_display_modes(): - """Every display mode that can be requested on-demand, with its plugin. - - /plugins/installed carries no mode information, so anything driving the - display from outside the web UI (the Home Assistant MQTT bridge, a script) - had to read each plugin's manifest.json off disk and reimplement the - fallbacks in PluginManager.get_plugin_display_modes to do it. This is the - same list the force-display dialog offers, from the source that owns it. - - Knowing each mode's plugin_id also matters because /display/on-demand/start - falls back to find_plugin_for_mode when plugin_id is omitted, and that - lookup only sees modes declared in a static manifest -- a plugin whose - modes are generated (each installed Starlark app is one) 404s there. - Sending the plugin_id from this list skips the lookup entirely. - - Query params: - include_disabled: '1' to list modes of disabled plugins too. They can - still be requested on-demand -- the controller enables the plugin - for the duration -- so they are reported with enabled: false - rather than omitted. - """ - try: - if not api_v3.plugin_manager: - return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - - # Discovery is lazy and normally triggered by whichever endpoint runs - # first, which is a person opening the dashboard. A caller that never - # visits it would otherwise see an empty list. - api_v3.plugin_manager.discover_plugins() - - include_disabled = request.args.get('include_disabled') in ('1', 'true', 'True') - full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {} - - modes = [] - for plugin_id, manifest in sorted(api_v3.plugin_manager.plugin_manifests.items()): - # A hand-edited or migrated config.json can hold a non-dict under a - # plugin id; DisplayController._reconcile guards the same shape, so - # it happens in practice. Without this, .get() raises AttributeError, - # the loop aborts and the endpoint answers 500 with no modes at all - # -- one bad section would blank every entity the MQTT bridge builds - # from this list. - plugin_config = full_config.get(plugin_id) - if not isinstance(plugin_config, dict): - if plugin_config is not None: - logger.warning( - "Config for plugin %r is %s, not an object; treating it as disabled", - plugin_id, type(plugin_config).__name__) - plugin_config = {} - enabled = bool(plugin_config.get('enabled', False)) - if not enabled and not include_disabled: - continue - plugin_name = (manifest or {}).get('name') or plugin_id - plugin_modes = api_v3.plugin_manager.get_plugin_display_modes(plugin_id) or [plugin_id] - for mode in plugin_modes: - # A single-mode plugin's mode is the plugin, so its own name is - # the readable label. Multi-mode plugins have no per-mode name - # anywhere, so the raw mode string is the only thing to show. - modes.append({ - 'mode': mode, - 'plugin_id': plugin_id, - 'plugin_name': plugin_name, - 'name': plugin_name if len(plugin_modes) == 1 else mode, - 'enabled': enabled, - }) - - return jsonify({'status': 'success', 'data': {'modes': modes}}) - except Exception as exc: - # describe_exception, not a bare message: test_web_error_detail.py - # enforces that every handler here returns it, because a device whose - # storage is failing otherwise answers "see logs for details" from the - # log viewer too. It redacts credentials out of the exception text. - # CodeQL flags this as stack-trace exposure across all ~75 handlers; - # it is the project's deliberate, reviewed trade-off. - logger.error('Error in get_display_modes', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 - -@api_v3.route('/display/on-demand/status', methods=['GET']) -def get_on_demand_status(): - """Return the current on-demand display state.""" - try: - cache = _ensure_cache_manager() - state = cache.get('display_on_demand_state', max_age=120) - if state is None: - state = { - 'active': False, - 'status': 'idle', - 'last_updated': None - } - service_status = _get_display_service_status() - return jsonify({ - 'status': 'success', - 'data': { - 'state': state, - 'service': service_status - } - }) - except Exception as exc: - logger.error('Error in get_on_demand_status', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 - -@api_v3.route('/display/on-demand/start', methods=['POST']) -def start_on_demand_display(): - """Request the display controller to run a specific plugin on-demand.""" - try: - data = request.get_json(silent=True) or {} - plugin_id = data.get('plugin_id') - mode = data.get('mode') - duration = data.get('duration') - pinned = bool(data.get('pinned', False)) - start_service = data.get('start_service', True) - - if not plugin_id and not mode: - return jsonify({'status': 'error', 'message': 'plugin_id or mode is required'}), 400 - - resolved_plugin = plugin_id - resolved_mode = mode - - if api_v3.plugin_manager: - if resolved_plugin and resolved_plugin not in api_v3.plugin_manager.plugin_manifests: - return jsonify({'status': 'error', 'message': f'Plugin {resolved_plugin} not found'}), 404 - - if resolved_plugin and not resolved_mode: - modes = api_v3.plugin_manager.get_plugin_display_modes(resolved_plugin) - resolved_mode = modes[0] if modes else resolved_plugin - elif resolved_mode and not resolved_plugin: - resolved_plugin = api_v3.plugin_manager.find_plugin_for_mode(resolved_mode) - if not resolved_plugin: - return jsonify({'status': 'error', 'message': f'Mode {resolved_mode} not found'}), 404 - - # Note: On-demand can work with disabled plugins - the display controller - # will temporarily enable them during initialization if needed - # We don't block the request here, but log it for debugging - if api_v3.config_manager and resolved_plugin: - config = api_v3.config_manager.load_config() - plugin_config = config.get(resolved_plugin, {}) - if 'enabled' in plugin_config and not plugin_config.get('enabled', False): - logger.info( - "On-demand request for disabled plugin '%s' - will be temporarily enabled", - resolved_plugin, - ) - - # Set the on-demand request in cache FIRST (before starting service) - # This ensures the request is available when the service starts/restarts - cache = _ensure_cache_manager() - request_id = data.get('request_id') or str(uuid.uuid4()) - request_payload = { - 'request_id': request_id, - 'action': 'start', - 'plugin_id': resolved_plugin, - 'mode': resolved_mode, - 'duration': duration, - 'pinned': pinned, - 'timestamp': time.time() - } - cache.set('display_on_demand_request', request_payload) - - # Check if display service is running (or will be started) - service_status = _get_display_service_status() - service_was_running = service_status.get('active', False) - - # Stop the display service first to ensure clean state when we will restart it - if service_was_running and start_service: - import time as time_module - logger.debug("Stopping display service before starting on-demand mode") - _stop_display_service() - # Wait a brief moment for the service to fully stop - time_module.sleep(1.5) - logger.debug("Display service stopped, now starting with on-demand request") - - if not service_status.get('active') and not start_service: - return jsonify({ - 'status': 'error', - 'message': 'Display service is not running. Please start the display service or enable "Start Service" option.', - 'service_status': service_status - }), 400 - - service_result = None - if start_service: - service_result = _ensure_display_service_running() - # Check if service actually started - if service_result and not service_result.get('active'): - return jsonify({ - 'status': 'error', - 'message': 'Failed to start display service. Please check service logs or start it manually.', - 'service_result': service_result - }), 500 - - # Service was restarted (or started fresh) with on-demand request in cache - # The display controller will read the request during initialization or when it polls - - response_data = { - 'request_id': request_id, - 'plugin_id': resolved_plugin, - 'mode': resolved_mode, - 'duration': duration, - 'pinned': pinned, - 'service': service_result - } - return jsonify({'status': 'success', 'data': response_data}) - except Exception as exc: - logger.error('Error in start_on_demand_display', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 - -@api_v3.route('/display/on-demand/stop', methods=['POST']) -def stop_on_demand_display(): - """Request the display controller to stop on-demand mode.""" - try: - data = request.get_json(silent=True) or {} - stop_service = data.get('stop_service', False) - - # Set the stop request in cache FIRST - # The display controller will poll this and restart without the on-demand filter - cache = _ensure_cache_manager() - request_id = data.get('request_id') or str(uuid.uuid4()) - request_payload = { - 'request_id': request_id, - 'action': 'stop', - 'timestamp': time.time() - } - cache.set('display_on_demand_request', request_payload) - - # Note: The display controller's _clear_on_demand() will handle the restart - # to restore normal operation with all plugins - - service_result = None - if stop_service: - service_result = _stop_display_service() - - return jsonify({ - 'status': 'success', - 'data': { - 'request_id': request_id, - 'service': service_result - } - }) - except Exception as exc: - logger.error('Error in stop_on_demand_display', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 - -@api_v3.route('/plugins/installed', methods=['GET']) -def get_installed_plugins(): - """Get installed plugins""" - try: - if not api_v3.plugin_manager or not api_v3.plugin_store_manager: - return jsonify({'status': 'error', 'message': 'Plugin managers not initialized'}), 500 - - import json - from pathlib import Path - - # Re-discover plugins to ensure we have the latest list - # This handles cases where plugins are added/removed after app startup - api_v3.plugin_manager.discover_plugins() - - # Get all installed plugin info from the plugin manager - all_plugin_info = api_v3.plugin_manager.get_all_plugin_info() - - # Load config once before the loop (not per-plugin) - full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {} - - def _build_plugin_entry(plugin_info): - plugin_id = plugin_info.get('id') - try: - return _build_plugin_entry_inner(plugin_info, plugin_id) - except Exception: - logger.exception("Error building plugin entry for %s — skipping", plugin_id) - return None - - def _build_plugin_entry_inner(plugin_info, plugin_id): - # Capture runtime state (state machine + error context) before the - # manifest merge below can shadow the 'state' key. get_all_plugin_info - # attaches this via PluginStateManager.get_state_info(); surfacing it - # lets the UI show *why* a plugin isn't running instead of just - # 'loaded: false'. - state_info = plugin_info.get('state') - plugin_state = None - plugin_error_info = None - if isinstance(state_info, dict): - plugin_state = state_info.get('state') - plugin_error_info = state_info.get('error_info') - - # Re-read manifest from disk to ensure we have the latest metadata - manifest_path = Path(api_v3.plugin_manager.plugins_dir) / plugin_id / "manifest.json" - if manifest_path.exists(): - try: - with open(manifest_path, 'r', encoding='utf-8') as f: - fresh_manifest = json.load(f) - if isinstance(fresh_manifest, dict): - plugin_info.update(fresh_manifest) - else: - logger.debug("Manifest for %s is not a dict (%s) — skipping merge", - plugin_id, type(fresh_manifest).__name__) - except (FileNotFoundError, PermissionError, json.JSONDecodeError) as e: - logger.debug("Could not read fresh manifest for %s: %s", plugin_id, e) - - # Enabled status: config is source of truth, fall back to instance - enabled = None - plugin_config = full_config.get(plugin_id, {}) - if 'enabled' in plugin_config: - enabled = bool(plugin_config['enabled']) - - # Single get_plugin() call shared for both enabled fallback and Vegas mode - plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) - if enabled is None: - enabled = plugin_instance.enabled if plugin_instance else True - - # Verified + latest published version from registry (no network call) - store_info = api_v3.plugin_store_manager.get_registry_info(plugin_id) - verified = store_info.get('verified', False) if store_info else False - latest_version = store_info.get('latest_version', '') if store_info else '' - installed_version = plugin_info.get('version', '') - update_available = _is_plugin_update_available(installed_version, latest_version) - - # Local git info (single subprocess on cache miss, zero on hit) - plugin_path = Path(api_v3.plugin_manager.plugins_dir) / plugin_id - local_git_info = api_v3.plugin_store_manager._get_local_git_info(plugin_path) if plugin_path.exists() else None - - if local_git_info: - sha = local_git_info.get('sha', '') - last_commit = local_git_info.get('short_sha') or (sha[:7] if sha else None) - branch = local_git_info.get('branch') - last_updated = local_git_info.get('date_iso') or local_git_info.get('date') - else: - last_updated = plugin_info.get('last_updated') - last_commit = plugin_info.get('last_commit') or plugin_info.get('last_commit_sha') - branch = plugin_info.get('branch') - if store_info: - last_updated = last_updated or store_info.get('last_updated') or store_info.get('last_updated_iso') - last_commit = last_commit or store_info.get('last_commit') or store_info.get('last_commit_sha') - branch = branch or store_info.get('branch') or store_info.get('default_branch') - - last_commit_message = plugin_info.get('last_commit_message') - if store_info and not last_commit_message: - last_commit_message = store_info.get('last_commit_message') - - # Vegas mode from instance, overridden by explicit config value - vegas_mode = None - vegas_content_type = None - if plugin_instance: - try: - if hasattr(plugin_instance, 'get_vegas_display_mode'): - mode = plugin_instance.get_vegas_display_mode() - vegas_mode = mode.value if hasattr(mode, 'value') else str(mode) - except (AttributeError, TypeError, ValueError) as e: - logger.debug("[%s] Failed to get vegas_display_mode: %s", plugin_id, e) - try: - if hasattr(plugin_instance, 'get_vegas_content_type'): - vegas_content_type = plugin_instance.get_vegas_content_type() - except (AttributeError, TypeError, ValueError) as e: - logger.debug("[%s] Failed to get vegas_content_type: %s", plugin_id, e) - - if 'vegas_mode' in plugin_config: - vegas_mode = plugin_config['vegas_mode'] - - return { - 'id': plugin_id, - 'name': plugin_info.get('name', plugin_id), - 'version': plugin_info.get('version', ''), - 'latest_version': latest_version, - 'update_available': update_available, - 'author': plugin_info.get('author', 'Unknown'), - 'category': plugin_info.get('category', 'General'), - 'description': plugin_info.get('description', 'No description available'), - 'tags': plugin_info.get('tags', []), - 'enabled': enabled, - 'verified': verified, - 'loaded': plugin_info.get('loaded', False), - 'state': plugin_state, - 'error_info': plugin_error_info, - 'last_updated': last_updated, - 'last_commit': last_commit, - 'last_commit_message': last_commit_message, - 'branch': branch, - 'web_ui_actions': plugin_info.get('web_ui_actions', []), - 'vegas_mode': vegas_mode, - 'vegas_content_type': vegas_content_type, - } - - from concurrent.futures import ThreadPoolExecutor - with ThreadPoolExecutor(max_workers=8) as executor: - results = list(executor.map(_build_plugin_entry, all_plugin_info)) - plugins = [r for r in results if r is not None] - plugins.extend(_starlark_virtual_plugins()) - - return jsonify({'status': 'success', 'data': {'plugins': plugins}}) - except Exception as e: - logger.error('Error in get_installed_plugins', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -def _installed_plugin_ids(): - """Best-effort list of installed plugin IDs for the web process. - - Health/metrics state is written by the separate display service to the - shared on-disk cache, so the tracker's in-memory set is empty here. We - enumerate the installed plugins and read each one's persisted summary by ID - instead of relying on the tracker's in-memory `get_all_*` view. - """ - pm = api_v3.plugin_manager - manifests = getattr(pm, 'plugin_manifests', None) - if not manifests: - # Only pay for a discovery scan when we haven't discovered anything yet; - # subsequent polls reuse the already-populated manifest map. - try: - pm.discover_plugins() - except Exception: - logger.debug('discover_plugins failed while listing plugin ids', exc_info=True) - manifests = getattr(pm, 'plugin_manifests', None) - try: - return list(manifests.keys()) if manifests else [] - except Exception: - logger.debug('listing plugin_manifests failed while building plugin ids', exc_info=True) - return [] - - -@api_v3.route('/plugins/health', methods=['GET']) -def get_plugin_health(): - """Get health metrics for all plugins""" - try: - if not api_v3.plugin_manager: - return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - - # Check if health tracker is available - if not hasattr(api_v3.plugin_manager, 'health_tracker') or not api_v3.plugin_manager.health_tracker: - return jsonify({ - 'status': 'success', - 'data': {}, - 'message': 'Health tracking not available' - }) - - tracker = api_v3.plugin_manager.health_tracker - # Build per-plugin summaries by ID so persisted (cross-process) health - # is included, then fold in any in-memory-only entries. - health_summaries = {} - for pid in _installed_plugin_ids(): - try: - # force_reload: this process only reads; bypass the in-memory - # snapshot so each poll reflects the display service's latest - # persisted state. - health_summaries[pid] = tracker.get_health_summary(pid, force_reload=True) - except Exception: - logger.debug('Could not read health summary for %s', pid, exc_info=True) - try: - for pid, summary in tracker.get_all_health_summaries().items(): - health_summaries.setdefault(pid, summary) - except Exception: - logger.debug('get_all_health_summaries failed', exc_info=True) - - return jsonify({ - 'status': 'success', - 'data': health_summaries - }) - except Exception as e: - logger.error('Error in get_plugin_health', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/health/', methods=['GET']) -def get_plugin_health_single(plugin_id): - """Get health metrics for a specific plugin""" - try: - if not api_v3.plugin_manager: - return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - - # Check if health tracker is available - if not hasattr(api_v3.plugin_manager, 'health_tracker') or not api_v3.plugin_manager.health_tracker: - return jsonify({ - 'status': 'error', - 'message': 'Health tracking not available' - }), 503 - - # Get health summary for specific plugin - health_summary = api_v3.plugin_manager.health_tracker.get_health_summary(plugin_id) - - return jsonify({ - 'status': 'success', - 'data': health_summary - }) - except Exception as e: - logger.error('Error in get_plugin_health_single', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/health//reset', methods=['POST']) -def reset_plugin_health(plugin_id): - """Reset health state for a plugin (manual recovery)""" - try: - if not api_v3.plugin_manager: - return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - - # Check if health tracker is available - if not hasattr(api_v3.plugin_manager, 'health_tracker') or not api_v3.plugin_manager.health_tracker: - return jsonify({ - 'status': 'error', - 'message': 'Health tracking not available' - }), 503 - - # Reset health state - api_v3.plugin_manager.health_tracker.reset_health(plugin_id) - - return jsonify({ - 'status': 'success', - 'message': f'Health state reset for plugin {plugin_id}' - }) - except Exception as e: - logger.error('Error in reset_plugin_health', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/metrics', methods=['GET']) -def get_plugin_metrics(): - """Get resource metrics for all plugins""" - try: - if not api_v3.plugin_manager: - return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - - # Check if resource monitor is available - if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: - return jsonify({ - 'status': 'success', - 'data': {}, - 'message': 'Resource monitoring not available' - }) - - monitor = api_v3.plugin_manager.resource_monitor - # Build per-plugin summaries by ID so persisted (cross-process) metrics - # are included, then fold in any in-memory-only entries. - metrics_summaries = {} - for pid in _installed_plugin_ids(): - try: - # force_reload: read-only path — bypass the in-memory snapshot so - # each poll reflects the display service's latest persisted metrics. - metrics_summaries[pid] = monitor.get_metrics_summary(pid, force_reload=True) - except Exception: - logger.debug('Could not read metrics summary for %s', pid, exc_info=True) - try: - for pid, summary in monitor.get_all_metrics_summaries().items(): - metrics_summaries.setdefault(pid, summary) - except Exception: - logger.debug('get_all_metrics_summaries failed', exc_info=True) - - return jsonify({ - 'status': 'success', - 'data': metrics_summaries - }) - except Exception as e: - logger.error('Error in get_plugin_metrics', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/metrics/', methods=['GET']) -def get_plugin_metrics_single(plugin_id): - """Get resource metrics for a specific plugin""" - try: - if not api_v3.plugin_manager: - return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - - # Check if resource monitor is available - if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: - return jsonify({ - 'status': 'error', - 'message': 'Resource monitoring not available' - }), 503 - - # Get metrics summary for specific plugin - metrics_summary = api_v3.plugin_manager.resource_monitor.get_metrics_summary(plugin_id) - - return jsonify({ - 'status': 'success', - 'data': metrics_summary - }) - except Exception as e: - logger.error('Error in get_plugin_metrics_single', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/metrics//reset', methods=['POST']) -def reset_plugin_metrics(plugin_id): - """Reset metrics for a plugin""" - try: - if not api_v3.plugin_manager: - return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - - # Check if resource monitor is available - if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: - return jsonify({ - 'status': 'error', - 'message': 'Resource monitoring not available' - }), 503 - - # Reset metrics - api_v3.plugin_manager.resource_monitor.reset_metrics(plugin_id) - - return jsonify({ - 'status': 'success', - 'message': f'Metrics reset for plugin {plugin_id}' - }) - except Exception as e: - logger.error('Error in reset_plugin_metrics', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/limits/', methods=['GET', 'POST']) -def manage_plugin_limits(plugin_id): - """Get or set resource limits for a plugin""" - try: - if not api_v3.plugin_manager: - return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 - - # Check if resource monitor is available - if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: - return jsonify({ - 'status': 'error', - 'message': 'Resource monitoring not available' - }), 503 - - if request.method == 'GET': - # Get limits - limits = api_v3.plugin_manager.resource_monitor.get_limits(plugin_id) - if limits: - return jsonify({ - 'status': 'success', - 'data': { - 'max_memory_mb': limits.max_memory_mb, - 'max_cpu_percent': limits.max_cpu_percent, - 'max_execution_time': limits.max_execution_time, - 'warning_threshold': limits.warning_threshold - } - }) - else: - return jsonify({ - 'status': 'success', - 'data': None, - 'message': 'No limits configured for this plugin' - }) - else: - # POST - Set limits - data = request.get_json(silent=True) or {} - from src.plugin_system.resource_monitor import ResourceLimits - - limits = ResourceLimits( - max_memory_mb=data.get('max_memory_mb'), - max_cpu_percent=data.get('max_cpu_percent'), - max_execution_time=data.get('max_execution_time'), - warning_threshold=data.get('warning_threshold', 0.8) - ) - - api_v3.plugin_manager.resource_monitor.set_limits(plugin_id, limits) - - return jsonify({ - 'status': 'success', - 'message': f'Resource limits updated for plugin {plugin_id}' - }) - except Exception as e: - logger.error('Error in manage_plugin_limits', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/toggle', methods=['POST']) -def toggle_plugin(): - """Toggle plugin enabled/disabled""" - try: - if not api_v3.plugin_manager or not api_v3.config_manager: - return jsonify({'status': 'error', 'message': 'Plugin or config manager not initialized'}), 500 - - # Support both JSON and form data (for HTMX submissions) - content_type = request.content_type or '' - - if 'application/json' in content_type: - data = request.get_json(silent=True) - if not data or 'plugin_id' not in data or 'enabled' not in data: - return jsonify({'status': 'error', 'message': 'plugin_id and enabled required'}), 400 - plugin_id = data['plugin_id'] - enabled = data['enabled'] - else: - # Form data or query string (HTMX submission) - plugin_id = request.args.get('plugin_id') or request.form.get('plugin_id') - if not plugin_id: - return jsonify({'status': 'error', 'message': 'plugin_id required'}), 400 - - # For checkbox toggle, if form was submitted, the checkbox was checked (enabled) - # If using HTMX with hx-trigger="change", we need to check if checkbox is checked - # The checkbox value or 'enabled' form field indicates the state - enabled_str = request.form.get('enabled', request.args.get('enabled', '')) - - # Handle various truthy/falsy values - if enabled_str.lower() in ('true', '1', 'on', 'yes'): - enabled = True - elif enabled_str.lower() in ('false', '0', 'off', 'no', ''): - # Empty string means checkbox was unchecked (toggle off) - enabled = False - else: - # Default: toggle based on current state - config = api_v3.config_manager.load_config() - current_enabled = config.get(plugin_id, {}).get('enabled', False) - enabled = not current_enabled - - # A Starlark app is not a plugin in plugin_manager's sense -- it is an - # entry in starlark-apps' own manifest -- so its enable/disable is - # handled here rather than falling through to the check below, which - # would answer "Plugin not found". - if plugin_id.startswith('starlark:'): - return _toggle_starlark_app(plugin_id[len('starlark:'):], enabled) - - # Check if plugin exists in manifests (discovered but may not be loaded) - if plugin_id not in api_v3.plugin_manager.plugin_manifests: - return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - - # Update config (this is what the display controller reads) - config = api_v3.config_manager.load_config() - if plugin_id not in config: - config[plugin_id] = {} - config[plugin_id]['enabled'] = enabled - - # Use atomic save if available - if hasattr(api_v3.config_manager, 'save_config_atomic'): - result = api_v3.config_manager.save_config_atomic(config, create_backup=True) - if result.status.value != 'success': - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - f"Failed to save configuration: {result.message}", - status_code=500 - ) - else: - api_v3.config_manager.save_config(config) - - # Update state manager if available - if api_v3.plugin_state_manager: - api_v3.plugin_state_manager.set_plugin_enabled(plugin_id, enabled) - - # Log operation - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "enable" if enabled else "disable", - plugin_id=plugin_id, - status="success" - ) - - # If plugin is loaded, also call its lifecycle methods - # Wrap in try/except to prevent lifecycle errors from failing the toggle - plugin = api_v3.plugin_manager.get_plugin(plugin_id) - if plugin: - try: - if enabled: - if hasattr(plugin, 'on_enable'): - plugin.on_enable() - else: - if hasattr(plugin, 'on_disable'): - plugin.on_disable() - except Exception as lifecycle_error: - # Log the error but don't fail the toggle - config is already saved - import logging - logging.warning(f"Lifecycle method error for {plugin_id}: {lifecycle_error}", exc_info=True) - - return success_response( - message=f"Plugin {plugin_id} {'enabled' if enabled else 'disabled'} successfully" - ) - except Exception as e: - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.PLUGIN_OPERATION_CONFLICT) - if api_v3.operation_history: - toggle_type = "enable" if ('data' in locals() and data.get('enabled')) else "disable" - api_v3.operation_history.record_operation( - toggle_type, - plugin_id=data.get('plugin_id') if 'data' in locals() else None, - status="failed", - error=str(e) - ) - return error_response( - error.error_code, - error.message, - details=error.details, - context=error.context, - status_code=500 - ) - -@api_v3.route('/plugins/operation/', methods=['GET']) -def get_operation_status(operation_id): - """Get status of a plugin operation""" - try: - if not api_v3.operation_queue: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'Operation queue not initialized', - status_code=500 - ) - - operation = api_v3.operation_queue.get_operation_status(operation_id) - if not operation: - return error_response( - ErrorCode.PLUGIN_NOT_FOUND, - f'Operation {operation_id} not found', - status_code=404 - ) - - return success_response(data=operation.to_dict()) - except Exception as e: - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) - return error_response( - error.error_code, - error.message, - details=error.details, - status_code=500 - ) - -@api_v3.route('/plugins/operation/history', methods=['GET']) -def get_operation_history() -> Response: - """Get operation history from the audit log.""" - if not api_v3.operation_history: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'Operation history not initialized', - status_code=500 - ) - - try: - limit = request.args.get('limit', 50, type=int) - plugin_id = request.args.get('plugin_id') - operation_type = request.args.get('operation_type') - except (ValueError, TypeError) as e: - return error_response(ErrorCode.INVALID_INPUT, f'Invalid query parameter: {e}', status_code=400) - - try: - history = api_v3.operation_history.get_history( - limit=limit, - plugin_id=plugin_id, - operation_type=operation_type - ) - except (AttributeError, RuntimeError) as e: - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) - return error_response(error.error_code, error.message, details=error.details, status_code=500) - - return success_response(data=[record.to_dict() for record in history]) - -@api_v3.route('/plugins/operation/history', methods=['DELETE']) -def clear_operation_history() -> Response: - """Clear operation history.""" - if not api_v3.operation_history: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'Operation history not initialized', - status_code=500 - ) - - try: - api_v3.operation_history.clear_history() - except (OSError, RuntimeError) as e: - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) - return error_response(error.error_code, error.message, details=error.details, status_code=500) - - return success_response(message='Operation history cleared') - -@api_v3.route('/plugins/state', methods=['GET']) -def get_plugin_state(): - """Get plugin state from state manager""" - try: - if not api_v3.plugin_state_manager: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'State manager not initialized', - status_code=500 - ) - - plugin_id = request.args.get('plugin_id') - - if plugin_id: - # Get state for specific plugin - state = api_v3.plugin_state_manager.get_plugin_state(plugin_id) - if not state: - return error_response( - ErrorCode.PLUGIN_NOT_FOUND, - f'Plugin {plugin_id} not found in state manager', - context={'plugin_id': plugin_id}, - status_code=404 - ) - return success_response(data=state.to_dict()) - else: - # Get all plugin states - all_states = api_v3.plugin_state_manager.get_all_states() - return success_response(data={ - plugin_id: state.to_dict() - for plugin_id, state in all_states.items() - }) - except Exception as e: - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) - return error_response( - error.error_code, - error.message, - details=error.details, - context=error.context, - status_code=500 - ) - -@api_v3.route('/plugins/state/reconcile', methods=['POST']) -def reconcile_plugin_state(): - """Reconcile plugin state across all sources""" - try: - if not api_v3.plugin_state_manager or not api_v3.plugin_manager: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'State manager or plugin manager not initialized', - status_code=500 - ) - - from src.plugin_system.state_reconciliation import StateReconciliation - - # Parse optional `force` flag from request body, guarding against - # non-dict bodies (bare string, array, null) that would raise AttributeError. - payload = request.get_json(silent=True) - if not isinstance(payload, dict): - payload = {} - force = _coerce_to_bool(payload.get('force', False)) - - reconciler = StateReconciliation( - state_manager=api_v3.plugin_state_manager, - config_manager=api_v3.config_manager, - plugin_manager=api_v3.plugin_manager, - plugins_dir=Path(api_v3.plugin_manager.plugins_dir) - ) - - result = reconciler.reconcile_state(force=force) - - return success_response( - data={ - 'inconsistencies_found': len(result.inconsistencies_found), - 'inconsistencies_fixed': len(result.inconsistencies_fixed), - 'inconsistencies_manual': len(result.inconsistencies_manual), - 'inconsistencies': [ - { - 'plugin_id': inc.plugin_id, - 'type': inc.inconsistency_type.value, - 'description': inc.description, - 'fix_action': inc.fix_action.value - } - for inc in result.inconsistencies_found - ], - 'fixed': [ - { - 'plugin_id': inc.plugin_id, - 'type': inc.inconsistency_type.value, - 'description': inc.description - } - for inc in result.inconsistencies_fixed - ], - 'manual_fix_required': [ - { - 'plugin_id': inc.plugin_id, - 'type': inc.inconsistency_type.value, - 'description': inc.description - } - for inc in result.inconsistencies_manual - ] - }, - message=result.message - ) - except Exception as e: - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) - return error_response( - error.error_code, - error.message, - details=error.details, - context=error.context, - status_code=500 - ) - -def _drop_stale_reconciliation_findings(unresolved): - """Re-check a stored reconciliation verdict against current state. - - The verdict is a snapshot written once per run, and a run that could not - apply a fix also refuses to retry -- so a resolved condition was reported - indefinitely. Best-effort: any failure here returns the list untouched, - because showing a stale warning beats failing the endpoint. - """ - try: - from src.plugin_system.state_reconciliation import still_unresolved - - config_keys = set(api_v3.config_manager.load_config() or {}) - installed = set() - plugins_dir = getattr(api_v3.plugin_manager, 'plugins_dir', None) - if plugins_dir: - for entry in Path(plugins_dir).iterdir(): - if entry.is_dir() and (entry / 'manifest.json').exists(): - installed.add(entry.name) - return still_unresolved(unresolved, config_keys, installed) - except Exception: - logger.debug("[Reconciliation] Could not re-check stored findings", exc_info=True) - return unresolved - - -@api_v3.route('/plugins/reconciliation-status', methods=['GET']) -def get_reconciliation_status(): - """Return the result of the last startup reconciliation from /tmp status file.""" - _recon_path = os.path.join(tempfile.gettempdir(), "ledmatrix_reconciliation.json") - try: - st = os.lstat(_recon_path) - except FileNotFoundError: - return jsonify({'status': 'success', 'data': {'done': False, 'unresolved': []}}) - if stat.S_ISLNK(st.st_mode) or not stat.S_ISREG(st.st_mode): - logger.warning("[Reconciliation] Status file is not a regular file: %s", _recon_path) - return jsonify({'status': 'success', 'data': {'done': False, 'unresolved': []}}) - try: - with open(_recon_path) as _f: - data = json.load(_f) - if data.get('unresolved'): - data['unresolved'] = _drop_stale_reconciliation_findings(data['unresolved']) - return jsonify({'status': 'success', 'data': data}) - except json.JSONDecodeError: - logger.exception("[Reconciliation] Failed to parse status file: %s", _recon_path) - return jsonify({'status': 'success', 'data': {'done': False, 'unresolved': []}}) - except PermissionError: - logger.exception("[Reconciliation] Permission denied reading status file: %s", _recon_path) - return jsonify({'status': 'success', 'data': {'done': False, 'unresolved': []}}) - -@api_v3.route('/plugins/config', methods=['GET']) -def get_plugin_config(): - """Get plugin configuration""" - try: - if not api_v3.config_manager: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'Config manager not initialized', - status_code=500 - ) - - plugin_id = request.args.get('plugin_id') - if not plugin_id: - return error_response( - ErrorCode.INVALID_INPUT, - 'plugin_id required', - context={'missing_params': ['plugin_id']}, - status_code=400 - ) - - # Get plugin configuration from config manager - main_config = api_v3.config_manager.load_config() - plugin_config = main_config.get(plugin_id, {}) - - # Merge with defaults from schema so form shows default values for missing fields - schema_mgr = api_v3.schema_manager - if schema_mgr: - try: - defaults = schema_mgr.generate_default_config(plugin_id, use_cache=True) - plugin_config = schema_mgr.merge_with_defaults(plugin_config, defaults) - except Exception as e: - # Log but don't fail - defaults merge is best effort - import logging - logging.warning(f"Could not merge defaults for {plugin_id}: {e}") - - # Special handling for of-the-day plugin: populate uploaded_files and categories from disk - if plugin_id == 'of-the-day' or plugin_id == 'ledmatrix-of-the-day': - # Get plugin directory - plugin_id in manifest is 'of-the-day', but directory is 'ledmatrix-of-the-day' - plugin_dir_name = 'ledmatrix-of-the-day' - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_dir_name) - # If not found, try with the plugin_id - if not plugin_dir or not Path(plugin_dir).exists(): - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_dir_name - if not plugin_dir.exists(): - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if plugin_dir and Path(plugin_dir).exists(): - data_dir = Path(plugin_dir) / 'of_the_day' - if data_dir.exists(): - # Scan for JSON files - uploaded_files = [] - categories_from_files = {} - - for json_file in data_dir.glob('*.json'): - try: - # Get file stats - stat = json_file.stat() - - # Read JSON to count entries - with open(json_file, 'r', encoding='utf-8') as f: - json_data = json.load(f) - entry_count = len(json_data) if isinstance(json_data, dict) else 0 - - # Extract category name from filename - category_name = json_file.stem - filename = json_file.name - - # Create file entry - file_entry = { - 'id': category_name, - 'category_name': category_name, - 'filename': filename, - 'original_filename': filename, - 'path': f'of_the_day/{filename}', - 'size': stat.st_size, - 'uploaded_at': datetime.fromtimestamp(stat.st_mtime).isoformat() + 'Z', - 'entry_count': entry_count - } - uploaded_files.append(file_entry) - - # Create/update category entry if not in config - if category_name not in plugin_config.get('categories', {}): - display_name = category_name.replace('_', ' ').title() - categories_from_files[category_name] = { - 'enabled': False, # Default to disabled, user can enable - 'data_file': f'of_the_day/{filename}', - 'display_name': display_name - } - else: - # Update with file info if needed - categories_from_files[category_name] = plugin_config['categories'][category_name] - # Ensure data_file is correct - categories_from_files[category_name]['data_file'] = f'of_the_day/{filename}' - - except Exception as e: - logger.debug("Could not read json file: %s", e) - continue - - # Update plugin_config with scanned files - if uploaded_files: - plugin_config['uploaded_files'] = uploaded_files - - # Merge categories from files with existing config - # Start with existing categories (preserve user settings like enabled/disabled) - existing_categories = plugin_config.get('categories', {}).copy() - - # Update existing categories with file info, add new ones from files - for cat_name, cat_data in categories_from_files.items(): - if cat_name in existing_categories: - # Preserve existing enabled state and display_name, but update data_file path - existing_categories[cat_name]['data_file'] = cat_data['data_file'] - if 'display_name' not in existing_categories[cat_name] or not existing_categories[cat_name]['display_name']: - existing_categories[cat_name]['display_name'] = cat_data['display_name'] - else: - # Add new category from file (default to disabled) - existing_categories[cat_name] = cat_data - - if existing_categories: - plugin_config['categories'] = existing_categories - - # Update category_order to include all categories - category_order = plugin_config.get('category_order', []).copy() - all_category_names = set(existing_categories.keys()) - for cat_name in all_category_names: - if cat_name not in category_order: - category_order.append(cat_name) - if category_order: - plugin_config['category_order'] = category_order - - # If no config exists, return defaults - if not plugin_config: - plugin_config = { - 'enabled': True, - 'display_duration': 30 - } - - return success_response(data=plugin_config) - except Exception as e: - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.CONFIG_LOAD_FAILED) - return error_response( - error.error_code, - error.message, - details=error.details, - context=error.context, - status_code=500 - ) - -@api_v3.route('/plugins/update', methods=['POST']) -def update_plugin(): - """Update plugin""" - try: - # Support both JSON and form data - content_type = request.content_type or '' - - if 'application/json' in content_type: - # JSON request - data, error = validate_request_json(['plugin_id']) - if error: - logger.debug("[UPDATE] JSON validation failed. Content-Type: %s", content_type) - return error - else: - # Form data or query string - plugin_id = request.args.get('plugin_id') or request.form.get('plugin_id') - if not plugin_id: - logger.debug("[UPDATE] Missing plugin_id. Content-Type: %s", content_type) - return error_response( - ErrorCode.INVALID_INPUT, - 'plugin_id required', - status_code=400 - ) - data = {'plugin_id': plugin_id} - - if not api_v3.plugin_store_manager: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'Plugin store manager not initialized', - status_code=500 - ) - - plugin_id = data['plugin_id'] - - # Always do direct updates (they're fast git pull operations) - # Operation queue is reserved for longer operations like install/uninstall - plugin_dir = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id - manifest_path = plugin_dir / "manifest.json" - - current_last_updated = None - current_commit = None - current_branch = None - - if manifest_path.exists(): - try: - import json - with open(manifest_path, 'r', encoding='utf-8') as f: - manifest = json.load(f) - current_last_updated = manifest.get('last_updated') - if manifest.get('local_only'): - logger.debug("Skipping update for local-only plugin: %s", plugin_id) - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "update", - plugin_id=plugin_id, - status="skipped", - details={"reason": "local_only"} - ) - return success_response(message=f'Plugin {plugin_id} is managed locally and does not receive registry updates') - except Exception as e: - logger.debug("Could not read local manifest for plugin: %s", e) - - if api_v3.plugin_store_manager: - git_info_before = api_v3.plugin_store_manager._get_local_git_info(plugin_dir) - if git_info_before: - current_commit = git_info_before.get('sha') - current_branch = git_info_before.get('branch') - - # Check if plugin is a git repo first (for better error messages) - plugin_path_dir = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id - is_git_repo = False - if plugin_path_dir.exists(): - git_info = api_v3.plugin_store_manager._get_local_git_info(plugin_path_dir) - is_git_repo = git_info is not None - if is_git_repo: - logger.debug("Plugin is a git repository, will update via git pull") - - remote_info = api_v3.plugin_store_manager.get_plugin_info(plugin_id, fetch_latest_from_github=True) - remote_commit = remote_info.get('last_commit_sha') if remote_info else None - remote_branch = remote_info.get('branch') if remote_info else None - - # Update the plugin - success = api_v3.plugin_store_manager.update_plugin(plugin_id) - - if success: - updated_last_updated = current_last_updated - try: - if manifest_path.exists(): - import json - with open(manifest_path, 'r', encoding='utf-8') as f: - manifest = json.load(f) - updated_last_updated = manifest.get('last_updated', current_last_updated) - except Exception as e: - logger.debug("Could not read updated manifest after update: %s", e) - - updated_commit = None - updated_branch = remote_branch or current_branch - if api_v3.plugin_store_manager: - git_info_after = api_v3.plugin_store_manager._get_local_git_info(plugin_dir) - if git_info_after: - updated_commit = git_info_after.get('sha') - updated_branch = git_info_after.get('branch') or updated_branch - - message = f'Plugin {plugin_id} updated successfully' - if current_commit and updated_commit and current_commit == updated_commit: - message = f'Plugin {plugin_id} already up to date (commit {updated_commit[:7]})' - elif updated_commit: - message = f'Plugin {plugin_id} updated to commit {updated_commit[:7]}' - if updated_branch: - message += f' on branch {updated_branch}' - elif updated_last_updated and updated_last_updated != current_last_updated: - message = f'Plugin {plugin_id} refreshed (Last Updated {updated_last_updated})' - - remote_commit_short = remote_commit[:7] if remote_commit else None - if remote_commit_short and updated_commit and remote_commit_short != updated_commit[:7]: - message += f' (remote latest {remote_commit_short})' - - # Invalidate schema cache - if api_v3.schema_manager: - api_v3.schema_manager.invalidate_cache(plugin_id) - - # Rediscover plugins - if api_v3.plugin_manager: - api_v3.plugin_manager.discover_plugins() - if plugin_id in api_v3.plugin_manager.plugins: - api_v3.plugin_manager.reload_plugin(plugin_id) - - # Update state and history - if api_v3.plugin_state_manager: - api_v3.plugin_state_manager.update_plugin_state( - plugin_id, - {'last_updated': datetime.now()} - ) - if api_v3.operation_history: - version = _get_plugin_version(plugin_id) - api_v3.operation_history.record_operation( - "update", - plugin_id=plugin_id, - status="success", - details={ - "version": version, - "previous_commit": current_commit[:7] if current_commit else None, - "commit": updated_commit[:7] if updated_commit else None, - "branch": updated_branch - } - ) - - return success_response( - data={ - 'last_updated': updated_last_updated, - 'commit': updated_commit - }, - message=message - ) - else: - plugin_path_dir = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id - if not plugin_path_dir.exists(): - client_msg = 'Plugin update failed: plugin not found' - else: - git_info = api_v3.plugin_store_manager._get_local_git_info(plugin_path_dir) - if not git_info: - plugin_info = api_v3.plugin_store_manager.get_plugin_info(plugin_id) - if not plugin_info: - client_msg = 'Plugin update failed: not found in registry' - else: - client_msg = 'Plugin update failed; check logs for details' - else: - client_msg = 'Plugin update failed; check logs for details' - logger.error("update_plugin failed for plugin_id=%s: %s", plugin_id, client_msg) - - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "update", - plugin_id=plugin_id, - status="failed", - error=client_msg, - details={ - "previous_commit": current_commit[:7] if current_commit else None, - "branch": current_branch - } - ) - - return error_response( - ErrorCode.PLUGIN_UPDATE_FAILED, - client_msg, - status_code=500 - ) - - except Exception as e: - logger.error("Unhandled exception in update endpoint", exc_info=True) - - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.PLUGIN_UPDATE_FAILED) - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "update", - plugin_id=data.get('plugin_id') if 'data' in locals() else None, - status="failed", - error=str(e) - ) - return error_response( - error.error_code, - error.message, - details=error.details, - context=error.context, - status_code=500 - ) - -def _do_transactional_uninstall(plugin_id, preserve_config): - """Execute an uninstall with snapshot-based rollback. - - Order of operations: - 1. Snapshot main config + secrets (abort on unexpected errors, proceed on expected I/O errors). - 2. Clean up plugin config (abort with 500 if this raises — avoids orphaned files). - 3. Unload plugin from runtime if loaded (rollback + 500 if this raises). - 4. Remove plugin files (rollback + 500 if this returns False or raises). - 5. Finish (remove state, invalidate caches). - - Rollback restores the config snapshot and, if the plugin had been - loaded before unload, calls load_plugin to restore runtime state. - - Returns (True, None) on success or (False, error_message) on failure. - """ - from src.exceptions import ConfigError - - # --- Step 1: snapshot main + secrets --- - main_snapshot = None - secrets_snapshot = None - try: - main_snapshot = api_v3.config_manager.get_raw_file_content('main') - except (OSError, ConfigError): - pass # Proceed without snapshot; narrow catch preserves TypeError/AttributeError - try: - secrets_snapshot = api_v3.config_manager.get_raw_file_content('secrets') - except (OSError, ConfigError): - pass - - # --- Step 2: cleanup config first (abort before touching filesystem) --- - if not preserve_config: - api_v3.config_manager.cleanup_plugin_config(plugin_id, remove_secrets=True) - - # Record whether the plugin was running before we touch anything. - was_loaded = ( - api_v3.plugin_manager is not None - and plugin_id in api_v3.plugin_manager.plugins - ) - - def _rollback(reload_plugin): - if main_snapshot is not None: - try: - api_v3.config_manager.save_raw_file_content('main', main_snapshot) - except Exception as restore_err: - logger.error("Failed to restore main config snapshot for %s: %s", plugin_id, restore_err) - if secrets_snapshot is not None: - try: - api_v3.config_manager.save_raw_file_content('secrets', secrets_snapshot) - except Exception as restore_err: - logger.error("Failed to restore secrets snapshot for %s: %s", plugin_id, restore_err) - if reload_plugin and api_v3.plugin_manager is not None: - try: - api_v3.plugin_manager.load_plugin(plugin_id) - except Exception as reload_err: - logger.error("Failed to reload plugin %s during rollback: %s", plugin_id, reload_err) - - # --- Step 3: unload --- - if was_loaded: - try: - api_v3.plugin_manager.unload_plugin(plugin_id) - except Exception as unload_err: - _rollback(reload_plugin=False) # unload failed — runtime state unchanged - return False, f"Failed to unload plugin {plugin_id}: {unload_err}" - - # --- Step 4: remove files --- - try: - success = api_v3.plugin_store_manager.uninstall_plugin(plugin_id) - except Exception as remove_err: - _rollback(reload_plugin=was_loaded) - return False, f"Failed to remove plugin {plugin_id}: {remove_err}" - - if not success: - _rollback(reload_plugin=was_loaded) - return False, f"Failed to uninstall plugin {plugin_id}" - - # --- Step 5: finish --- - if api_v3.schema_manager: - api_v3.schema_manager.invalidate_cache(plugin_id) - if api_v3.plugin_state_manager: - api_v3.plugin_state_manager.remove_plugin_state(plugin_id) - # Persistently record the uninstall so a later core `git pull` update - # cannot resurrect a built-in plugin (committed under plugin-repos/) that - # the user removed. Best-effort: never fail the uninstall over this. - try: - api_v3.plugin_store_manager.record_uninstalled_plugin(plugin_id) - except Exception as record_err: - logger.warning("Could not record uninstall for %s: %s", plugin_id, record_err) - return True, None - - -@api_v3.route('/plugins/uninstall', methods=['POST']) -def uninstall_plugin(): - """Uninstall plugin""" - try: - # Validate request - data, error = validate_request_json(['plugin_id']) - if error: - return error - - if not api_v3.plugin_store_manager: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'Plugin store manager not initialized', - status_code=500 - ) - - plugin_id = data['plugin_id'] - preserve_config = data.get('preserve_config', False) - - # Both queued and direct paths use the same transactional helper so - # snapshot/rollback behaviour is consistent regardless of deployment. - if api_v3.operation_queue: - def uninstall_callback(operation): - """Callback to execute plugin uninstallation via transactional helper.""" - success, error_msg = _do_transactional_uninstall(plugin_id, preserve_config) - if not success: - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "uninstall", - plugin_id=plugin_id, - status="failed", - error=error_msg - ) - raise Exception(error_msg or f'Failed to uninstall plugin {plugin_id}') - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "uninstall", - plugin_id=plugin_id, - status="success", - details={"preserve_config": preserve_config} - ) - return {'success': True, 'message': 'Plugin uninstalled successfully'} - - # Enqueue operation - operation_id = api_v3.operation_queue.enqueue_operation( - OperationType.UNINSTALL, - plugin_id, - operation_callback=uninstall_callback - ) - - return success_response( - data={'operation_id': operation_id}, - message='Plugin uninstallation queued' - ) - else: - # Direct (non-queued) transactional uninstall - success, error_msg = _do_transactional_uninstall(plugin_id, preserve_config) - - if success: - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "uninstall", - plugin_id=plugin_id, - status="success", - details={"preserve_config": preserve_config} - ) - return success_response(message='Plugin uninstalled successfully') - else: - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "uninstall", - plugin_id=plugin_id, - status="failed", - error=error_msg - ) - return error_response( - ErrorCode.PLUGIN_UNINSTALL_FAILED, - error_msg or 'Plugin uninstall failed', - status_code=500 - ) - - except Exception as e: - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.PLUGIN_UNINSTALL_FAILED) - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "uninstall", - plugin_id=data.get('plugin_id') if 'data' in locals() else None, - status="failed", - error=str(e) - ) - return error_response( - error.error_code, - error.message, - details=error.details, - context=error.context, - status_code=500 - ) - -@api_v3.route('/plugins/install', methods=['POST']) -def install_plugin(): - """Install plugin from store""" - try: - if not api_v3.plugin_store_manager: - return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - - data = request.get_json(silent=True) - if not data or 'plugin_id' not in data: - return jsonify({'status': 'error', 'message': 'plugin_id required'}), 400 - - plugin_id = data['plugin_id'] - branch = data.get('branch') # Optional branch parameter - - # Install the plugin - # Log the plugins directory being used for debugging - plugins_dir = api_v3.plugin_store_manager.plugins_dir - branch_info = f" (branch: {branch})" if branch else "" - logger.info("Installing plugin to directory: %s", plugins_dir) - - # Use operation queue if available - if api_v3.operation_queue: - def install_callback(operation): - """Callback to execute plugin installation.""" - success = api_v3.plugin_store_manager.install_plugin(plugin_id, branch=branch) - - if success: - # Invalidate schema cache - if api_v3.schema_manager: - api_v3.schema_manager.invalidate_cache(plugin_id) - - # Discover and load the new plugin - if api_v3.plugin_manager: - api_v3.plugin_manager.discover_plugins() - api_v3.plugin_manager.load_plugin(plugin_id) - - # Update state manager - if api_v3.plugin_state_manager: - api_v3.plugin_state_manager.set_plugin_installed(plugin_id) - - # Record in history - if api_v3.operation_history: - version = _get_plugin_version(plugin_id) - api_v3.operation_history.record_operation( - "install", - plugin_id=plugin_id, - status="success", - details={"version": version, "branch": branch} - ) - - branch_msg = f" (branch: {branch})" if branch else "" - return {'success': True, 'message': f'Plugin {plugin_id} installed successfully{branch_msg}'} - else: - error_msg = f'Failed to install plugin {plugin_id}' - if branch: - error_msg += f' (branch: {branch})' - plugin_info = api_v3.plugin_store_manager.get_plugin_info(plugin_id) - if not plugin_info: - error_msg += ' (plugin not found in registry)' - - # Record failure in history - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "install", - plugin_id=plugin_id, - status="failed", - error=error_msg, - details={"branch": branch} - ) - - raise Exception(error_msg) - - # Enqueue operation - operation_id = api_v3.operation_queue.enqueue_operation( - OperationType.INSTALL, - plugin_id, - operation_callback=install_callback - ) - - branch_msg = f" (branch: {branch})" if branch else "" - return success_response( - data={'operation_id': operation_id}, - message=f'Plugin {plugin_id} installation queued{branch_msg}' - ) - else: - # Fallback to direct installation - success = api_v3.plugin_store_manager.install_plugin(plugin_id, branch=branch) - - if success: - if api_v3.schema_manager: - api_v3.schema_manager.invalidate_cache(plugin_id) - if api_v3.plugin_manager: - api_v3.plugin_manager.discover_plugins() - api_v3.plugin_manager.load_plugin(plugin_id) - if api_v3.plugin_state_manager: - api_v3.plugin_state_manager.set_plugin_installed(plugin_id) - if api_v3.operation_history: - version = _get_plugin_version(plugin_id) - api_v3.operation_history.record_operation( - "install", - plugin_id=plugin_id, - status="success", - details={"version": version, "branch": branch} - ) - - branch_msg = f" (branch: {branch})" if branch else "" - return success_response(message=f'Plugin installed successfully{branch_msg}') - else: - error_msg = f'Failed to install plugin {plugin_id}' - if branch: - error_msg += f' (branch: {branch})' - plugin_info = api_v3.plugin_store_manager.get_plugin_info(plugin_id) - if not plugin_info: - error_msg += ' (plugin not found in registry)' - - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "install", - plugin_id=plugin_id, - status="failed", - error=error_msg, - details={"branch": branch} - ) - - return error_response( - ErrorCode.PLUGIN_INSTALL_FAILED, - error_msg, - status_code=500 - ) - - except Exception as e: - logger.error('Error in install_plugin', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/install-from-url', methods=['POST']) -def install_plugin_from_url(): - """Install plugin from custom GitHub URL""" - try: - if not api_v3.plugin_store_manager: - return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - - data = request.get_json(silent=True) - if not data or 'repo_url' not in data: - return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 - - # A non-string repo_url is a client mistake, not a server fault: - # .strip() would raise and the catch-all would report it as a 500. - if not isinstance(data['repo_url'], str) or not data['repo_url'].strip(): - return jsonify({'status': 'error', 'message': 'repo_url must be a non-empty string'}), 400 - - repo_url = data['repo_url'].strip() - plugin_id = data.get('plugin_id') # Optional, for monorepo installations - plugin_path = data.get('plugin_path') # Optional, for monorepo subdirectory - branch = data.get('branch') # Optional branch parameter - - # Install the plugin - result = api_v3.plugin_store_manager.install_from_url( - repo_url=repo_url, - plugin_id=plugin_id, - plugin_path=plugin_path, - branch=branch - ) - - if result.get('success'): - # Invalidate schema cache for the installed plugin - installed_plugin_id = result.get('plugin_id') - if api_v3.schema_manager and installed_plugin_id: - api_v3.schema_manager.invalidate_cache(installed_plugin_id) - - # Discover and load the new plugin - if api_v3.plugin_manager and installed_plugin_id: - api_v3.plugin_manager.discover_plugins() - api_v3.plugin_manager.load_plugin(installed_plugin_id) - - branch_msg = f" (branch: {result.get('branch', branch)})" if (result.get('branch') or branch) else "" - response_data = { - 'status': 'success', - 'message': f"Plugin {installed_plugin_id} installed successfully{branch_msg}", - 'plugin_id': installed_plugin_id, - 'name': result.get('name') - } - if result.get('branch'): - response_data['branch'] = result.get('branch') - return jsonify(response_data) - else: - return jsonify({ - 'status': 'error', - 'message': result.get('error', 'Failed to install plugin from URL') - }), 500 - - except Exception as e: - logger.error('Error in install_plugin_from_url', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/registry-from-url', methods=['POST']) -def get_registry_from_url(): - """Get plugin list from a registry-style monorepo URL""" - try: - if not api_v3.plugin_store_manager: - return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - - data = request.get_json(silent=True) - if not data or 'repo_url' not in data: - return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 - - # A non-string repo_url is a client mistake, not a server fault: - # .strip() would raise and the catch-all would report it as a 500. - if not isinstance(data['repo_url'], str) or not data['repo_url'].strip(): - return jsonify({'status': 'error', 'message': 'repo_url must be a non-empty string'}), 400 - - repo_url = data['repo_url'].strip() - - # Get registry from the URL - registry = api_v3.plugin_store_manager.fetch_registry_from_url(repo_url) - - if registry: - return jsonify({ - 'status': 'success', - 'plugins': registry.get('plugins', []), - 'registry_url': repo_url - }) - else: - return jsonify({ - 'status': 'error', - 'message': 'Failed to fetch registry from URL or URL does not contain a valid registry' - }), 400 - - except Exception as e: - logger.error('Error in get_registry_from_url', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/saved-repositories', methods=['GET']) -def get_saved_repositories(): - """Get all saved repositories""" - try: - if not api_v3.saved_repositories_manager: - return jsonify({'status': 'error', 'message': 'Saved repositories manager not initialized'}), 500 - - repositories = api_v3.saved_repositories_manager.get_all() - return jsonify({'status': 'success', 'data': {'repositories': repositories}}) - except Exception as e: - logger.error('Error in get_saved_repositories', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/saved-repositories', methods=['POST']) -def add_saved_repository(): - """Add a repository to saved list""" - try: - if not api_v3.saved_repositories_manager: - return jsonify({'status': 'error', 'message': 'Saved repositories manager not initialized'}), 500 - - data = request.get_json(silent=True) - if not data or 'repo_url' not in data: - return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 - - # A non-string repo_url is a client mistake, not a server fault: - # .strip() would raise and the catch-all would report it as a 500. - if not isinstance(data['repo_url'], str) or not data['repo_url'].strip(): - return jsonify({'status': 'error', 'message': 'repo_url must be a non-empty string'}), 400 - - repo_url = data['repo_url'].strip() - name = data.get('name') - - success = api_v3.saved_repositories_manager.add(repo_url, name) - - if success: - return jsonify({ - 'status': 'success', - 'message': 'Repository saved successfully', - 'data': {'repositories': api_v3.saved_repositories_manager.get_all()} - }) - else: - return jsonify({ - 'status': 'error', - 'message': 'Repository already exists or failed to save' - }), 400 - except Exception as e: - logger.error('Error in add_saved_repository', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/saved-repositories', methods=['DELETE']) -def remove_saved_repository(): - """Remove a repository from saved list""" - try: - if not api_v3.saved_repositories_manager: - return jsonify({'status': 'error', 'message': 'Saved repositories manager not initialized'}), 500 - - data = request.get_json(silent=True) - if not data or 'repo_url' not in data: - return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 - - repo_url = data['repo_url'] - - success = api_v3.saved_repositories_manager.remove(repo_url) - - if success: - return jsonify({ - 'status': 'success', - 'message': 'Repository removed successfully', - 'data': {'repositories': api_v3.saved_repositories_manager.get_all()} - }) - else: - return jsonify({ - 'status': 'error', - 'message': 'Repository not found' - }), 404 - except Exception as e: - logger.error('Error in remove_saved_repository', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/store/list', methods=['GET']) -def list_plugin_store(): - """Search plugin store""" - try: - if not api_v3.plugin_store_manager: - return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - - query = request.args.get('query', '') - category = request.args.get('category', '') - tags = request.args.getlist('tags') - # Default to fetching commit metadata to ensure accurate commit timestamps - fetch_commit_param = request.args.get('fetch_commit_info', request.args.get('fetch_latest_versions', '')).lower() - fetch_commit = fetch_commit_param != 'false' - - # Search plugins from the registry (including saved repositories) - plugins = api_v3.plugin_store_manager.search_plugins( - query=query, - category=category, - tags=tags, - fetch_commit_info=fetch_commit, - include_saved_repos=True, - saved_repositories_manager=api_v3.saved_repositories_manager - ) - - # Format plugins for the web interface - formatted_plugins = [] - for plugin in plugins: - formatted_plugins.append({ - 'id': plugin.get('id'), - 'name': plugin.get('name'), - 'author': plugin.get('author'), - 'category': plugin.get('category'), - 'description': plugin.get('description'), - 'tags': plugin.get('tags', []), - 'stars': plugin.get('stars', 0), - 'verified': plugin.get('verified', False), - 'repo': plugin.get('repo', ''), - 'last_updated': plugin.get('last_updated') or plugin.get('last_updated_iso', ''), - 'last_updated_iso': plugin.get('last_updated_iso', ''), - 'last_commit': plugin.get('last_commit') or plugin.get('last_commit_sha'), - 'last_commit_message': plugin.get('last_commit_message'), - 'last_commit_author': plugin.get('last_commit_author'), - 'version': plugin.get('latest_version') or plugin.get('version', ''), - 'branch': plugin.get('branch') or plugin.get('default_branch'), - 'default_branch': plugin.get('default_branch'), - 'plugin_path': plugin.get('plugin_path', '') - }) - - return jsonify({'status': 'success', 'data': {'plugins': formatted_plugins}}) - except Exception as e: - logger.error('Error in list_plugin_store', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/store/github-status', methods=['GET']) -def get_github_auth_status(): - """Check if GitHub authentication is configured and validate token""" - try: - if not api_v3.plugin_store_manager: - return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - - token = api_v3.plugin_store_manager.github_token - - # Check if GitHub token is configured - if not token or len(token) == 0: - return jsonify({ - 'status': 'success', - 'data': { - 'token_status': 'none', - 'authenticated': False, - 'rate_limit': 60, - 'message': 'No GitHub token configured', - 'error': None - } - }) - - # Validate the token - is_valid, error_message = api_v3.plugin_store_manager._validate_github_token(token) - - if is_valid: - return jsonify({ - 'status': 'success', - 'data': { - 'token_status': 'valid', - 'authenticated': True, - 'rate_limit': 5000, - 'message': 'GitHub API authenticated', - 'error': None - } - }) - else: - return jsonify({ - 'status': 'success', - 'data': { - 'token_status': 'invalid', - 'authenticated': False, - 'rate_limit': 60, - 'message': f'GitHub token is invalid: {error_message}' if error_message else 'GitHub token is invalid', - 'error': error_message - } - }) - except Exception as e: - logger.error('Error in get_github_auth_status', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/store/refresh', methods=['POST']) -def refresh_plugin_store(): - """Refresh plugin store repository""" - try: - if not api_v3.plugin_store_manager: - return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 - - data = request.get_json(silent=True) or {} - fetch_commit_info = data.get('fetch_commit_info', data.get('fetch_latest_versions', False)) - - # Force refresh the registry - registry = api_v3.plugin_store_manager.fetch_registry(force_refresh=True) - plugin_count = len(registry.get('plugins', [])) - - message = 'Plugin store refreshed' - if fetch_commit_info: - message += ' (with refreshed commit metadata from GitHub)' - - return jsonify({ - 'status': 'success', - 'message': message, - 'plugin_count': plugin_count - }) - except Exception as e: - logger.error('Error in refresh_plugin_store', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -def deep_merge(base_dict, update_dict): - """ - Deep merge update_dict into base_dict. - For nested dicts, recursively merge. For other types, update_dict takes precedence. - - Lists are intentionally REPLACED wholesale, never index-merged: form posts - carry complete arrays, and index-merging would resurrect items the user - deleted. This also applies to the parallel secrets lists produced by - separate_secrets — a newly saved secrets list is authoritative. - """ - result = base_dict.copy() - for key, value in update_dict.items(): - if key in result and isinstance(result[key], dict) and isinstance(value, dict): - # Recursively merge nested dicts - result[key] = deep_merge(result[key], value) - else: - # For non-dict values or new keys, use the update value - result[key] = value - return result - - -def _parse_form_value(value): - """ - Parse a form value into the appropriate Python type. - Handles booleans, numbers, JSON arrays/objects, and strings. - """ - import json - - if value is None: - return None - - # Handle string values - if isinstance(value, str): - stripped = value.strip() - - # Check for boolean strings - if stripped.lower() == 'true': - return True - if stripped.lower() == 'false': - return False - if stripped.lower() in ('null', 'none') or stripped == '': - return None - - # Try parsing as JSON (for arrays and objects) - do this BEFORE number parsing - # This handles RGB arrays like "[255, 0, 0]" correctly - if stripped.startswith('[') or stripped.startswith('{'): - try: - return json.loads(stripped) - except json.JSONDecodeError: - pass - - # Try parsing as number - try: - if '.' in stripped: - return float(stripped) - return int(stripped) - except ValueError: - pass - - # Return as string (original value, not stripped) - return value - - return value - - -def _get_schema_property(schema, key_path): - """ - Get the schema property for a given key path (supports dot notation). - - Args: - schema: The JSON schema dict - key_path: Dot-separated path like "customization.time_text.font" - - Returns: - The property schema dict or None if not found - """ - if not schema or 'properties' not in schema: - return None - - parts = key_path.split('.') - current = schema['properties'] - i = 0 - - while i < len(parts): - # Try progressively longer candidates, longest first, so schema keys that - # themselves contain dots (e.g. league keys like "fifa.world") are matched - # instead of being mistaken for nested "fifa" -> "world" objects. - matched = False - for j in range(len(parts), i, -1): - candidate = '.'.join(parts[i:j]) - if isinstance(current, dict) and candidate in current: - prop = current[candidate] - # Consumed all remaining parts — this is the target property. - if j == len(parts): - return prop - # Navigate deeper through an object with properties. - if isinstance(prop, dict) and 'properties' in prop: - current = prop['properties'] - i = j - matched = True - break - # Matched a non-object before consuming the path — can't go deeper. - return None - if not matched: - return None - - return None - - -def _is_field_required(key_path, schema): - """ - Check if a field is required according to the schema. - - Args: - key_path: Dot-separated path like "mqtt.username" - schema: The JSON schema dict - - Returns: - True if field is required, False otherwise - """ - if not schema or 'properties' not in schema: - return False - - parts = key_path.split('.') - if len(parts) == 1: - # Top-level field - required = schema.get('required', []) - return parts[0] in required - else: - # Nested field - navigate to parent object - parent_path = '.'.join(parts[:-1]) - field_name = parts[-1] - - # Get parent property - parent_prop = _get_schema_property(schema, parent_path) - if not parent_prop or 'properties' not in parent_prop: - return False - - # Check if field is required in parent - required = parent_prop.get('required', []) - return field_name in required - - -# Sentinel object to indicate a field should be skipped (not set in config) -_SKIP_FIELD = object() - -def _parse_form_value_with_schema(value, key_path, schema): - """ - Parse a form value using schema information to determine correct type. - Handles arrays (comma-separated strings), objects, and other types. - - Args: - value: The form value (usually a string) - key_path: Dot-separated path like "category_order" or "customization.time_text.font" - schema: The plugin's JSON schema - - Returns: - Parsed value with correct type, or _SKIP_FIELD to indicate the field should not be set - """ - import json - - # Get the schema property for this field - prop = _get_schema_property(schema, key_path) - - # Handle None/empty values - if value is None or (isinstance(value, str) and value.strip() == ''): - # If schema says it's an array, return empty array instead of None - if prop and prop.get('type') == 'array': - return [] - # If schema says it's an object, return empty dict instead of None - if prop and prop.get('type') == 'object': - return {} - # If it's an optional string field, preserve empty string instead of None - if prop and prop.get('type') == 'string': - if not _is_field_required(key_path, schema): - return "" # Return empty string for optional string fields - # For number/integer fields, check if they have defaults or are required - if prop: - prop_type = prop.get('type') - if prop_type in ('number', 'integer'): - # If field has a default, use it - if 'default' in prop: - return prop['default'] - # If field is not required and has no default, skip setting it - if not _is_field_required(key_path, schema): - return _SKIP_FIELD - # If field is required but empty, return None (validation will fail, which is correct) - return None - return None - - # Handle string values - if isinstance(value, str): - stripped = value.strip() - - # Check for boolean strings - if stripped.lower() == 'true': - return True - if stripped.lower() == 'false': - return False - # "on"/"off" come from HTML checkboxes — only coerce when schema says boolean - if prop and prop.get('type') == 'boolean': - if stripped.lower() == 'on': - return True - if stripped.lower() == 'off': - return False - - # Handle arrays based on schema - if prop and prop.get('type') == 'array': - # Try parsing as JSON first (handles "[1,2,3]" format) - if stripped.startswith('['): - try: - return json.loads(stripped) - except json.JSONDecodeError: - pass - - # Otherwise, treat as comma-separated string - if stripped: - # Split by comma and strip each item - items = [item.strip() for item in stripped.split(',') if item.strip()] - # Try to convert items to numbers if schema items are numbers - items_schema = prop.get('items', {}) - if items_schema.get('type') in ('number', 'integer'): - try: - return [int(item) if '.' not in item else float(item) for item in items] - except ValueError: - pass - return items - return [] - - # Handle objects based on schema - if prop and prop.get('type') == 'object': - # Try parsing as JSON - if stripped.startswith('{'): - try: - return json.loads(stripped) - except json.JSONDecodeError: - pass - # If it's not JSON, return empty dict (form shouldn't send objects as strings) - return {} - - # Try parsing as JSON (for arrays and objects) - do this BEFORE number parsing - if stripped.startswith('[') or stripped.startswith('{'): - try: - return json.loads(stripped) - except json.JSONDecodeError: - pass - - # Handle numbers based on schema - if prop: - prop_type = prop.get('type') - if prop_type == 'integer': - try: - return int(stripped) - except ValueError: - return prop.get('default', 0) - elif prop_type == 'number': - try: - return float(stripped) - except ValueError: - return prop.get('default', 0.0) - - # Try parsing as number (fallback) — skip when schema explicitly says string - if not (prop and prop.get('type') == 'string'): - try: - if '.' in stripped: - return float(stripped) - return int(stripped) - except ValueError: - pass - - # Return as string - return value - - return value - - -def _resolve_key_segments(key_path, config): - """Split a dot-notation path into segments, greedily preserving keys that - themselves contain dots (e.g. league keys like "fifa.world"). - - At each level the longest candidate that matches a key already present in the - config wins; otherwise the path splits on the next dot (the normal - nested-create case). Because dotted keys such as ``leagues."fifa.world"`` - always exist in the saved config being updated, this routes the value to the - real league object instead of fabricating a ``leagues.fifa.world`` tree. - """ - parts = key_path.split('.') - segments = [] - node = config - i = 0 - while i < len(parts): - matched = False - if isinstance(node, dict): - for j in range(len(parts), i, -1): - candidate = '.'.join(parts[i:j]) - if candidate in node: - segments.append(candidate) - node = node[candidate] - i = j - matched = True - break - if not matched: - part = parts[i] - segments.append(part) - node = node.get(part) if isinstance(node, dict) else None - i += 1 - return segments - - -def _set_nested_value(config, key_path, value): - """ - Set a value in a nested dict using dot notation path. - Handles existing nested dicts correctly by merging instead of replacing. - Keys containing dots (e.g. league keys like "fifa.world") are preserved when - they already exist in the config rather than being split into nested objects. - - Args: - config: The config dict to modify - key_path: Dot-separated path (e.g., "customization.period_text.font") - value: The value to set (or _SKIP_FIELD to skip setting) - """ - # Skip setting if value is the sentinel - if value is _SKIP_FIELD: - return - - segments = _resolve_key_segments(key_path, config) - current = config - - # Navigate/create intermediate dicts - for seg in segments[:-1]: - if seg not in current: - current[seg] = {} - elif not isinstance(current[seg], dict): - # If the existing value is not a dict, replace it with a dict - current[seg] = {} - current = current[seg] - - # Set the final value (don't overwrite with empty dict if value is None and we want to preserve structure) - if value is not None or segments[-1] not in current: - current[segments[-1]] = value - - -def _set_missing_booleans_to_false(config, schema_props, form_keys, prefix='', config_node=None): - """Walk schema and set missing boolean form fields to False. - - HTML checkboxes don't submit values when unchecked. When saving plugin config, - the backend starts from existing config (to support partial form updates), which - means an unchecked checkbox's old ``True`` value persists. This function detects - boolean schema properties not present in the form submission and explicitly sets - them to ``False``. - - The top-level ``enabled`` field is excluded because it has its own preservation - logic in the save endpoint. - - Handles boolean fields inside nested objects and inside arrays of objects - (e.g. ``feeds.custom_feeds.0.enabled``). - - Args: - config: The root plugin config dict (used for pure-dict paths) - schema_props: Schema ``properties`` dict at the current nesting level - form_keys: Set of form field names that were submitted - prefix: Dot-notation prefix for the current nesting level - config_node: The current config subtree when inside an array item (avoids - using _set_nested_value which corrupts lists) - """ - # Determine which config node to operate on - node = config_node if config_node is not None else config - - for prop_name, prop_schema in schema_props.items(): - if not isinstance(prop_schema, dict): - continue - - full_path = f"{prefix}.{prop_name}" if prefix else prop_name - prop_type = prop_schema.get('type') - - if prop_type == 'boolean' and full_path != 'enabled': - # If this boolean wasn't submitted in the form, it's an unchecked checkbox - if full_path not in form_keys: - if config_node is not None: - # Inside an array item — set directly on the item dict - node[prop_name] = False - else: - # Pure dict path — use helper - _set_nested_value(config, full_path, False) - - elif prop_type == 'object' and 'properties' in prop_schema: - # Recurse into nested objects - if config_node is not None: - # Inside an array item — ensure nested dict exists in item - if prop_name not in node or not isinstance(node[prop_name], dict): - node[prop_name] = {} - _set_missing_booleans_to_false( - config, prop_schema['properties'], form_keys, full_path, - config_node=node[prop_name] - ) - else: - _set_missing_booleans_to_false( - config, prop_schema['properties'], form_keys, full_path - ) - - elif prop_type == 'array': - # Handle arrays of objects that may contain boolean fields - # Form keys use indexed notation: "path.0.field", "path.1.field" - items_schema = prop_schema.get('items', {}) - if isinstance(items_schema, dict) and items_schema.get('type') == 'object' and 'properties' in items_schema: - array_prefix = f"{full_path}." - # Collect unique item indices from submitted form keys - indices = set() - for k in form_keys: - if k.startswith(array_prefix): - # Extract index: "path.0.field" -> "0" - rest = k[len(array_prefix):] - idx = rest.split('.', 1)[0] - if idx.isdigit(): - indices.add(int(idx)) - - if not indices: - continue - - # Navigate to the array in the config (create if missing) - if config_node is not None: - if prop_name not in node or not isinstance(node[prop_name], list): - node[prop_name] = [] - array_list = node[prop_name] - else: - # Navigate from root config through dict keys to get the list - parts = full_path.split('.') - current = config - for part in parts[:-1]: - if part not in current or not isinstance(current[part], dict): - current[part] = {} - current = current[part] - arr_key = parts[-1] - if arr_key not in current or not isinstance(current[arr_key], list): - current[arr_key] = [] - array_list = current[arr_key] - - # Recurse into each array item so its missing booleans get set to False - for idx in indices: - # Ensure list is long enough and item is a dict - while len(array_list) <= idx: - array_list.append({}) - if not isinstance(array_list[idx], dict): - array_list[idx] = {} - item_prefix = f"{full_path}.{idx}" - _set_missing_booleans_to_false( - config, items_schema['properties'], form_keys, item_prefix, - config_node=array_list[idx] - ) - - -def _enhance_schema_with_core_properties(schema): - """ - Enhance schema with core plugin properties (enabled, display_duration, live_priority). - These properties are system-managed and should always be allowed even if not in the plugin's schema. - - Args: - schema: The original JSON schema dict - - Returns: - Enhanced schema dict with core properties injected - """ - import copy - - if not schema: - return schema - - # Core plugin properties that should always be allowed - # These match the definitions in SchemaManager.validate_config_against_schema() - core_properties = { - "enabled": { - "type": "boolean", - "default": True, - "description": "Enable or disable this plugin" - }, - "display_duration": { - "type": "number", - "default": 15, - "minimum": 1, - "maximum": 300, - "description": "How long to display this plugin in seconds" - }, - "live_priority": { - "type": "boolean", - "default": False, - "description": "Enable live priority takeover when plugin has live content" - } - } - - # Create a deep copy of the schema to modify (to avoid mutating the original) - enhanced_schema = copy.deepcopy(schema) - if "properties" not in enhanced_schema: - enhanced_schema["properties"] = {} - - # Inject core properties if they're not already defined in the schema - for prop_name, prop_def in core_properties.items(): - if prop_name not in enhanced_schema["properties"]: - enhanced_schema["properties"][prop_name] = copy.deepcopy(prop_def) - - return enhanced_schema - - -def _filter_config_by_schema(config, schema, prefix=''): - """ - Filter config to only include fields defined in the schema. - Removes fields not in schema, especially important when additionalProperties is false. - - Args: - config: The config dict to filter - schema: The JSON schema dict - prefix: Prefix for nested paths (used recursively) - - Returns: - Filtered config dict containing only schema-defined fields - """ - if not schema or 'properties' not in schema: - return config - - filtered = {} - schema_props = schema.get('properties', {}) - - for key, value in config.items(): - if key not in schema_props: - # Field not in schema, skip it - continue - - prop_schema = schema_props[key] - - # Handle nested objects recursively - if isinstance(value, dict) and prop_schema.get('type') == 'object' and 'properties' in prop_schema: - filtered[key] = _filter_config_by_schema(value, prop_schema, f"{prefix}.{key}" if prefix else key) - else: - # Keep the value as-is for non-object types - filtered[key] = value - - return filtered - - -@api_v3.route('/plugins/config', methods=['POST']) -def save_plugin_config(): - """Save plugin configuration, separating secrets from regular config""" - try: - if not api_v3.config_manager: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'Config manager not initialized', - status_code=500 - ) - - # Support both JSON and form data (for HTMX submissions) - content_type = request.content_type or '' - - if 'application/json' in content_type: - # JSON request - data, error = validate_request_json(['plugin_id']) - if error: - return error - plugin_id = data['plugin_id'] - plugin_config = data.get('config', {}) - else: - # Form data (HTMX submission) - # plugin_id comes from query string, config from form fields - plugin_id = request.args.get('plugin_id') - if not plugin_id: - return error_response( - ErrorCode.INVALID_INPUT, - 'plugin_id required in query string', - status_code=400 - ) - - # Load existing config as base (partial form updates should merge, not replace) - existing_config = {} - if api_v3.config_manager: - full_config = api_v3.config_manager.load_config() - existing_config = full_config.get(plugin_id, {}).copy() - - # Get schema manager instance (needed for type conversion) - schema_mgr = api_v3.schema_manager - if not schema_mgr: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'Schema manager not initialized', - status_code=500 - ) - - # Load plugin schema BEFORE processing form data (needed for type conversion) - schema = schema_mgr.load_schema(plugin_id, use_cache=False) - - # Start with existing config and apply form updates - plugin_config = existing_config - - # Convert form data to config dict - # Form fields can use dot notation for nested values (e.g., "transition.type") - form_data = request.form.to_dict() - - # First pass: handle bracket notation array fields (e.g., "field_name[]" from checkbox-group) - # These fields use getlist() to preserve all values, then replace in form_data - # Sentinel empty value ("") allows clearing array to [] when all checkboxes unchecked - bracket_array_fields = {} # Maps base field path to list of values - for key in request.form.keys(): - # Check if key ends with "[]" (bracket notation for array fields) - if key.endswith('[]'): - base_path = key[:-2] # Remove "[]" suffix - values = request.form.getlist(key) - # Filter out sentinel empty string - if only sentinel present, array should be [] - # If sentinel + values present, use the actual values - filtered_values = [v for v in values if v and v.strip()] - # If no non-empty values but key exists, it means all checkboxes unchecked (empty array) - bracket_array_fields[base_path] = filtered_values - # Remove the bracket notation key from form_data if present - if key in form_data: - del form_data[key] - - # Process bracket notation fields and set directly in plugin_config - # Use JSON encoding instead of comma-join to handle values containing commas - import json - for base_path, values in bracket_array_fields.items(): - # Get schema property to verify it's an array - base_prop = _get_schema_property(schema, base_path) - if base_prop and base_prop.get('type') == 'array': - # Filter out empty values and sentinel empty strings - filtered_values = [v for v in values if v and v.strip()] - # Set directly in plugin_config (values are already strings, no need to parse) - # Empty array (all unchecked) is represented as [] - _set_nested_value(plugin_config, base_path, filtered_values) - logger.debug(f"Processed bracket notation array field {base_path}: {values} -> {filtered_values}") - # Remove from form_data to avoid double processing - if base_path in form_data: - del form_data[base_path] - - # Second pass: detect and combine array index fields (e.g., "text_color.0", "text_color.1" -> "text_color" as array) - # This handles cases where forms send array fields as indexed inputs - array_fields = {} # Maps base field path to list of (index, value) tuples - processed_keys = set() - indexed_base_paths = set() # Track which base paths have indexed fields - - for key, value in form_data.items(): - # Check if this looks like an array index field (ends with .0, .1, .2, etc.) - if '.' in key: - parts = key.rsplit('.', 1) # Split on last dot - if len(parts) == 2: - base_path, last_part = parts - # Check if last part is a numeric string (array index) - if last_part.isdigit(): - # Get schema property for the base path to verify it's an array - base_prop = _get_schema_property(schema, base_path) - if base_prop and base_prop.get('type') == 'array': - # This is an array index field - index = int(last_part) - if base_path not in array_fields: - array_fields[base_path] = [] - array_fields[base_path].append((index, value)) - processed_keys.add(key) - indexed_base_paths.add(base_path) - continue - - # Process combined array fields - for base_path, index_values in array_fields.items(): - # Sort by index and extract values - index_values.sort(key=lambda x: x[0]) - values = [v for _, v in index_values] - # Combine values into comma-separated string for parsing - combined_value = ', '.join(str(v) for v in values) - # Parse as array using schema - parsed_value = _parse_form_value_with_schema(combined_value, base_path, schema) - # Debug logging - logger.debug(f"Combined indexed array field {base_path}: {values} -> {combined_value} -> {parsed_value}") - # Only set if not skipped - if parsed_value is not _SKIP_FIELD: - _set_nested_value(plugin_config, base_path, parsed_value) - - # Process remaining (non-indexed) fields - # Skip any base paths that were processed as indexed arrays - for key, value in form_data.items(): - if key not in processed_keys: - # Skip if this key is a base path that was processed as indexed array - # (to avoid overwriting the combined array with a single value) - if key not in indexed_base_paths: - # Parse value using schema to determine correct type - parsed_value = _parse_form_value_with_schema(value, key, schema) - # Debug logging for array fields - if schema: - prop = _get_schema_property(schema, key) - if prop and prop.get('type') == 'array': - logger.debug(f"Array field {key}: form value='{value}' -> parsed={parsed_value}") - # Use helper to set nested values correctly (skips if _SKIP_FIELD) - if parsed_value is not _SKIP_FIELD: - _set_nested_value(plugin_config, key, parsed_value) - - # Post-process: Fix array fields that might have been incorrectly structured - # This handles cases where array fields are stored as dicts (e.g., from indexed form fields) - def fix_array_structures(config_dict, schema_props, prefix=''): - """Recursively fix array structures (convert dicts with numeric keys to arrays, fix length issues)""" - for prop_key, prop_schema in schema_props.items(): - prop_type = prop_schema.get('type') - - if prop_type == 'array': - # Navigate to the field location - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if isinstance(parent, dict) and part in parent: - parent = parent[part] - else: - parent = None - break - - if parent is not None and isinstance(parent, dict) and prop_key in parent: - current_value = parent[prop_key] - # If it's a dict with numeric string keys, convert to array - if isinstance(current_value, dict) and not isinstance(current_value, list): - try: - # Check if all keys are numeric strings (array indices) - keys = [k for k in current_value.keys()] - if all(k.isdigit() for k in keys): - # Convert to sorted array by index - sorted_keys = sorted(keys, key=int) - array_value = [current_value[k] for k in sorted_keys] - # Convert array elements to correct types based on schema - items_schema = prop_schema.get('items', {}) - item_type = items_schema.get('type') - if item_type in ('number', 'integer'): - converted_array = [] - for v in array_value: - if isinstance(v, str): - try: - if item_type == 'integer': - converted_array.append(int(v)) - else: - converted_array.append(float(v)) - except (ValueError, TypeError, OverflowError): - converted_array.append(v) - else: - converted_array.append(v) - array_value = converted_array - parent[prop_key] = array_value - current_value = array_value # Update for length check below - except (ValueError, KeyError, TypeError): - # Conversion failed, check if we should use default - pass - - # If it's an array, ensure correct types and check minItems - if isinstance(current_value, list): - # First, ensure array elements are correct types - items_schema = prop_schema.get('items', {}) - item_type = items_schema.get('type') - if item_type in ('number', 'integer'): - converted_array = [] - for v in current_value: - if isinstance(v, str): - try: - if item_type == 'integer': - converted_array.append(int(v)) - else: - converted_array.append(float(v)) - except (ValueError, TypeError, OverflowError): - converted_array.append(v) - else: - converted_array.append(v) - parent[prop_key] = converted_array - current_value = converted_array - - # Then check minItems - min_items = prop_schema.get('minItems') - if min_items is not None and len(current_value) < min_items: - # Use default if available, otherwise keep as-is (validation will catch it) - default = prop_schema.get('default') - if default and isinstance(default, list) and len(default) >= min_items: - parent[prop_key] = default - else: - # Top-level field - if prop_key in config_dict: - current_value = config_dict[prop_key] - # If it's a dict with numeric string keys, convert to array - if isinstance(current_value, dict) and not isinstance(current_value, list): - try: - keys = list(current_value.keys()) - if keys and all(str(k).isdigit() for k in keys): - sorted_keys = sorted(keys, key=lambda x: int(str(x))) - array_value = [current_value[k] for k in sorted_keys] - # Convert array elements to correct types based on schema - items_schema = prop_schema.get('items', {}) - item_type = items_schema.get('type') - if item_type in ('number', 'integer'): - converted_array = [] - for v in array_value: - if isinstance(v, str): - try: - if item_type == 'integer': - converted_array.append(int(v)) - else: - converted_array.append(float(v)) - except (ValueError, TypeError, OverflowError): - converted_array.append(v) - else: - converted_array.append(v) - array_value = converted_array - config_dict[prop_key] = array_value - current_value = array_value # Update for length check below - except (ValueError, KeyError, TypeError) as e: - logger.debug(f"Failed to convert {prop_key} to array: {e}") - - # If it's an array, ensure correct types and check minItems - if isinstance(current_value, list): - # First, ensure array elements are correct types - items_schema = prop_schema.get('items', {}) - item_type = items_schema.get('type') - if item_type in ('number', 'integer'): - converted_array = [] - for v in current_value: - if isinstance(v, str): - try: - if item_type == 'integer': - converted_array.append(int(v)) - else: - converted_array.append(float(v)) - except (ValueError, TypeError, OverflowError): - converted_array.append(v) - else: - converted_array.append(v) - config_dict[prop_key] = converted_array - current_value = converted_array - - # Then check minItems - min_items = prop_schema.get('minItems') - if min_items is not None and len(current_value) < min_items: - default = prop_schema.get('default') - if default and isinstance(default, list) and len(default) >= min_items: - config_dict[prop_key] = default - - # Recurse into nested objects - elif prop_type == 'object' and 'properties' in prop_schema: - nested_prefix = f"{prefix}.{prop_key}" if prefix else prop_key - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if isinstance(parent, dict) and part in parent: - parent = parent[part] - else: - parent = None - break - nested_dict = parent.get(prop_key) if parent is not None and isinstance(parent, dict) else None - else: - nested_dict = config_dict.get(prop_key) - - if isinstance(nested_dict, dict): - # Pass no prefix: config_dict is already the navigated sub-dict, - # so path segments from the parent would mis-navigate it. - fix_array_structures(nested_dict, prop_schema['properties']) - - # Also ensure array fields that are None get converted to empty arrays - def ensure_array_defaults(config_dict, schema_props, prefix=''): - """Recursively ensure array fields have defaults if None""" - for prop_key, prop_schema in schema_props.items(): - prop_type = prop_schema.get('type') - - if prop_type == 'array': - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if isinstance(parent, dict) and part in parent: - parent = parent[part] - else: - parent = None - break - - if parent is not None and isinstance(parent, dict): - if prop_key not in parent or parent[prop_key] is None: - default = prop_schema.get('default', []) - parent[prop_key] = default if default else [] - else: - if prop_key not in config_dict or config_dict[prop_key] is None: - default = prop_schema.get('default', []) - config_dict[prop_key] = default if default else [] - - elif prop_type == 'object' and 'properties' in prop_schema: - nested_prefix = f"{prefix}.{prop_key}" if prefix else prop_key - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if isinstance(parent, dict) and part in parent: - parent = parent[part] - else: - parent = None - break - nested_dict = parent.get(prop_key) if parent is not None and isinstance(parent, dict) else None - else: - nested_dict = config_dict.get(prop_key) - - if nested_dict is None: - if prefix: - parent_parts = prefix.split('.') - parent = config_dict - for part in parent_parts: - if part not in parent: - parent[part] = {} - parent = parent[part] - if prop_key not in parent: - parent[prop_key] = {} - nested_dict = parent[prop_key] - else: - if prop_key not in config_dict: - config_dict[prop_key] = {} - nested_dict = config_dict[prop_key] - - if isinstance(nested_dict, dict): - # Pass no prefix: config_dict is already navigated. - ensure_array_defaults(nested_dict, prop_schema['properties']) - - if schema and 'properties' in schema: - # First, fix any dict structures that should be arrays - # This must be called BEFORE validation to convert dicts with numeric keys to arrays - fix_array_structures(plugin_config, schema['properties']) - # Then, ensure None arrays get defaults - ensure_array_defaults(plugin_config, schema['properties']) - - # Debug: Log the structure after fixing - if 'feeds' in plugin_config and 'custom_feeds' in plugin_config.get('feeds', {}): - custom_feeds = plugin_config['feeds']['custom_feeds'] - logger.debug(f"After fix_array_structures: custom_feeds type={type(custom_feeds)}, value={custom_feeds}") - - # Force fix for feeds.custom_feeds if it's still a dict (fallback) - if 'feeds' in plugin_config: - feeds_config = plugin_config.get('feeds') or {} - if feeds_config and 'custom_feeds' in feeds_config and isinstance(feeds_config['custom_feeds'], dict): - custom_feeds_dict = feeds_config['custom_feeds'] - # Check if all keys are numeric - keys = list(custom_feeds_dict.keys()) - if keys and all(str(k).isdigit() for k in keys): - # Convert to array - sorted_keys = sorted(keys, key=lambda x: int(str(x))) - feeds_config['custom_feeds'] = [custom_feeds_dict[k] for k in sorted_keys] - logger.info(f"Force-converted feeds.custom_feeds from dict to array: {len(feeds_config['custom_feeds'])} items") - - # Fix unchecked boolean checkboxes: HTML checkboxes don't submit values - # when unchecked, so the existing config value (potentially True) persists. - # Walk the schema and set any boolean fields missing from form data to False. - if schema and 'properties' in schema: - form_keys = set(request.form.keys()) - _set_missing_booleans_to_false(plugin_config, schema['properties'], form_keys) - - # Get schema manager instance (for JSON requests) - schema_mgr = api_v3.schema_manager - if not schema_mgr: - return error_response( - ErrorCode.SYSTEM_ERROR, - 'Schema manager not initialized', - status_code=500 - ) - - # Load plugin schema using SchemaManager (force refresh to get latest schema) - # For JSON requests, schema wasn't loaded yet - if 'application/json' in content_type: - schema = schema_mgr.load_schema(plugin_id, use_cache=False) - - # JSON path: fix numeric-keyed dicts that should be arrays. - # JS dotToNested() converts feeds.custom_feeds.0.name → {'0': {name:...}} - # instead of [{name:...}]. The form-data path has fix_array_structures for this; - # mirror that logic here for JSON submissions. - if 'application/json' in content_type and schema and 'properties' in schema: - def _fix_json_arrays(cfg, props): - for k, ps in props.items(): - if not isinstance(cfg, dict) or k not in cfg: - continue - pt = ps.get('type') - val = cfg[k] - if pt == 'array': - items_schema = ps.get('items', {}) - item_type = items_schema.get('type') - if isinstance(val, dict): - keys = list(val.keys()) - if keys and all(str(x).isdigit() for x in keys): - sorted_keys = sorted(keys, key=lambda x: int(str(x))) - arr = [val[sk] for sk in sorted_keys] - if item_type in ('integer', 'number'): - converted = [] - for v in arr: - if isinstance(v, str): - try: - converted.append(int(v) if item_type == 'integer' else float(v)) - except (ValueError, TypeError, OverflowError): - converted.append(v) - else: - converted.append(v) - arr = converted - cfg[k] = arr - elif not keys: - cfg[k] = [] - # Recurse into each element when items are objects with properties, - # covering both freshly-converted and already-list values. - if item_type == 'object' and 'properties' in items_schema: - for elem in (cfg[k] if isinstance(cfg[k], list) else []): - if isinstance(elem, dict): - _fix_json_arrays(elem, items_schema['properties']) - elif pt == 'object' and 'properties' in ps and isinstance(val, dict): - _fix_json_arrays(val, ps['properties']) - _fix_json_arrays(plugin_config, schema['properties']) - - # PRE-PROCESSING: Preserve 'enabled' state if not in request - # This prevents overwriting the enabled state when saving config from a form that doesn't include the toggle - if 'enabled' not in plugin_config: - try: - current_config = api_v3.config_manager.load_config() - if plugin_id in current_config and 'enabled' in current_config[plugin_id]: - plugin_config['enabled'] = current_config[plugin_id]['enabled'] - # logger.debug(f"Preserving enabled state for {plugin_id}: {plugin_config['enabled']}") - elif api_v3.plugin_manager: - # Fallback to plugin instance if config doesn't have it - plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) - if plugin_instance: - plugin_config['enabled'] = plugin_instance.enabled - # Final fallback: default to True if plugin is loaded (matches BasePlugin default) - if 'enabled' not in plugin_config: - plugin_config['enabled'] = True - except Exception as e: - logger.debug("Error preserving enabled state: %s", e) - # Default to True on error to avoid disabling plugins - plugin_config['enabled'] = True - - # Find secret fields (supports nested schemas and array-item secrets) - secret_fields = set() - - if schema and 'properties' in schema: - secret_fields = find_secret_fields(schema['properties']) - - # Apply defaults from schema to config BEFORE validation - # This ensures required fields with defaults are present before validation - # Store preserved enabled value before merge to protect it from defaults - preserved_enabled = None - if 'enabled' in plugin_config: - preserved_enabled = plugin_config['enabled'] - - if schema: - defaults = schema_mgr.generate_default_config(plugin_id, use_cache=True) - plugin_config = schema_mgr.merge_with_defaults(plugin_config, defaults) - - # After merging defaults, replace any None array values with their schema defaults. - # merge_with_defaults gives user config higher priority, so a None submitted by - # the client can survive the merge — this pass cleans those up. - def _fix_none_arrays(cfg, props): - for k, pschema in props.items(): - if pschema.get('type') == 'array': - if isinstance(cfg, dict) and (k not in cfg or cfg[k] is None): - cfg[k] = pschema.get('default', []) - elif pschema.get('type') == 'object' and 'properties' in pschema: - if isinstance(cfg, dict) and isinstance(cfg.get(k), dict): - _fix_none_arrays(cfg[k], pschema['properties']) - - if schema and 'properties' in schema and isinstance(plugin_config, dict): - _fix_none_arrays(plugin_config, schema['properties']) - - # Ensure enabled state is preserved after defaults merge - # Defaults should not overwrite an explicitly preserved enabled value - if preserved_enabled is not None: - # Restore preserved value if it was changed by defaults merge - if plugin_config.get('enabled') != preserved_enabled: - plugin_config['enabled'] = preserved_enabled - - # Normalize config data: convert string numbers to integers/floats where schema expects numbers - # This handles form data which sends everything as strings - def normalize_config_values(config, schema_props, prefix=''): - """Recursively normalize config values based on schema types""" - if not isinstance(config, dict) or not isinstance(schema_props, dict): - return config - - normalized = {} - for key, value in config.items(): - field_path = f"{prefix}.{key}" if prefix else key - - if key not in schema_props: - # Field not in schema, keep as-is (will be caught by additionalProperties check if needed) - normalized[key] = value - continue - - prop_schema = schema_props[key] - prop_type = prop_schema.get('type') - - # Handle union types (e.g., ["integer", "null"]) - if isinstance(prop_type, list): - # Check if null is allowed and value is empty/null - if 'null' in prop_type: - # Handle various representations of null/empty - if value is None: - normalized[key] = None - continue - elif isinstance(value, str): - # Strip whitespace and check for null representations - value_stripped = value.strip() - if value_stripped == '' or value_stripped.lower() in ('null', 'none', 'undefined'): - normalized[key] = None - continue - - # Try to normalize based on non-null types in the union - # Check integer first (more specific than number) - if 'integer' in prop_type: - if isinstance(value, str): - value_stripped = value.strip() - if value_stripped == '': - # Empty string with null allowed - already handled above, but double-check - if 'null' in prop_type: - normalized[key] = None - continue - try: - normalized[key] = int(value_stripped) - continue - except (ValueError, TypeError, OverflowError): - pass - elif isinstance(value, (int, float)): - normalized[key] = int(value) - continue - - # Check number (less specific, but handles floats) - if 'number' in prop_type: - if isinstance(value, str): - value_stripped = value.strip() - if value_stripped == '': - # Empty string with null allowed - already handled above, but double-check - if 'null' in prop_type: - normalized[key] = None - continue - try: - normalized[key] = float(value_stripped) - continue - except (ValueError, TypeError, OverflowError): - pass - elif isinstance(value, (int, float)): - normalized[key] = float(value) - continue - - # Check boolean - if 'boolean' in prop_type: - if isinstance(value, str): - normalized[key] = value.strip().lower() in ('true', '1', 'on', 'yes') - continue - - # If no conversion worked and null is allowed, try to set to None - # This handles cases where the value is an empty string or can't be converted - if 'null' in prop_type: - if isinstance(value, str): - value_stripped = value.strip() - if value_stripped == '' or value_stripped.lower() in ('null', 'none', 'undefined'): - normalized[key] = None - continue - # If it's already None, keep it - if value is None: - normalized[key] = None - continue - - # If no conversion worked, keep original value (will fail validation, but that's expected) - # Log a warning for debugging - logger.warning(f"Could not normalize field {field_path}: value={repr(value)}, type={type(value)}, schema_type={prop_type}") - normalized[key] = value - continue - - if isinstance(value, dict) and prop_type == 'object' and 'properties' in prop_schema: - # Recursively normalize nested objects - normalized[key] = normalize_config_values(value, prop_schema['properties'], field_path) - elif isinstance(value, list) and prop_type == 'array' and 'items' in prop_schema: - # Normalize array items - items_schema = prop_schema['items'] - item_type = items_schema.get('type') - - # Handle union types in array items - if isinstance(item_type, list): - normalized_array = [] - for v in value: - # Check if null is allowed - if 'null' in item_type: - if v is None or v == '' or (isinstance(v, str) and v.lower() in ('null', 'none')): - normalized_array.append(None) - continue - - # Try to normalize based on non-null types - if 'integer' in item_type: - if isinstance(v, str): - try: - normalized_array.append(int(v)) - continue - except (ValueError, TypeError, OverflowError): - pass - elif isinstance(v, (int, float)): - normalized_array.append(int(v)) - continue - elif 'number' in item_type: - if isinstance(v, str): - try: - normalized_array.append(float(v)) - continue - except (ValueError, TypeError, OverflowError): - pass - elif isinstance(v, (int, float)): - normalized_array.append(float(v)) - continue - - # If no conversion worked, keep original value - normalized_array.append(v) - normalized[key] = normalized_array - elif item_type == 'integer': - # Convert string numbers to integers - normalized_array = [] - for v in value: - if isinstance(v, str): - try: - normalized_array.append(int(v)) - except (ValueError, TypeError, OverflowError): - normalized_array.append(v) - elif isinstance(v, (int, float)): - normalized_array.append(int(v)) - else: - normalized_array.append(v) - normalized[key] = normalized_array - elif item_type == 'number': - # Convert string numbers to floats - normalized_array = [] - for v in value: - if isinstance(v, str): - try: - normalized_array.append(float(v)) - except (ValueError, TypeError, OverflowError): - normalized_array.append(v) - else: - normalized_array.append(v) - normalized[key] = normalized_array - elif item_type == 'object' and 'properties' in items_schema: - # Recursively normalize array of objects - normalized_array = [] - for v in value: - if isinstance(v, dict): - normalized_array.append( - normalize_config_values(v, items_schema['properties'], f"{field_path}[]") - ) - else: - normalized_array.append(v) - normalized[key] = normalized_array - else: - normalized[key] = value - elif prop_type == 'integer': - # Convert string to integer - if isinstance(value, str): - try: - normalized[key] = int(value) - except (ValueError, TypeError, OverflowError): - normalized[key] = value - else: - normalized[key] = value - elif prop_type == 'number': - # Convert string to float - if isinstance(value, str): - try: - normalized[key] = float(value) - except (ValueError, TypeError, OverflowError): - normalized[key] = value - else: - normalized[key] = value - elif prop_type == 'boolean': - # Convert string booleans - if isinstance(value, str): - normalized[key] = value.lower() in ('true', '1', 'on', 'yes') - else: - normalized[key] = value - else: - normalized[key] = value - - return normalized - - # Normalize config before validation - if schema and 'properties' in schema: - plugin_config = normalize_config_values(plugin_config, schema['properties']) - - # Filter config to only include schema-defined fields (important when additionalProperties is false) - # Use enhanced schema with core properties to ensure core properties are preserved during filtering - if schema and 'properties' in schema: - enhanced_schema_for_filtering = _enhance_schema_with_core_properties(schema) - plugin_config = _filter_config_by_schema(plugin_config, enhanced_schema_for_filtering) - - # Debug logging for union type fields (temporary) - if 'rotation_settings' in plugin_config and 'random_seed' in plugin_config.get('rotation_settings', {}): - seed_value = plugin_config['rotation_settings']['random_seed'] - logger.debug(f"After normalization, random_seed value: {repr(seed_value)}, type: {type(seed_value)}") - - # Validate configuration against schema before saving - if schema: - # Log what we're validating for debugging - logger.info(f"Validating config for {plugin_id}") - # Only the shape. plugin_config still holds the submitted secret - # values at this point -- separate_secrets does not run until - # below -- so logging it wrote live credentials to the journal. - logger.info(f"Config keys being validated: {list(plugin_config.keys())}") - - # Get enhanced schema keys (including injected core properties) - # We need to create an enhanced schema to get the actual allowed keys - import copy - enhanced_schema = copy.deepcopy(schema) - if "properties" not in enhanced_schema: - enhanced_schema["properties"] = {} - - # Core properties that are always injected during validation - core_properties = ["enabled", "display_duration", "live_priority"] - for prop_name in core_properties: - if prop_name not in enhanced_schema["properties"]: - # Add placeholder to get the full list of allowed keys - enhanced_schema["properties"][prop_name] = {"type": "any"} - - is_valid, validation_errors = schema_mgr.validate_config_against_schema( - plugin_config, schema, plugin_id - ) - if not is_valid: - # Log validation errors for debugging - logger.error(f"Config validation failed for {plugin_id}") - logger.error(f"Validation errors: {validation_errors}") - # Keys only, for the same reason as above. - logger.error(f"Config keys that failed: {list(plugin_config.keys())}") - logger.error(f"Schema properties: {list(enhanced_schema.get('properties', {}).keys())}") - - # Also print to console for immediate visibility - import json - logger.warning("Config validation failed for plugin (see debug logs)") - - # Log raw form data if this was a form submission - if 'application/json' not in (request.content_type or ''): - form_data = request.form.to_dict() - return error_response( - ErrorCode.CONFIG_VALIDATION_FAILED, - 'Configuration validation failed', - details='; '.join(validation_errors) if validation_errors else 'Unknown validation error', - context={ - 'plugin_id': plugin_id, - 'validation_errors': validation_errors, - 'config_keys': list(plugin_config.keys()), - 'schema_keys': list(enhanced_schema.get('properties', {}).keys()) - }, - suggested_fixes=[ - 'Review validation errors above', - 'Check config against schema', - 'Verify all required fields are present' - ], - status_code=400 - ) - - # Separate secrets from regular config (handles nested configs and - # array-item secrets — see src/web_interface/secret_helpers.py) - regular_config, secrets_config = separate_secrets(plugin_config, secret_fields) - # The config form renders secrets masked, so every save posts - # them back blank. Without this the blank is merged over the - # stored value and the credential is destroyed by the act of - # changing an unrelated setting. A blank means "unchanged". - secrets_config = remove_empty_secrets(secrets_config) - - # Get current configs - current_config = api_v3.config_manager.load_config() - current_secrets = api_v3.config_manager.get_raw_file_content('secrets') - - # Deep merge plugin configuration in main config (preserves nested structures) - if plugin_id not in current_config: - current_config[plugin_id] = {} - - current_config[plugin_id] = deep_merge(current_config[plugin_id], regular_config) - - # Deep merge plugin secrets in secrets config - if secrets_config: - if plugin_id not in current_secrets: - current_secrets[plugin_id] = {} - # See above -- secrets lists must merge element-wise. - current_secrets[plugin_id] = merge_secrets( - current_secrets[plugin_id], secrets_config) - # Save secrets file - try: - api_v3.config_manager.save_raw_file_content('secrets', current_secrets) - except PermissionError as e: - # Log the error with more details - import os - secrets_path = api_v3.config_manager.secrets_path - secrets_dir = os.path.dirname(secrets_path) if secrets_path else None - - # Check permissions - dir_readable = os.access(secrets_dir, os.R_OK) if secrets_dir and os.path.exists(secrets_dir) else False - dir_writable = os.access(secrets_dir, os.W_OK) if secrets_dir and os.path.exists(secrets_dir) else False - file_writable = os.access(secrets_path, os.W_OK) if secrets_path and os.path.exists(secrets_path) else False - - logger.error( - f"Permission error saving secrets config for {plugin_id}: {e}\n" - f"Secrets path: {secrets_path}\n" - f"Directory readable: {dir_readable}, writable: {dir_writable}\n" - f"File writable: {file_writable}", - exc_info=True - ) - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - f"Failed to save secrets configuration: Permission denied. Check file permissions on {secrets_path}", - status_code=500 - ) - except Exception as e: - # Log the error but don't fail the entire config save - import os - secrets_path = api_v3.config_manager.secrets_path - logger.error("Error saving secrets config for %s (path=%s)", plugin_id, secrets_path, exc_info=True) - # Return error response with more context - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - "Failed to save secrets configuration; see logs for details", - status_code=500 - ) - - # Save the updated main config using atomic save - success, error_msg = _save_config_atomic(api_v3.config_manager, current_config, create_backup=True) - if not success: - return error_response( - ErrorCode.CONFIG_SAVE_FAILED, - f"Failed to save configuration: {error_msg}", - status_code=500 - ) - - # If the plugin is loaded, notify it of the config change with merged config - try: - if api_v3.plugin_manager: - plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) - if plugin_instance: - # Reload merged config (includes secrets) and pass the plugin-specific section - merged_config = api_v3.config_manager.load_config() - plugin_full_config = merged_config.get(plugin_id, {}) - if hasattr(plugin_instance, 'on_config_change'): - plugin_instance.on_config_change(plugin_full_config) - - # Update plugin state manager and call lifecycle methods based on enabled state - # This ensures the plugin state is synchronized with the config - enabled = plugin_full_config.get('enabled', plugin_instance.enabled) - - # Update state manager if available - if api_v3.plugin_state_manager: - api_v3.plugin_state_manager.set_plugin_enabled(plugin_id, enabled) - - # Call lifecycle methods to ensure plugin state matches config - try: - if enabled: - if hasattr(plugin_instance, 'on_enable'): - plugin_instance.on_enable() - else: - if hasattr(plugin_instance, 'on_disable'): - plugin_instance.on_disable() - except Exception as lifecycle_error: - # Log the error but don't fail the save - config is already saved - import logging - logging.warning(f"Lifecycle method error for {plugin_id}: {lifecycle_error}", exc_info=True) - except Exception as hook_err: - # Do not fail the save if hook fails; just log - logger.warning("on_config_change failed: %s", hook_err) - - secret_count = len(secrets_config) - message = f'Plugin {plugin_id} configuration saved successfully' - if secret_count > 0: - message += f' ({secret_count} secret field(s) saved to config_secrets.json)' - - return success_response(message=message) - except Exception as e: - from src.web_interface.errors import WebInterfaceError - error = WebInterfaceError.from_exception(e, ErrorCode.CONFIG_SAVE_FAILED) - if api_v3.operation_history: - api_v3.operation_history.record_operation( - "configure", - plugin_id=data.get('plugin_id') if 'data' in locals() else None, - status="failed", - error=str(e) - ) - return error_response( - error.error_code, - error.message, - details=error.details, - context=error.context, - status_code=500 - ) - -@api_v3.route('/plugins/schema', methods=['GET']) -def get_plugin_schema(): - """Get plugin configuration schema""" - try: - plugin_id = request.args.get('plugin_id') - if not plugin_id: - return jsonify({'status': 'error', 'message': 'plugin_id required'}), 400 - - # Get schema manager instance - schema_mgr = api_v3.schema_manager - if not schema_mgr: - return jsonify({'status': 'error', 'message': 'Schema manager not initialized'}), 500 - - # Load schema using SchemaManager (uses caching) - schema = schema_mgr.load_schema(plugin_id, use_cache=True) - - if schema: - # Offer installed visual skins as a dropdown (returns a copy; - # the cached schema and validation are never enum-restricted) - try: - current_skin = None - if api_v3.config_manager: - config = api_v3.config_manager.load_config() - current_skin = config.get(plugin_id, {}).get('skin') - injected = schema_mgr.inject_skin_selector(schema, plugin_id, current_skin) - if isinstance(injected, dict): - schema = injected - except Exception: - logger.debug('Skin selector injection failed for %s', plugin_id, exc_info=True) - return jsonify({'status': 'success', 'data': {'schema': schema}}) - - # Return a simple default schema if file not found - default_schema = { - 'type': 'object', - 'properties': { - 'enabled': { - 'type': 'boolean', - 'title': 'Enable Plugin', - 'description': 'Enable or disable this plugin', - 'default': True - }, - 'display_duration': { - 'type': 'integer', - 'title': 'Display Duration', - 'description': 'How long to show content (seconds)', - 'minimum': 5, - 'maximum': 300, - 'default': 30 - } - } - } - - return jsonify({'status': 'success', 'data': {'schema': default_schema}}) - except Exception as e: - logger.error('Error in get_plugin_schema', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/skins', methods=['GET']) -def list_skins(): - """List installed visual skins (docs/SKIN_SYSTEM.md). - - Optional ?plugin_id=... filters to skins matching that plugin. - """ - try: - from src.skin_system import skin_runtime - - plugin_id = request.args.get('plugin_id') - if plugin_id: - skins = skin_runtime.skins_for_plugin(plugin_id) - else: - # The discovery cache self-invalidates on directory/manifest - # mtime changes, so no force_refresh — keeps Pi disk I/O down. - skins = skin_runtime.discover_skins() - - payload = [] - for skin_id, manifest in sorted(skins.items()): - skin_dir = Path(manifest['_skin_dir']) - preview = manifest.get('preview') - payload.append({ - 'id': skin_id, - 'name': manifest.get('name', skin_id), - 'version': manifest.get('version'), - 'author': manifest.get('author'), - 'description': manifest.get('description', ''), - 'skin_api_version': manifest.get('skin_api_version'), - 'targets': manifest.get('targets', {}), - 'modes': manifest.get('modes', []), - 'has_preview': bool(preview and (skin_dir / preview).is_file()), - }) - return jsonify({'status': 'success', 'data': {'skins': payload}}) - except Exception as e: - logger.error('Error in list_skins', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/config/reset', methods=['POST']) -def reset_plugin_config(): - """Reset plugin configuration to schema defaults""" - try: - if not api_v3.config_manager: - return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 - - data = request.get_json(silent=True) or {} - plugin_id = data.get('plugin_id') - preserve_secrets = data.get('preserve_secrets', True) - - if not plugin_id: - return jsonify({'status': 'error', 'message': 'plugin_id required'}), 400 - - # Get schema manager instance - schema_mgr = api_v3.schema_manager - if not schema_mgr: - return jsonify({'status': 'error', 'message': 'Schema manager not initialized'}), 500 - - # Generate defaults from schema - defaults = schema_mgr.generate_default_config(plugin_id, use_cache=True) - - # Get current configs - current_config = api_v3.config_manager.load_config() - current_secrets = api_v3.config_manager.get_raw_file_content('secrets') - - # Load schema to identify secret fields - schema = schema_mgr.load_schema(plugin_id, use_cache=True) - secret_fields = set() - - if schema and 'properties' in schema: - secret_fields = find_secret_fields(schema['properties']) - - # Separate defaults into regular and secret configs - default_regular, default_secrets = separate_secrets(defaults, secret_fields) - - # Update main config with defaults - current_config[plugin_id] = default_regular - - # Update secrets config (preserve existing secrets if preserve_secrets=True) - if preserve_secrets: - # Keep existing secrets for this plugin - if plugin_id in current_secrets: - # Merge defaults with existing secrets - existing_secrets = current_secrets[plugin_id] - for key, value in default_secrets.items(): - if key not in existing_secrets or not existing_secrets[key]: - existing_secrets[key] = value - else: - current_secrets[plugin_id] = default_secrets - else: - # Replace all secrets with defaults - current_secrets[plugin_id] = default_secrets - - # Save updated configs - api_v3.config_manager.save_config(current_config) - if default_secrets or not preserve_secrets: - api_v3.config_manager.save_raw_file_content('secrets', current_secrets) - - # Notify plugin of config change if loaded - try: - if api_v3.plugin_manager: - plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) - if plugin_instance: - merged_config = api_v3.config_manager.load_config() - plugin_full_config = merged_config.get(plugin_id, {}) - if hasattr(plugin_instance, 'on_config_change'): - plugin_instance.on_config_change(plugin_full_config) - except Exception as hook_err: - logger.warning("on_config_change failed: %s", hook_err) - - return jsonify({ - 'status': 'success', - 'message': f'Plugin {plugin_id} configuration reset to defaults', - 'data': {'config': defaults} - }) - except Exception as e: - logger.error('Error in reset_plugin_config', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/action', methods=['POST']) -def execute_plugin_action(): - """Execute a plugin-defined action (e.g., authentication)""" - try: - # Try to get JSON data, with better error handling - try: - data = request.get_json(force=True) or {} - except Exception as e: - import logging - logger = logging.getLogger(__name__) - logger.error(f"Error parsing JSON in execute_plugin_action: {e}") - return jsonify({ - 'status': 'error', - 'message': 'Invalid JSON in request body', - 'content_type': request.content_type }), 400 - - plugin_id = data.get('plugin_id') - action_id = data.get('action_id') - action_params = data.get('params', {}) - - if not plugin_id or not action_id: - return jsonify({ - 'status': 'error', - 'message': 'plugin_id and action_id required', - 'received': {'plugin_id': plugin_id, 'action_id': action_id, 'has_params': bool(action_params)} - }), 400 - - # Get plugin directory - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): - return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - - # Load manifest to get action definition - manifest_path = Path(plugin_dir) / 'manifest.json' - if not manifest_path.exists(): - return jsonify({'status': 'error', 'message': 'Plugin manifest not found'}), 404 - - with open(manifest_path, 'r', encoding='utf-8') as f: - manifest = json.load(f) - - web_ui_actions = manifest.get('web_ui_actions', []) - action_def = None - for action in web_ui_actions: - if action.get('id') == action_id: - action_def = action - break - - if not action_def: - return jsonify({'status': 'error', 'message': f'Action {action_id} not found in plugin manifest'}), 404 - - # Set LEDMATRIX_ROOT environment variable - env = os.environ.copy() - env['LEDMATRIX_ROOT'] = str(PROJECT_ROOT) - - # Execute action based on type - action_type = action_def.get('type', 'script') - - if action_type == 'script': - # Execute a Python script - script_path = action_def.get('script') - if not script_path: - return jsonify({'status': 'error', 'message': 'Script path not defined for action'}), 400 - - script_file = Path(plugin_dir) / script_path - if not script_file.exists(): - return jsonify({'status': 'error', 'message': f'Script not found: {script_path}'}), 404 - - # Handle multi-step actions (like Spotify OAuth) - step = action_params.get('step') - - if step == '2' and action_params.get('redirect_url'): - # Step 2: Complete authentication with redirect URL - redirect_url = action_params.get('redirect_url') - import tempfile - import json as json_lib - - redirect_url_escaped = json_lib.dumps(redirect_url) - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as wrapper: - wrapper.write(f'''import sys -import subprocess -import os - -# Set LEDMATRIX_ROOT -os.environ['LEDMATRIX_ROOT'] = r"{PROJECT_ROOT}" - -# Run the script and provide redirect URL -proc = subprocess.Popen( - [sys.executable, r"{script_file}"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - env=os.environ -) - -# Send redirect URL to stdin -redirect_url = {redirect_url_escaped} -stdout, _ = proc.communicate(input=redirect_url + "\\n", timeout=120) -print(stdout) -sys.exit(proc.returncode) -''') - wrapper_path = wrapper.name - - try: - result = subprocess.run( - ['python3', wrapper_path], - capture_output=True, - text=True, - timeout=120, - env=env - ) - os.unlink(wrapper_path) - - if result.returncode == 0: - return jsonify({ - 'status': 'success', - 'message': action_def.get('success_message', 'Action completed successfully'), - 'output': result.stdout - }) - else: - return jsonify({ - 'status': 'error', - 'message': action_def.get('error_message', 'Action failed'), - 'output': result.stdout + result.stderr - }), 400 - except subprocess.TimeoutExpired: - if os.path.exists(wrapper_path): - os.unlink(wrapper_path) - return jsonify({'status': 'error', 'message': 'Action timed out'}), 408 - else: - # Regular script execution - pass params via stdin if provided - if action_params: - # Pass params as JSON via stdin - import tempfile - import json as json_lib - - params_json = json_lib.dumps(action_params) - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as wrapper: - wrapper.write(f'''import sys -import subprocess -import os -import json - -# Set LEDMATRIX_ROOT -os.environ['LEDMATRIX_ROOT'] = r"{PROJECT_ROOT}" - -# Run the script and provide params as JSON via stdin -proc = subprocess.Popen( - [sys.executable, r"{script_file}"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - env=os.environ -) - -# Send params as JSON to stdin -params = {params_json} -stdout, _ = proc.communicate(input=json.dumps(params), timeout=120) -print(stdout) -sys.exit(proc.returncode) -''') - wrapper_path = wrapper.name - - try: - result = subprocess.run( - ['python3', wrapper_path], - capture_output=True, - text=True, - timeout=120, - env=env - ) - os.unlink(wrapper_path) - - # Try to parse output as JSON - try: - output_data = json.loads(result.stdout) - if result.returncode == 0: - return jsonify(output_data) - else: - return jsonify({ - 'status': 'error', - 'message': output_data.get('message', action_def.get('error_message', 'Action failed')), - 'output': result.stdout + result.stderr - }), 400 - except json.JSONDecodeError: - # Output is not JSON, return as text - if result.returncode == 0: - return jsonify({ - 'status': 'success', - 'message': action_def.get('success_message', 'Action completed successfully'), - 'output': result.stdout - }) - else: - return jsonify({ - 'status': 'error', - 'message': action_def.get('error_message', 'Action failed'), - 'output': result.stdout + result.stderr - }), 400 - except subprocess.TimeoutExpired: - if os.path.exists(wrapper_path): - os.unlink(wrapper_path) - return jsonify({'status': 'error', 'message': 'Action timed out'}), 408 - else: - # No params - check for OAuth flow first, then run script normally - # Step 1: Get initial data (like auth URL) - # For OAuth flows, we might need to import the script as a module - if action_def.get('oauth_flow'): - # Import script as module to get auth URL - import sys - import importlib.util - - spec = importlib.util.spec_from_file_location("plugin_action", script_file) - action_module = importlib.util.module_from_spec(spec) - sys.modules["plugin_action"] = action_module - - try: - spec.loader.exec_module(action_module) - - # Try to get auth URL using common patterns - auth_url = None - if hasattr(action_module, 'get_auth_url'): - auth_url = action_module.get_auth_url() - elif hasattr(action_module, 'load_spotify_credentials'): - # Spotify-specific pattern - client_id, client_secret, redirect_uri = action_module.load_spotify_credentials() - if all([client_id, client_secret, redirect_uri]): - from spotipy.oauth2 import SpotifyOAuth - sp_oauth = SpotifyOAuth( - client_id=client_id, - client_secret=client_secret, - redirect_uri=redirect_uri, - scope=getattr(action_module, 'SCOPE', ''), - cache_path=getattr(action_module, 'SPOTIFY_AUTH_CACHE_PATH', None), - open_browser=False - ) - auth_url = sp_oauth.get_authorize_url() - - if auth_url: - return jsonify({ - 'status': 'success', - 'message': action_def.get('step1_message', 'Authorization URL generated'), - 'auth_url': auth_url, - 'requires_step2': True - }) - else: - return jsonify({ - 'status': 'error', - 'message': 'Could not generate authorization URL' - }), 400 - except Exception as e: - logger.error("Error executing action step 1", exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) - }), 500 - else: - # Simple script execution - result = subprocess.run( - ['python3', str(script_file)], - capture_output=True, - text=True, - timeout=60, - env=env - ) - - # Try to parse output as JSON - try: - import json as json_module - output_data = json_module.loads(result.stdout) - if result.returncode == 0: - return jsonify(output_data) - else: - return jsonify({ - 'status': 'error', - 'message': output_data.get('message', action_def.get('error_message', 'Action failed')), - 'output': result.stdout + result.stderr - }), 400 - except json.JSONDecodeError: - # Output is not JSON, return as text - if result.returncode == 0: - return jsonify({ - 'status': 'success', - 'message': action_def.get('success_message', 'Action completed successfully'), - 'output': result.stdout - }) - else: - return jsonify({ - 'status': 'error', - 'message': action_def.get('error_message', 'Action failed'), - 'output': result.stdout + result.stderr - }), 400 - - elif action_type == 'endpoint': - # Call a plugin-defined HTTP endpoint (future feature) - return jsonify({'status': 'error', 'message': 'Endpoint actions not yet implemented'}), 501 - - else: - return jsonify({'status': 'error', 'message': f'Unknown action type: {action_type}'}), 400 - - except subprocess.TimeoutExpired: - return jsonify({'status': 'error', 'message': 'Action timed out'}), 408 - except Exception as e: - logger.error('Error in execute_plugin_action', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/authenticate/spotify', methods=['POST']) -def authenticate_spotify(): - """Run Spotify authentication script""" - try: - data = request.get_json(silent=True) or {} - redirect_url = data.get('redirect_url', '').strip() - - # Get plugin directory - plugin_id = 'ledmatrix-music' - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): - return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - - auth_script = Path(plugin_dir) / 'authenticate_spotify.py' - if not auth_script.exists(): - return jsonify({'status': 'error', 'message': 'Authentication script not found'}), 404 - - # Set LEDMATRIX_ROOT environment variable - env = os.environ.copy() - env['LEDMATRIX_ROOT'] = str(PROJECT_ROOT) - - if redirect_url: - # Step 2: Complete authentication with redirect URL - # Create a wrapper script that provides the redirect URL as input - import tempfile - - # Create a wrapper script that provides the redirect URL - import json - redirect_url_escaped = json.dumps(redirect_url) # Properly escape the URL - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as wrapper: - wrapper.write(f'''import sys -import subprocess -import os - -# Set LEDMATRIX_ROOT -os.environ['LEDMATRIX_ROOT'] = r"{PROJECT_ROOT}" - -# Run the auth script and provide redirect URL -proc = subprocess.Popen( - [sys.executable, r"{auth_script}"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - env=os.environ -) - -# Send redirect URL to stdin -redirect_url = {redirect_url_escaped} -stdout, _ = proc.communicate(input=redirect_url + "\\n", timeout=120) -print(stdout) -sys.exit(proc.returncode) -''') - wrapper_path = wrapper.name - - try: - result = subprocess.run( - ['python3', wrapper_path], - capture_output=True, - text=True, - timeout=120, - env=env - ) - - if result.returncode == 0: - return jsonify({ - 'status': 'success', - 'message': 'Spotify authentication completed successfully', - 'output': result.stdout - }) - else: - return jsonify({ - 'status': 'error', - 'message': 'Spotify authentication failed', - 'output': result.stdout + result.stderr - }), 400 - except subprocess.TimeoutExpired: - return jsonify({'status': 'error', 'message': 'Authentication timed out'}), 408 - finally: - # The wrapper carries the user's redirect URL, so it must not - # survive the request on any path — including a failure to - # launch, which the previous per-branch unlinks missed. - if os.path.exists(wrapper_path): - os.unlink(wrapper_path) - else: - # Step 1: Get authorization URL - # Import the script's functions directly to get the auth URL - import sys - import importlib.util - - # Load the authentication script as a module - spec = importlib.util.spec_from_file_location("auth_spotify", auth_script) - auth_module = importlib.util.module_from_spec(spec) - sys.modules["auth_spotify"] = auth_module - - # Set LEDMATRIX_ROOT before loading - os.environ['LEDMATRIX_ROOT'] = str(PROJECT_ROOT) - - try: - spec.loader.exec_module(auth_module) - - # Get credentials and create OAuth object - client_id, client_secret, redirect_uri = auth_module.load_spotify_credentials() - if not all([client_id, client_secret, redirect_uri]): - return jsonify({ - 'status': 'error', - 'message': 'Could not load Spotify credentials. Please check config/config_secrets.json.' - }), 400 - - from spotipy.oauth2 import SpotifyOAuth - sp_oauth = SpotifyOAuth( - client_id=client_id, - client_secret=client_secret, - redirect_uri=redirect_uri, - scope=auth_module.SCOPE, - cache_path=auth_module.SPOTIFY_AUTH_CACHE_PATH, - open_browser=False - ) - - auth_url = sp_oauth.get_authorize_url() - - return jsonify({ - 'status': 'success', - 'message': 'Authorization URL generated', - 'auth_url': auth_url - }) - except Exception as e: - logger.error("Error getting Spotify auth URL", exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) - }), 500 - - except Exception as e: - logger.error('Error in authenticate_spotify', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/authenticate/ytm', methods=['POST']) -def authenticate_ytm(): - """Run YouTube Music authentication script""" - try: - # Get plugin directory - plugin_id = 'ledmatrix-music' - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): - return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - - auth_script = Path(plugin_dir) / 'authenticate_ytm.py' - if not auth_script.exists(): - return jsonify({'status': 'error', 'message': 'Authentication script not found'}), 404 - - # Set LEDMATRIX_ROOT environment variable - env = os.environ.copy() - env['LEDMATRIX_ROOT'] = str(PROJECT_ROOT) - - # Run the authentication script - result = subprocess.run( - ['python3', str(auth_script)], - capture_output=True, - text=True, - timeout=60, - env=env - ) - - if result.returncode == 0: - return jsonify({ - 'status': 'success', - 'message': 'YouTube Music authentication completed successfully', - 'output': result.stdout - }) - else: - return jsonify({ - 'status': 'error', - 'message': 'YouTube Music authentication failed', - 'output': result.stdout + result.stderr - }), 400 - - except subprocess.TimeoutExpired: - return jsonify({'status': 'error', 'message': 'Authentication timed out'}), 408 - except Exception as e: - logger.error('Error in authenticate_ytm', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/fonts/catalog', methods=['GET']) -def get_fonts_catalog(): - """Get fonts catalog""" - try: - # Check cache first (5 minute TTL) - try: - from web_interface.cache import get_cached, set_cached - cached_result = get_cached('fonts_catalog', ttl_seconds=300) - if cached_result is not None: - return jsonify({'status': 'success', 'data': {'catalog': cached_result}}) - except ImportError: - # Cache not available, continue without caching - get_cached = None - set_cached = None - - # Try to import freetype, but continue without it if unavailable - try: - import freetype - freetype_available = True - except ImportError: - freetype_available = False - - # Scan assets/fonts directory for actual font files - fonts_dir = PROJECT_ROOT / "assets" / "fonts" - catalog = {} - - if fonts_dir.exists() and fonts_dir.is_dir(): - for filename in os.listdir(fonts_dir): - if filename.endswith(('.ttf', '.otf', '.bdf')): - filepath = fonts_dir / filename - # Generate family name from filename (without extension) - family_name = os.path.splitext(filename)[0] - - # Try to get font metadata using freetype (for TTF/OTF) - metadata = {} - if filename.endswith(('.ttf', '.otf')) and freetype_available: - try: - face = freetype.Face(str(filepath)) - if face.valid: - # Get font family name from font file - family_name_from_font = face.family_name.decode('utf-8') if face.family_name else family_name - metadata = { - 'family': family_name_from_font, - 'style': face.style_name.decode('utf-8') if face.style_name else 'Regular', - 'num_glyphs': face.num_glyphs, - 'units_per_em': face.units_per_EM - } - # Use font's family name if available - if family_name_from_font: - family_name = family_name_from_font - except Exception: - # If freetype fails, use filename-based name - pass - - # Store relative path from project root - relative_path = str(filepath.relative_to(PROJECT_ROOT)) - font_type = 'ttf' if filename.endswith('.ttf') else 'otf' if filename.endswith('.otf') else 'bdf' - - # Generate human-readable display name from family_name - display_name = family_name.replace('-', ' ').replace('_', ' ') - # Add space before capital letters for camelCase names - display_name = re.sub(r'([a-z])([A-Z])', r'\1 \2', display_name) - # Add space before numbers that follow letters - display_name = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', display_name) - # Clean up multiple spaces - display_name = ' '.join(display_name.split()) - - # Use filename (without extension) as unique key to avoid collisions - # when multiple files share the same family_name from font metadata - catalog_key = os.path.splitext(filename)[0] - - # Check if this is a system font (cannot be deleted) - is_system = catalog_key.lower() in SYSTEM_FONTS - - catalog[catalog_key] = { - 'filename': filename, - 'family_name': family_name, - 'display_name': display_name, - 'path': relative_path, - 'type': font_type, - 'is_system': is_system, - 'metadata': metadata if metadata else None - } - - # Cache the result (5 minute TTL) if available - if set_cached: - try: - set_cached('fonts_catalog', catalog, ttl_seconds=300) - except Exception: - logger.error("[FontCatalog] Failed to cache fonts_catalog", exc_info=True) - - return jsonify({'status': 'success', 'data': {'catalog': catalog}}) - except Exception as e: - logger.error("%s failed", request.path, exc_info=True) - return jsonify({'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e)}), 500 - -@api_v3.route('/fonts/tokens', methods=['GET']) -def get_font_tokens(): - """Get font size tokens""" - try: - # This would integrate with the actual font system - # For now, return sample tokens - tokens = { - 'xs': 6, - 'sm': 8, - 'md': 10, - 'lg': 12, - 'xl': 14, - 'xxl': 16 - } - return jsonify({'status': 'success', 'data': {'tokens': tokens}}) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/fonts/overrides', methods=['GET']) -def get_fonts_overrides(): - """Get font overrides""" - try: - # This would integrate with the actual font system - # For now, return empty overrides - overrides = {} - return jsonify({'status': 'success', 'data': {'overrides': overrides}}) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/fonts/overrides', methods=['POST']) -def save_fonts_overrides(): - """Save font overrides""" - try: - data = request.get_json(silent=True) - if not data: - return jsonify({'status': 'error', 'message': 'No data provided'}), 400 - - # This would integrate with the actual font system - return jsonify({'status': 'success', 'message': 'Font overrides saved'}) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/fonts/overrides/', methods=['DELETE']) -def delete_font_override(element_key): - """Delete font override""" - try: - # This would integrate with the actual font system - return jsonify({'status': 'success', 'message': f'Font override for {element_key} deleted'}) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/fonts/upload', methods=['POST']) -def upload_font(): - """Upload font file""" - try: - if 'font_file' not in request.files: - return jsonify({'status': 'error', 'message': 'No font file provided'}), 400 - - font_file = request.files['font_file'] - if font_file.filename == '': - return jsonify({'status': 'error', 'message': 'No file selected'}), 400 - - # Validate filename - is_valid, error_msg = validate_file_upload( - font_file.filename, - max_size_mb=10, - allowed_extensions=['.ttf', '.otf', '.bdf'] - ) - if not is_valid: - return jsonify({'status': 'error', 'message': error_msg}), 400 - - font_family = request.form.get('font_family', '') - - if not font_family: - return jsonify({'status': 'error', 'message': 'Font file and family name required'}), 400 - - # Validate font family name - if not font_family.replace('_', '').replace('-', '').isalnum(): - return jsonify({'status': 'error', 'message': 'Font family name must contain only letters, numbers, underscores, and hyphens'}), 400 - - # Save the font file to assets/fonts directory - fonts_dir = PROJECT_ROOT / "assets" / "fonts" - fonts_dir.mkdir(parents=True, exist_ok=True) - - # Create filename from family name - original_ext = os.path.splitext(font_file.filename)[1].lower() - safe_filename = f"{font_family}{original_ext}" - filepath = fonts_dir / safe_filename - - # Check if file already exists - if filepath.exists(): - return jsonify({'status': 'error', 'message': f'Font with name {font_family} already exists'}), 400 - - # Save the file - font_file.save(str(filepath)) - - # Clear font catalog cache - try: - from web_interface.cache import delete_cached - delete_cached('fonts_catalog') - except ImportError as e: - logger.warning("[FontUpload] Cache module not available: %s", e) - except Exception: - logger.error("[FontUpload] Failed to clear fonts_catalog cache", exc_info=True) - - return jsonify({ - 'status': 'success', - 'message': f'Font {font_family} uploaded successfully', - 'font_family': font_family, - 'filename': safe_filename, - 'path': f'assets/fonts/{safe_filename}' - }) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - - -@api_v3.route('/fonts/preview', methods=['GET']) -def get_font_preview() -> tuple[Response, int] | Response: - """Generate a preview image of text rendered with a specific font""" - try: - from PIL import Image, ImageDraw, ImageFont - import io - import base64 - - # Limits to prevent DoS via large image generation on constrained devices - MAX_TEXT_CHARS = 100 - MAX_TEXT_LINES = 3 - MAX_DIM = 1024 # Max width or height in pixels - MAX_PIXELS = 500000 # Max total pixels (e.g., ~700x700) - - font_filename = request.args.get('font', '') - text = request.args.get('text', 'Sample Text 123') - bg_color = request.args.get('bg', '000000') - fg_color = request.args.get('fg', 'ffffff') - - # Validate text length and line count early - if len(text) > MAX_TEXT_CHARS: - return jsonify({'status': 'error', 'message': f'Text exceeds maximum length of {MAX_TEXT_CHARS} characters'}), 400 - if text.count('\n') >= MAX_TEXT_LINES: - return jsonify({'status': 'error', 'message': f'Text exceeds maximum of {MAX_TEXT_LINES} lines'}), 400 - - # Safe integer parsing for size - try: - size = int(request.args.get('size', 12)) - except (ValueError, TypeError, OverflowError): - return jsonify({'status': 'error', 'message': 'Invalid font size'}), 400 - - if not font_filename: - return jsonify({'status': 'error', 'message': 'Font filename required'}), 400 - - # Validate size - if size < 4 or size > 72: - return jsonify({'status': 'error', 'message': 'Font size must be between 4 and 72'}), 400 - - # Security: Validate font_filename to prevent path traversal - # Only allow alphanumeric, hyphen, underscore, and dot (for extension) - safe_name = Path(font_filename).name # Strip any directory components - if safe_name != font_filename or '..' in font_filename: - return jsonify({'status': 'error', 'message': 'Invalid font filename'}), 400 - - # Validate extension - allowed_extensions = ['.ttf', '.otf', '.bdf'] - has_valid_ext = any(safe_name.lower().endswith(ext) for ext in allowed_extensions) - name_without_ext = safe_name.rsplit('.', 1)[0] if '.' in safe_name else safe_name - - # Find the font file - fonts_dir = PROJECT_ROOT / "assets" / "fonts" - if not fonts_dir.exists(): - return jsonify({'status': 'error', 'message': 'Fonts directory not found'}), 404 - - font_path = fonts_dir / safe_name - - if not font_path.exists() and not has_valid_ext: - # Try finding by family name (without extension) - for ext in allowed_extensions: - potential_path = fonts_dir / f"{name_without_ext}{ext}" - if potential_path.exists(): - font_path = potential_path - break - - # Final security check: ensure path is within fonts_dir - try: - font_path.resolve().relative_to(fonts_dir.resolve()) - except ValueError: - return jsonify({'status': 'error', 'message': 'Invalid font path'}), 400 - - if not font_path.exists(): - return jsonify({'status': 'error', 'message': f'Font file not found: {font_filename}'}), 404 - - # Parse colors - try: - bg_rgb = tuple(int(bg_color[i:i+2], 16) for i in (0, 2, 4)) - fg_rgb = tuple(int(fg_color[i:i+2], 16) for i in (0, 2, 4)) - except (ValueError, IndexError): - bg_rgb = (0, 0, 0) - fg_rgb = (255, 255, 255) - - # Load font - font = None - if str(font_path).endswith('.bdf'): - # BDF fonts require complex per-glyph rendering via freetype - # Return explicit error rather than showing misleading preview with default font - return jsonify({ - 'status': 'error', - 'message': 'BDF font preview not supported. BDF fonts will render correctly on the LED matrix.' - }), 400 - else: - # TTF/OTF fonts - try: - font = ImageFont.truetype(str(font_path), size) - except (IOError, OSError) as e: - # IOError/OSError raised for invalid/corrupt font files - logger.warning("[FontPreview] Failed to load font %s: %s", font_path, e) - font = ImageFont.load_default() - - # Calculate text size - temp_img = Image.new('RGB', (1, 1)) - temp_draw = ImageDraw.Draw(temp_img) - bbox = temp_draw.textbbox((0, 0), text, font=font) - text_width = bbox[2] - bbox[0] - text_height = bbox[3] - bbox[1] - - # Create image with padding - padding = 10 - img_width = max(text_width + padding * 2, 100) - img_height = max(text_height + padding * 2, 30) - - # Validate resulting image size to prevent memory/CPU spikes - if img_width > MAX_DIM or img_height > MAX_DIM: - return jsonify({'status': 'error', 'message': 'Requested image too large'}), 400 - if img_width * img_height > MAX_PIXELS: - return jsonify({'status': 'error', 'message': 'Requested image too large'}), 400 - - img = Image.new('RGB', (img_width, img_height), bg_rgb) - draw = ImageDraw.Draw(img) - - # Center text - x = (img_width - text_width) // 2 - y = (img_height - text_height) // 2 - - draw.text((x, y), text, font=font, fill=fg_rgb) - - # Convert to base64 - buffer = io.BytesIO() - img.save(buffer, format='PNG') - buffer.seek(0) - img_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8') - - return jsonify({ - 'status': 'success', - 'data': { - 'image': f'data:image/png;base64,{img_base64}', - 'width': img_width, - 'height': img_height - } - }) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - - -@api_v3.route('/fonts/', methods=['DELETE']) -def delete_font(font_family: str) -> tuple[Response, int] | Response: - """Delete a user-uploaded font file""" - try: - # Security: Validate font_family to prevent path traversal - # Reject if it contains path separators or .. - if '..' in font_family or '/' in font_family or '\\' in font_family: - return jsonify({'status': 'error', 'message': 'Invalid font family name'}), 400 - - # Only allow safe characters: alphanumeric, hyphen, underscore, dot - if not re.match(r'^[a-zA-Z0-9_\-\.]+$', font_family): - return jsonify({'status': 'error', 'message': 'Invalid font family name'}), 400 - - # Check if this is a system font (uses module-level SYSTEM_FONTS frozenset) - if font_family.lower() in SYSTEM_FONTS: - return jsonify({'status': 'error', 'message': 'Cannot delete system fonts'}), 403 - - # Find and delete the font file - fonts_dir = PROJECT_ROOT / "assets" / "fonts" - - # Ensure fonts directory exists - if not fonts_dir.exists() or not fonts_dir.is_dir(): - return jsonify({'status': 'error', 'message': 'Fonts directory not found'}), 404 - - deleted = False - deleted_filename = None - - # Only try valid font extensions (no empty string to avoid matching directories) - for ext in ['.ttf', '.otf', '.bdf']: - potential_path = fonts_dir / f"{font_family}{ext}" - - # Security: Verify path is within fonts_dir - try: - potential_path.resolve().relative_to(fonts_dir.resolve()) - except ValueError: - continue # Path escapes fonts_dir, skip - - if potential_path.exists() and potential_path.is_file(): - potential_path.unlink() - deleted = True - deleted_filename = f"{font_family}{ext}" - break - - if not deleted: - # Try case-insensitive match within fonts directory - font_family_lower = font_family.lower() - for filename in os.listdir(fonts_dir): - # Only consider files with valid font extensions - if not any(filename.lower().endswith(ext) for ext in ['.ttf', '.otf', '.bdf']): - continue - - name_without_ext = os.path.splitext(filename)[0] - if name_without_ext.lower() == font_family_lower: - filepath = fonts_dir / filename - - # Security: Verify path is within fonts_dir - try: - filepath.resolve().relative_to(fonts_dir.resolve()) - except ValueError: - continue # Path escapes fonts_dir, skip - - if filepath.is_file(): - filepath.unlink() - deleted = True - deleted_filename = filename - break - - if not deleted: - return jsonify({'status': 'error', 'message': f'Font not found: {font_family}'}), 404 - - # Clear font catalog cache - try: - from web_interface.cache import delete_cached - delete_cached('fonts_catalog') - except ImportError as e: - logger.warning("[FontDelete] Cache module not available: %s", e) - except Exception: - logger.error("[FontDelete] Failed to clear fonts_catalog cache", exc_info=True) - - return jsonify({ - 'status': 'success', - 'message': f'Font {deleted_filename} deleted successfully' - }) - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - - -@api_v3.route('/plugins/assets/upload', methods=['POST']) -def upload_plugin_asset(): - """Upload asset files for a plugin""" - try: - plugin_id = request.form.get('plugin_id') - if not plugin_id: - return jsonify({'status': 'error', 'message': 'plugin_id is required'}), 400 - - if 'files' not in request.files: - return jsonify({'status': 'error', 'message': 'No files provided'}), 400 - - files = request.files.getlist('files') - if not files or all(not f.filename for f in files): - return jsonify({'status': 'error', 'message': 'No files provided'}), 400 - - # Validate file count - if len(files) > 10: - return jsonify({'status': 'error', 'message': 'Maximum 10 files per upload'}), 400 - - # Setup plugin assets directory - assets_dir = PROJECT_ROOT / 'assets' / 'plugins' / plugin_id / 'uploads' - assets_dir.mkdir(parents=True, exist_ok=True) - - # Load metadata file - metadata_file = assets_dir / '.metadata.json' - if metadata_file.exists(): - with open(metadata_file, 'r') as f: - metadata = json.load(f) - else: - metadata = {} - - uploaded_files = [] - total_size = 0 - max_size_per_file = 5 * 1024 * 1024 # 5MB - max_total_size = 50 * 1024 * 1024 # 50MB - - # Calculate current total size - for entry in metadata.values(): - if 'size' in entry: - total_size += entry.get('size', 0) - - for file in files: - if not file.filename: - continue - - # Validate file type - allowed_extensions = ['.png', '.jpg', '.jpeg', '.bmp', '.gif'] - file_ext = '.' + file.filename.lower().split('.')[-1] - if file_ext not in allowed_extensions: - return jsonify({ - 'status': 'error', - 'message': f'Invalid file type: {file_ext}. Allowed: {allowed_extensions}' - }), 400 - - # Read file to check size and validate - file.seek(0, os.SEEK_END) - file_size = file.tell() - file.seek(0) - - if file_size > max_size_per_file: - return jsonify({ - 'status': 'error', - 'message': f'File {file.filename} exceeds 5MB limit' - }), 400 - - if total_size + file_size > max_total_size: - return jsonify({ - 'status': 'error', - 'message': f'Upload would exceed 50MB total storage limit' - }), 400 - - # Validate file is actually an image (check magic bytes) - file_content = file.read(8) - file.seek(0) - is_valid_image = False - if file_content.startswith(b'\x89PNG\r\n\x1a\n'): # PNG - is_valid_image = True - elif file_content[:2] == b'\xff\xd8': # JPEG - is_valid_image = True - elif file_content[:2] == b'BM': # BMP - is_valid_image = True - elif file_content[:6] in [b'GIF87a', b'GIF89a']: # GIF - is_valid_image = True - - if not is_valid_image: - return jsonify({ - 'status': 'error', - 'message': f'File {file.filename} is not a valid image file' - }), 400 - - # Generate unique filename - timestamp = int(time.time()) - file_hash = hashlib.md5(file_content + file.filename.encode()).hexdigest()[:8] - safe_filename = f"image_{timestamp}_{file_hash}{file_ext}" - file_path = assets_dir / safe_filename - - # Ensure filename is unique - counter = 1 - while file_path.exists(): - safe_filename = f"image_{timestamp}_{file_hash}_{counter}{file_ext}" - file_path = assets_dir / safe_filename - counter += 1 - - # Save file - file.save(str(file_path)) - - # Make file readable - os.chmod(file_path, 0o644) - - # Generate unique ID - image_id = str(uuid.uuid4()) - - # Store metadata - relative_path = f"assets/plugins/{plugin_id}/uploads/{safe_filename}" - metadata[image_id] = { - 'id': image_id, - 'filename': safe_filename, - 'path': relative_path, - 'size': file_size, - 'uploaded_at': datetime.utcnow().isoformat() + 'Z', - 'original_filename': file.filename - } - - uploaded_files.append({ - 'id': image_id, - 'filename': safe_filename, - 'path': relative_path, - 'size': file_size, - 'uploaded_at': metadata[image_id]['uploaded_at'] - }) - - total_size += file_size - - # Save metadata - with open(metadata_file, 'w') as f: - json.dump(metadata, f, indent=2) - - return jsonify({ - 'status': 'success', - 'uploaded_files': uploaded_files, - 'total_files': len(metadata) - }) - - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/of-the-day/json/upload', methods=['POST']) -def upload_of_the_day_json(): - """Upload JSON files for of-the-day plugin""" - try: - if 'files' not in request.files: - return jsonify({'status': 'error', 'message': 'No files provided'}), 400 - - files = request.files.getlist('files') - if not files or all(not f.filename for f in files): - return jsonify({'status': 'error', 'message': 'No files provided'}), 400 - - # Get plugin directory - plugin_id = 'ledmatrix-of-the-day' - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): - return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - - # Setup of_the_day directory - data_dir = Path(plugin_dir) / 'of_the_day' - data_dir.mkdir(parents=True, exist_ok=True) - - uploaded_files = [] - max_size_per_file = 5 * 1024 * 1024 # 5MB - - for file in files: - if not file.filename: - continue - - # Validate file extension - if not file.filename.lower().endswith('.json'): - return jsonify({ - 'status': 'error', - 'message': f'File {file.filename} must be a JSON file (.json)' - }), 400 - - # Read and validate file size - file.seek(0, os.SEEK_END) - file_size = file.tell() - file.seek(0) - - if file_size > max_size_per_file: - return jsonify({ - 'status': 'error', - 'message': f'File {file.filename} exceeds 5MB limit' - }), 400 - - # Read and validate JSON content - try: - file_content = file.read().decode('utf-8') - json_data = json.loads(file_content) - except json.JSONDecodeError as e: - return jsonify({ - 'status': 'error', - 'message': 'Invalid JSON in request body' - }), 400 - except UnicodeDecodeError: - return jsonify({ - 'status': 'error', - 'message': f'File {file.filename} is not valid UTF-8 text' - }), 400 - - # Validate JSON structure (must be object with day number keys) - if not isinstance(json_data, dict): - return jsonify({ - 'status': 'error', - 'message': f'JSON in {file.filename} must be an object with day numbers (1-365) as keys' - }), 400 - - # Check if keys are valid day numbers - for key in json_data.keys(): - try: - day_num = int(key) - if day_num < 1 or day_num > 365: - return jsonify({ - 'status': 'error', - 'message': f'Day number {day_num} in {file.filename} is out of range (must be 1-365)' - }), 400 - except ValueError: - return jsonify({ - 'status': 'error', - 'message': f'Invalid key "{key}" in {file.filename}: must be a day number (1-365)' - }), 400 - - # Generate safe filename from original (preserve user's filename) - original_filename = file.filename - safe_filename = original_filename.lower().replace(' ', '_') - # Ensure it's a valid filename - safe_filename = ''.join(c for c in safe_filename if c.isalnum() or c in '._-') - if not safe_filename.endswith('.json'): - safe_filename += '.json' - - file_path = data_dir / safe_filename - - # If file exists, add counter - counter = 1 - base_name = safe_filename.replace('.json', '') - while file_path.exists(): - safe_filename = f"{base_name}_{counter}.json" - file_path = data_dir / safe_filename - counter += 1 - - # Save file - with open(file_path, 'w', encoding='utf-8') as f: - json.dump(json_data, f, indent=2, ensure_ascii=False) - - # Make file readable - os.chmod(file_path, 0o644) - - # Extract category name from filename (remove .json extension) - category_name = safe_filename.replace('.json', '') - display_name = category_name.replace('_', ' ').title() - - # Update plugin config to add category - try: - sys.path.insert(0, str(plugin_dir)) - from scripts.update_config import add_category_to_config - add_category_to_config(category_name, f'of_the_day/{safe_filename}', display_name) - except Exception as e: - logger.warning("Could not update config: %s", e) - # Continue anyway - file is uploaded - - # Generate file ID (use category name as ID for simplicity) - file_id = category_name - - uploaded_files.append({ - 'id': file_id, - 'filename': safe_filename, - 'original_filename': original_filename, - 'path': f'of_the_day/{safe_filename}', - 'size': file_size, - 'uploaded_at': datetime.utcnow().isoformat() + 'Z', - 'category_name': category_name, - 'display_name': display_name, - 'entry_count': len(json_data) - }) - - return jsonify({ - 'status': 'success', - 'uploaded_files': uploaded_files, - 'total_files': len(uploaded_files) - }) - - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/of-the-day/json/delete', methods=['POST']) -def delete_of_the_day_json(): - """Delete a JSON file from of-the-day plugin""" - try: - data = request.get_json(silent=True) or {} - file_id = data.get('file_id') # This is the category_name - - if not file_id: - return jsonify({'status': 'error', 'message': 'file_id is required'}), 400 - - # Get plugin directory - plugin_id = 'ledmatrix-of-the-day' - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): - return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - - data_dir = Path(plugin_dir) / 'of_the_day' - filename = f"{file_id}.json" - file_path = data_dir / filename - - if not file_path.exists(): - return jsonify({'status': 'error', 'message': f'File {filename} not found'}), 404 - - # Delete file - file_path.unlink() - - # Update config to remove category - try: - sys.path.insert(0, str(plugin_dir)) - from scripts.update_config import remove_category_from_config - remove_category_from_config(file_id) - except Exception as e: - logger.warning("Could not update config: %s", e) - - return jsonify({ - 'status': 'success', - 'message': f'File {filename} deleted successfully' - }) - - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins//static/', methods=['GET']) -def serve_plugin_static(plugin_id, file_path): - """Serve static files from plugin directory""" - try: - # Get plugin directory - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): - return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - - # Resolve file path (prevent directory traversal) - plugin_dir = Path(plugin_dir).resolve() - requested_file = (plugin_dir / file_path).resolve() - - # Security check: ensure file is within plugin directory - if not str(requested_file).startswith(str(plugin_dir)): - return jsonify({'status': 'error', 'message': 'Invalid file path'}), 403 - - # Check if file exists - if not requested_file.exists() or not requested_file.is_file(): - return jsonify({'status': 'error', 'message': 'File not found'}), 404 - - # Determine content type - content_type = 'text/plain' - if file_path.endswith('.html'): - content_type = 'text/html' - elif file_path.endswith('.js'): - content_type = 'application/javascript' - elif file_path.endswith('.css'): - content_type = 'text/css' - elif file_path.endswith('.json'): - content_type = 'application/json' - - # Read and return file - with open(requested_file, 'r', encoding='utf-8') as f: - content = f.read() - - return Response(content, mimetype=content_type) - - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - - -_MAX_CREDENTIAL_BACKUPS = 5 - - -def _prune_credential_backups(plugin_dir: Path) -> None: - """Keep only the newest _MAX_CREDENTIAL_BACKUPS credential backups. - - Every re-upload copies the previous credentials.json aside. Without - pruning those accumulate for the life of the install — each one a - complete set of OAuth client credentials sitting in the plugin - directory. - """ - backups = sorted( - plugin_dir.glob('credentials.json.backup.*'), - key=lambda p: p.stat().st_mtime, - reverse=True, - ) - for stale in backups[_MAX_CREDENTIAL_BACKUPS:]: - try: - stale.unlink() - except OSError: - logger.warning("Could not remove old credential backup %s", stale.name) - - -@api_v3.route('/plugins/calendar/upload-credentials', methods=['POST']) -def upload_calendar_credentials(): - """Upload credentials.json file for calendar plugin""" - try: - if 'file' not in request.files: - return jsonify({'status': 'error', 'message': 'No file provided'}), 400 - - file = request.files['file'] - if not file or not file.filename: - return jsonify({'status': 'error', 'message': 'No file provided'}), 400 - - # Validate file extension - if not file.filename.lower().endswith('.json'): - return jsonify({'status': 'error', 'message': 'File must be a JSON file (.json)'}), 400 - - # Validate file size (max 1MB for credentials) - file.seek(0, os.SEEK_END) - file_size = file.tell() - file.seek(0) - - if file_size > 1024 * 1024: # 1MB - return jsonify({'status': 'error', 'message': 'File exceeds 1MB limit'}), 400 - - # Validate it's valid JSON - try: - file_content = file.read() - file.seek(0) - creds_data = json.loads(file_content) - except json.JSONDecodeError: - return jsonify({'status': 'error', 'message': 'File is not valid JSON'}), 400 - - # Validate it looks like Google OAuth credentials. A bare scalar, a - # list, true/null — all valid JSON, none of them credentials. Reject - # rather than save: a file written as credentials.json but unusable - # as credentials only fails later, somewhere less obvious. - if not isinstance(creds_data, dict) or not ( - 'installed' in creds_data or 'web' in creds_data): - return jsonify({ - 'status': 'error', - 'message': 'File does not appear to be a valid Google OAuth credentials file' - }), 400 - - # Get plugin directory - plugin_id = 'calendar' - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) - else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id - - if not plugin_dir or not Path(plugin_dir).exists(): - return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - - # Save file to plugin directory - credentials_path = Path(plugin_dir) / 'credentials.json' - - # Backup existing file if it exists - if credentials_path.exists(): - backup_path = Path(plugin_dir) / f'credentials.json.backup.{int(time.time())}' - import shutil - shutil.copy2(credentials_path, backup_path) - _prune_credential_backups(Path(plugin_dir)) - - # Save new file - file.save(str(credentials_path)) - - # Set proper permissions - os.chmod(credentials_path, 0o600) # Read/write for owner only - - return jsonify({ - 'status': 'success', - 'message': 'Credentials file uploaded successfully', - 'path': str(credentials_path) - }) - - except Exception as e: - logger.error('Error in upload_calendar_credentials', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -# calendarList.list pages at 250 entries maximum. Ten pages is far past any -# real account and exists only so a malformed nextPageToken cannot spin here. -_CALENDAR_LIST_MAX_PAGES = 10 - - -def _calendar_plugin_dir() -> Optional[Path]: - """Where the calendar plugin is installed, or None if it is not.""" - if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory('calendar') - else: - plugin_dir = PROJECT_ROOT / 'plugins' / 'calendar' - if not plugin_dir: - return None - plugin_dir = Path(plugin_dir) - return plugin_dir if plugin_dir.exists() else None - - -def _run_calendar_registration(plugin_dir: Path, stdin_payload: str): - """Run the plugin's OAuth script and return the JSON object it prints. - - The script decides between web and terminal mode by whether stdin is a - tty, so it must be given a pipe. It emits one JSON object on stdout; the - last parsable line is taken, because an import warning or a library's - stderr redirection can land in front of it. - - Returns (payload, error_message). Exactly one is None. - """ - script = plugin_dir / 'calendar_registration.py' - if not script.exists(): - return None, 'Authentication script not found in the calendar plugin' - - try: - result = subprocess.run( # nosec B603 - fixed script path inside the plugin dir - [sys.executable, str(script)], - input=stdin_payload, - capture_output=True, - text=True, - timeout=120, - cwd=str(plugin_dir), - ) - except subprocess.TimeoutExpired: - return None, 'Authentication timed out after 120s' - except OSError as e: - logger.error('Could not run calendar_registration.py', exc_info=True) - return None, 'Could not run the authentication script: %s' % describe_exception(e) - - for line in reversed((result.stdout or '').splitlines()): - line = line.strip() - if not line: - continue - try: - payload = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(payload, dict): - return payload, None - - raw = (result.stderr or result.stdout or '').strip() - # The unredacted text goes to the log, where it is worth having in full. - # What comes back over HTTP is redacted: this is a script that handles - # OAuth client secrets, and its stderr can quote them. - if raw: - logger.error('calendar_registration.py failed (exit %s): %s', - result.returncode, raw) - return None, 'Authentication script produced no result%s' % ( - ': %s' % redact_text(raw) if raw else '') - - -@api_v3.route('/plugins/calendar/authenticate', methods=['POST']) -def authenticate_calendar(): - """Google OAuth for the calendar plugin, in the two steps it requires. - - Step 1 (no body) returns the consent URL to open. Step 2 posts back the - URL Google redirected to -- it fails to load, because the redirect points - at a loopback address nothing is listening on, but the address bar carries - the authorization code -- and the script exchanges it for a token. - - Two calls rather than one because the user has to visit Google in between. - The script persists the PKCE verifier from step 1 for step 2 to reuse; the - exchange fails with "Missing code verifier" otherwise. - """ - try: - plugin_dir = _calendar_plugin_dir() - if plugin_dir is None: - return jsonify({ - 'status': 'error', - 'message': 'The calendar plugin is not installed' - }), 404 - - if not (plugin_dir / 'credentials.json').exists(): - return jsonify({ - 'status': 'error', - 'message': ('No credentials.json yet. Upload your Google OAuth ' - 'client file first (Step 1).') - }), 400 - - data = request.get_json(silent=True) or {} - redirect_url = (data.get('redirect_url') or data.get('code') or '').strip() - - payload, error = _run_calendar_registration(plugin_dir, redirect_url) - if error: - return jsonify({'status': 'error', 'message': error}), 500 - if payload.get('status') != 'success': - # The script's own diagnosis is more useful than anything that - # could be reconstructed here -- but it interpolates exceptions - # into its messages, so it reaches the client redacted and the - # original goes to the log. - logger.error('calendar authentication failed: %s', payload) - safe = dict(payload) - safe['message'] = redact_text(str(payload.get('message', '') - or 'Authentication failed')) - return jsonify(safe), 400 - return jsonify(payload) - - except Exception as e: - logger.error('Error in authenticate_calendar', exc_info=True) - return jsonify({'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e)}), 500 - - -@api_v3.route('/plugins/calendar/list-calendars', methods=['GET']) -def list_calendar_calendars(): - """The calendars this account can see, for the config picker. - - Reads the token the OAuth flow wrote rather than shelling out again: the - picker is used interactively and a subprocess per click is slower than the - API call it would be wrapping. - """ - try: - plugin_dir = _calendar_plugin_dir() - if plugin_dir is None: - return jsonify({ - 'status': 'error', - 'message': 'The calendar plugin is not installed' - }), 404 - - token_file = plugin_dir / 'token.pickle' - if not token_file.exists(): - return jsonify({ - 'status': 'error', - 'message': ('Not authenticated with Google yet. Complete Step 2 ' - 'first, then load your calendars.') - }), 400 - - try: - import pickle - from google.auth.transport.requests import Request as GoogleRequest - from googleapiclient.discovery import build as build_google_service - except ImportError as e: - return jsonify({ - 'status': 'error', - # The name of the missing module is the whole diagnosis, but it - # arrives as an exception, so it goes through the redactor like - # any other -- an ImportError can quote a path. - 'message': ('The Google API libraries are not installed. Install ' - "the calendar plugin's requirements.txt. (%s)" - % describe_exception(e)) - }), 500 - - with open(token_file, 'rb') as handle: - # Written only by this plugin's own OAuth flow, into its own - # directory, and read here exactly as the plugin itself reads it. - creds = pickle.load(handle) # nosec B301 - locally generated token - - if creds and creds.expired and creds.refresh_token: - creds.refresh(GoogleRequest()) - with open(token_file, 'wb') as handle: - pickle.dump(creds, handle) - os.chmod(token_file, 0o600) - - if not creds or not creds.valid: - return jsonify({ - 'status': 'error', - 'message': ('Stored Google credentials are no longer valid. ' - 'Run Step 2 again to re-authenticate.') - }), 400 - - service = build_google_service('calendar', 'v3', credentials=creds) - - # calendarList.list returns 100 entries per page by default and caps at - # 250, handing back a nextPageToken when there are more. Taking only - # the first page would silently hide calendars from the picker, and the - # user would have no way to tell the list was truncated. - entries = [] - page_token = None - for _ in range(_CALENDAR_LIST_MAX_PAGES): - response = service.calendarList().list( - maxResults=250, pageToken=page_token).execute() - entries.extend(response.get('items', [])) - page_token = response.get('nextPageToken') - if not page_token: - break - else: - # 2500 calendars in, something is wrong with the account or the - # token is looping; show what was collected rather than spin. - logger.warning( - 'calendarList paging stopped at %d pages with more remaining', - _CALENDAR_LIST_MAX_PAGES) - - calendars = [{ - 'id': entry.get('id'), - # The picker labels each row with summary and falls back to the id - # only in its own display, so send something either way. - 'summary': entry.get('summary') or entry.get('id'), - 'primary': bool(entry.get('primary', False)), - } for entry in entries if entry.get('id')] - - # Primary first, then alphabetically: the list is usually short but the - # one the user wants is almost always their own calendar. - calendars.sort(key=lambda c: (not c['primary'], c['summary'].lower())) - - return jsonify({'status': 'success', 'calendars': calendars}) - - except Exception as e: - logger.error('Error in list_calendar_calendars', exc_info=True) - return jsonify({'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e)}), 500 - - -@api_v3.route('/plugins/assets/delete', methods=['POST']) -def delete_plugin_asset(): - """Delete an asset file for a plugin""" - try: - data = request.get_json() - plugin_id = data.get('plugin_id') - image_id = data.get('image_id') - - if not plugin_id or not image_id: - return jsonify({'status': 'error', 'message': 'plugin_id and image_id are required'}), 400 - - # Get asset directory - assets_dir = PROJECT_ROOT / 'assets' / 'plugins' / plugin_id / 'uploads' - metadata_file = assets_dir / '.metadata.json' - - if not metadata_file.exists(): - return jsonify({'status': 'error', 'message': 'Metadata file not found'}), 404 - - # Load metadata - with open(metadata_file, 'r') as f: - metadata = json.load(f) - - if image_id not in metadata: - return jsonify({'status': 'error', 'message': 'Image not found'}), 404 - - # Delete file - file_path = PROJECT_ROOT / metadata[image_id]['path'] - if file_path.exists(): - file_path.unlink() - - # Remove from metadata - del metadata[image_id] - - # Save metadata - with open(metadata_file, 'w') as f: - json.dump(metadata, f, indent=2) - - return jsonify({'status': 'success', 'message': 'Image deleted successfully'}) - - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/plugins/assets/list', methods=['GET']) -def list_plugin_assets(): - """List asset files for a plugin""" - try: - plugin_id = request.args.get('plugin_id') - if not plugin_id: - return jsonify({'status': 'error', 'message': 'plugin_id is required'}), 400 - - # Get asset directory - assets_dir = PROJECT_ROOT / 'assets' / 'plugins' / plugin_id / 'uploads' - metadata_file = assets_dir / '.metadata.json' - - if not metadata_file.exists(): - return jsonify({'status': 'success', 'data': {'assets': []}}) - - # Load metadata - with open(metadata_file, 'r') as f: - metadata = json.load(f) - - # Convert to list - assets = list(metadata.values()) - - return jsonify({'status': 'success', 'data': {'assets': assets}}) - - except Exception as e: - logger.error('Unhandled exception', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/display/current-status', methods=['GET']) -def get_current_display_status(): - """Return the display mode/plugin currently intended to be shown. - - Published by the display process (display_controller._publish_current_mode_state) - to the shared cache whenever the active mode changes, so the web UI (e.g. the - System Logs page) can show what's on screen without querying the display - process directly. - """ - try: - cache = _ensure_cache_manager() - state = cache.get('display_current_state', max_age=120) - if state is None: - state = { - 'mode': None, - 'plugin_id': None, - 'last_updated': None, - } - return jsonify({'status': 'success', 'data': state}) - except Exception as e: - logger.error('Error in get_current_display_status', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/logs', methods=['GET']) -def get_logs(): - """Get system logs from journalctl""" - try: - if not _JOURNALCTL: - return jsonify({'status': 'error', 'message': 'journalctl not found on this system'}), 503 - # Get recent logs from journalctl - _cmd = ([_SUDO, _JOURNALCTL] if _SUDO else [_JOURNALCTL]) + [ - '-u', 'ledmatrix.service', '-u', 'ledmatrix-web.service', - '-n', '100', '--no-pager', '--output=short-iso'] - result = subprocess.run( - _cmd, - capture_output=True, - text=True, - timeout=5 - ) - - if result.returncode == 0: - logs_text = result.stdout.strip() - return jsonify({ - 'status': 'success', - 'data': { - 'logs': logs_text if logs_text else 'No logs available from ledmatrix or ledmatrix-web service' - } - }) - else: - return jsonify({ - 'status': 'error', - 'message': f'Failed to get logs: {result.stderr}' - }), 500 - - except subprocess.TimeoutExpired: - return jsonify({ - 'status': 'error', - 'message': 'Timeout while fetching logs' - }), 500 - except Exception as e: - logger.error("%s failed", request.path, exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e) - }), 500 - -# Multi-Display Sync Endpoints -@api_v3.route('/sync/status', methods=['GET']) -def get_sync_status(): - """Return live multi-display sync status written by the display process.""" - import os as _os - status_file = "/tmp/led_matrix_sync_status.json" - # Also surface config so the UI can show the configured role even before - # the display process has written a status file. - cfg_role = "standalone" - cfg_port = 5765 - if api_v3.config_manager: - try: - cfg = api_v3.config_manager.load_config().get("sync", {}) - cfg_role = cfg.get("role", "standalone") - cfg_port = int(cfg.get("port", 5765)) - except Exception: - pass - - if _os.path.exists(status_file): - try: - with open(status_file) as f: - live = json.load(f) - return jsonify({"status": "success", "data": live}) - except Exception: - pass - - # Status file not yet written — return config-only placeholder - return jsonify({ - "status": "success", - "data": { - "role": cfg_role, - "port": cfg_port, - "state": "starting", - } - }) - - -# WiFi Management Endpoints -@api_v3.route('/wifi/status', methods=['GET']) -def get_wifi_status(): - """Get current WiFi connection status""" - try: - from src.wifi_manager import WiFiManager - - wifi_manager = WiFiManager() - status = wifi_manager.get_wifi_status() - - # Get auto-enable setting from config - auto_enable_ap = wifi_manager.config.get("auto_enable_ap_mode", True) # Default: True (safe due to grace period) - - return jsonify({ - 'status': 'success', - 'data': { - 'connected': status.connected, - 'ssid': status.ssid, - 'ip_address': status.ip_address, - 'signal': status.signal, - 'ap_mode_active': status.ap_mode_active, - 'auto_enable_ap_mode': auto_enable_ap - } - }) - except Exception as e: - logger.error("%s failed", request.path, exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e) - }), 500 - -@api_v3.route('/wifi/scan', methods=['GET']) -def scan_wifi_networks(): - """Scan for available WiFi networks - - If AP mode is active, it will be temporarily disabled during scanning - and automatically re-enabled afterward. Users connected to the AP will - be briefly disconnected during this process. - """ - try: - from src.wifi_manager import WiFiManager - - wifi_manager = WiFiManager() - - # Check if AP mode is active before scanning (for user notification) - ap_was_active = wifi_manager._is_ap_mode_active() - - # Perform the scan (this will handle AP mode disabling/enabling internally) - networks, _was_cached = wifi_manager.scan_networks() - - # Convert to dict format - networks_data = [ - { - 'ssid': net.ssid, - 'signal': net.signal, - 'security': net.security, - 'frequency': net.frequency - } - for net in networks - ] - - response_data = { - 'status': 'success', - 'data': networks_data - } - - # Inform user if AP mode was temporarily disabled - if ap_was_active: - response_data['message'] = ( - f'Found {len(networks_data)} networks. ' - 'Note: AP mode was temporarily disabled during scanning and has been re-enabled. ' - 'If you were connected to the setup network, you may need to reconnect.' - ) - - return jsonify(response_data) - except Exception as e: - logger.error("Error scanning WiFi networks", exc_info=True) - error_message = 'An error occurred while scanning WiFi networks; see logs for details' - - # Provide more specific error messages for common issues - error_str = str(e).lower() - if 'permission' in error_str or 'sudo' in error_str: - error_message = ( - 'Permission error while scanning. ' - 'The WiFi scan requires appropriate permissions. ' - 'Please ensure the application has necessary privileges.' - ) - elif 'timeout' in error_str: - error_message = ( - 'WiFi scan timed out. ' - 'The scan took too long to complete. ' - 'This may happen if the WiFi interface is busy or in use.' - ) - elif 'no wifi' in error_str or 'not available' in error_str: - error_message = ( - 'WiFi scanning tools are not available. ' - 'Please ensure NetworkManager (nmcli) or iwlist is installed.' - ) - - return jsonify({ - 'status': 'error', - 'message': error_message - }), 500 - -@api_v3.route('/wifi/connect', methods=['POST']) -def connect_wifi(): - """Connect to a WiFi network""" - try: - from src.wifi_manager import WiFiManager - - data = request.get_json(silent=True) - if not data: - return jsonify({ - 'status': 'error', - 'message': 'Request body is required' - }), 400 - - if 'ssid' not in data: - return jsonify({ - 'status': 'error', - 'message': 'SSID is required' - }), 400 - - ssid = data['ssid'] - if not ssid or not ssid.strip(): - return jsonify({ - 'status': 'error', - 'message': 'SSID cannot be empty' - }), 400 - - ssid = ssid.strip() - password = data.get('password', '') or '' - - wifi_manager = WiFiManager() - success, message = wifi_manager.connect_to_network(ssid, password) - - if success: - return jsonify({ - 'status': 'success', - 'message': message - }) - else: - return jsonify({ - 'status': 'error', - 'message': message or 'Failed to connect to network' - }), 400 - except Exception as e: - logger.error("Error connecting to WiFi", exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) - }), 500 - -@api_v3.route('/wifi/disconnect', methods=['POST']) -def disconnect_wifi(): - """Disconnect from the current WiFi network""" - try: - from src.wifi_manager import WiFiManager - - wifi_manager = WiFiManager() - success, message = wifi_manager.disconnect_from_network() - - if success: - return jsonify({ - 'status': 'success', - 'message': message - }) - else: - return jsonify({ - 'status': 'error', - 'message': message or 'Failed to disconnect from network' - }), 400 - except Exception as e: - logger.error("Error disconnecting from WiFi", exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) - }), 500 - -@api_v3.route('/wifi/ap/enable', methods=['POST']) -def enable_ap_mode(): - """Enable access point mode""" - try: - from src.wifi_manager import WiFiManager - - wifi_manager = WiFiManager() - _force_raw = (request.get_json(silent=True) or {}).get('force', False) - force = _force_raw is True or (isinstance(_force_raw, str) and _force_raw.lower() in ('true', '1')) - success, message = wifi_manager.enable_ap_mode(force=force) - - if success: - return jsonify({ - 'status': 'success', - 'message': message - }) - else: - return jsonify({ - 'status': 'error', - 'message': message - }), 400 - except Exception as e: - logger.error("%s failed", request.path, exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e) - }), 500 - -@api_v3.route('/wifi/ap/disable', methods=['POST']) -def disable_ap_mode(): - """Disable access point mode""" - try: - from src.wifi_manager import WiFiManager - - wifi_manager = WiFiManager() - success, message = wifi_manager.disable_ap_mode() - - if success: - return jsonify({ - 'status': 'success', - 'message': message - }) - else: - return jsonify({ - 'status': 'error', - 'message': message - }), 400 - except Exception as e: - logger.error("%s failed", request.path, exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e) - }), 500 - -@api_v3.route('/wifi/ap/auto-enable', methods=['GET']) -def get_auto_enable_ap_mode(): - """Get auto-enable AP mode setting""" - try: - from src.wifi_manager import WiFiManager - - wifi_manager = WiFiManager() - auto_enable = wifi_manager.config.get("auto_enable_ap_mode", True) # Default: True (safe due to grace period) - - return jsonify({ - 'status': 'success', - 'data': { - 'auto_enable_ap_mode': auto_enable - } - }) - except Exception as e: - logger.error("%s failed", request.path, exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e) - }), 500 - -@api_v3.route('/wifi/ap/auto-enable', methods=['POST']) -def set_auto_enable_ap_mode(): - """Set auto-enable AP mode setting""" - try: - from src.wifi_manager import WiFiManager - - data = request.get_json(silent=True) - if data is None or 'auto_enable_ap_mode' not in data: - return jsonify({ - 'status': 'error', - 'message': 'auto_enable_ap_mode is required' - }), 400 - - auto_enable = bool(data['auto_enable_ap_mode']) - - wifi_manager = WiFiManager() - wifi_manager.config["auto_enable_ap_mode"] = auto_enable - wifi_manager._save_config() - - return jsonify({ - 'status': 'success', - 'message': f'Auto-enable AP mode set to {auto_enable}', - 'data': { - 'auto_enable_ap_mode': auto_enable - } - }) - except Exception as e: - logger.error("%s failed", request.path, exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', - 'details': describe_exception(e) - }), 500 - -@api_v3.route('/wifi/radio', methods=['GET']) -def get_wifi_radio(): - """Get current WiFi radio state (enabled/disabled) and wired-fallback status.""" - try: - from src.wifi_manager import WiFiManager - - wifi_manager = WiFiManager() - state = wifi_manager.get_wifi_radio_state() - - return jsonify({ - 'status': 'success', - 'data': state - }) - except Exception as e: - logger.error("Error getting WiFi radio state", exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) - }), 500 - -@api_v3.route('/wifi/radio', methods=['POST']) -def set_wifi_radio(): - """Turn the WiFi radio on or off. - - Body: {"enabled": bool, "force": bool (optional)}. Disabling is refused - unless Ethernet is connected or force=True, to avoid locking the user out - of this web interface. - """ - try: - from src.wifi_manager import WiFiManager - - data = request.get_json(silent=True) or {} - if 'enabled' not in data: - return jsonify({ - 'status': 'error', - 'message': 'enabled is required' - }), 400 - - # Parse defensively: bool("false") is True, so mirror the string-aware - # coercion used for `force` — the endpoint is a public contract, not just - # the shipped UI (which always sends real JSON booleans). - _enabled_raw = data['enabled'] - enabled = _enabled_raw is True or (isinstance(_enabled_raw, str) and _enabled_raw.lower() in ('true', '1', 'yes')) - _force_raw = data.get('force', False) - force = _force_raw is True or (isinstance(_force_raw, str) and _force_raw.lower() in ('true', '1', 'yes')) - - wifi_manager = WiFiManager() - success, message, reason = wifi_manager.set_wifi_radio(enabled, force=force) - - if success: - return jsonify({ - 'status': 'success', - 'message': message, - 'data': wifi_manager.get_wifi_radio_state() - }) - else: - return jsonify({ - 'status': 'error', - 'message': message, - 'reason': reason - }), 400 - except Exception as e: - logger.error("Error setting WiFi radio state", exc_info=True) - return jsonify({ - 'status': 'error', - 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) - }), 500 - -@api_v3.route('/cache/list', methods=['GET']) -def list_cache_files(): - """List all cache files with metadata""" - try: - if not api_v3.cache_manager: - # Initialize cache manager if not already initialized - from src.cache_manager import CacheManager - api_v3.cache_manager = CacheManager() - - cache_files = api_v3.cache_manager.list_cache_files() - cache_dir = api_v3.cache_manager.get_cache_dir() - - return jsonify({ - 'status': 'success', - 'data': { - 'cache_files': cache_files, - 'cache_dir': cache_dir, - 'total_files': len(cache_files) - } - }) - except Exception as e: - logger.error('Error in list_cache_files', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - -@api_v3.route('/cache/delete', methods=['POST']) -def delete_cache_file(): - """Delete a specific cache file by key""" - try: - if not api_v3.cache_manager: - # Initialize cache manager if not already initialized - from src.cache_manager import CacheManager - api_v3.cache_manager = CacheManager() - - data = request.get_json(silent=True) - if not data or 'key' not in data: - return jsonify({'status': 'error', 'message': 'cache key is required'}), 400 - - cache_key = data['key'] - - # Delete the cache file - api_v3.cache_manager.clear_cache(cache_key) - - return jsonify({ - 'status': 'success', - 'message': f'Cache file for key "{cache_key}" deleted successfully' - }) - except Exception as e: - logger.error('Error in delete_cache_file', exc_info=True) - return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 - - -# ============================================================================= -# Error Aggregation Endpoints -# ============================================================================= - -@api_v3.route('/errors/summary', methods=['GET']) -def get_error_summary(): - """ - Get summary of all errors for monitoring and debugging. - - Returns error counts, detected patterns, and recent errors. - """ - try: - aggregator = get_error_aggregator() - summary = aggregator.get_error_summary() - return success_response(data=summary, message="Error summary retrieved") - except Exception as e: - logger.error(f"Error getting error summary: {e}", exc_info=True) - return error_response( - error_code=ErrorCode.SYSTEM_ERROR, - message="Failed to retrieve error summary", - status_code=500 - ) - - -@api_v3.route('/errors/plugin/', methods=['GET']) -def get_plugin_errors(plugin_id): - """ - Get error health status for a specific plugin. - - Args: - plugin_id: Plugin identifier - - Returns health status and error statistics for the plugin. - """ - try: - aggregator = get_error_aggregator() - health = aggregator.get_plugin_health(plugin_id) - return success_response(data=health, message="Plugin health retrieved") - except Exception as e: - logger.error(f"Error getting plugin health for {plugin_id}: {e}", exc_info=True) - return error_response( - error_code=ErrorCode.SYSTEM_ERROR, - message=f"Failed to retrieve health for plugin {plugin_id}", - status_code=500 - ) - - -@api_v3.route('/errors/clear', methods=['POST']) -def clear_old_errors(): - """ - Clear error records older than specified age. - - Request body (optional): - max_age_hours: Maximum age in hours (default: 24, max: 8760 = 1 year) - """ - try: - data = request.get_json(silent=True) or {} - raw_max_age = data.get('max_age_hours', 24) - - # Validate and coerce max_age_hours - try: - max_age_hours = int(raw_max_age) - if max_age_hours < 1: - return error_response( - error_code=ErrorCode.INVALID_INPUT, - message="max_age_hours must be at least 1", - context={'provided_value': raw_max_age}, - status_code=400 - ) - if max_age_hours > 8760: # 1 year max - return error_response( - error_code=ErrorCode.INVALID_INPUT, - message="max_age_hours cannot exceed 8760 (1 year)", - context={'provided_value': raw_max_age}, - status_code=400 - ) - except (ValueError, TypeError, OverflowError): - return error_response( - error_code=ErrorCode.INVALID_INPUT, - message="max_age_hours must be a valid integer", - context={'provided_value': str(raw_max_age)}, - status_code=400 - ) - - aggregator = get_error_aggregator() - cleared_count = aggregator.clear_old_records(max_age_hours=max_age_hours) - - return success_response( - data={'cleared_count': cleared_count}, - message=f"Cleared {cleared_count} error records older than {max_age_hours} hours" - ) - except Exception as e: - logger.error(f"Error clearing old errors: {e}", exc_info=True) - return error_response( - error_code=ErrorCode.SYSTEM_ERROR, - message="Failed to clear old errors", - status_code=500 - ) - - -# --------------------------------------------------------------------------- -# Backup / Restore -# --------------------------------------------------------------------------- - -def _resolve_backup_export_dir() -> Path: - """Where exported backups live: beside the install, not inside it. - - They used to be written to ``/config/backups/exports``. That is - inside the directory a reinstall deletes, so the documented recovery path - -- export a backup, then reinstall -- destroyed the backup it had just - told the user to make. Anyone who downloaded the ZIP was fine; anyone - relying on the on-device copy was not. - - Falls back to the old location when the parent directory is not writable, - so an unusual layout degrades to previous behaviour instead of failing to - export at all. - """ - preferred = PROJECT_ROOT.parent / "ledmatrix-backups" - fallback = PROJECT_ROOT / "config" / "backups" / "exports" - try: - preferred.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile(dir=preferred, prefix=".writetest-"): - pass - return preferred - except OSError as e: - logger.warning( - f"[Backup] Export dir {preferred} is not writable ({e}); " - f"falling back to {fallback}, which a reinstall will delete" - ) - return fallback - - -_BACKUP_EXPORT_DIR = _resolve_backup_export_dir() - - -def _safe_backup_path(filename: str) -> Path: - """Resolve a filename to an absolute path inside the export dir, - rejecting any traversal attempts. Returns None if unsafe.""" - # Use basename first (CodeQL-recognized sanitizer) then validate format - filename = os.path.basename(filename or '') - if not filename or not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9._-]{0,200}\.zip$', filename): - return None - path = (_BACKUP_EXPORT_DIR / filename).resolve() - try: - path.relative_to(_BACKUP_EXPORT_DIR.resolve()) - except ValueError: - return None - return path - - -@api_v3.route('/backup/preview', methods=['GET']) -def backup_preview(): - """Return a summary of what a new backup would include.""" - try: - from src.backup_manager import preview_backup_contents - data = preview_backup_contents(PROJECT_ROOT) - return jsonify({'status': 'success', 'data': data}) - except Exception as e: - logger.error("backup_preview failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 - - -@api_v3.route('/backup/list', methods=['GET']) -def backup_list(): - """List backup ZIPs stored in the export directory.""" - try: - _BACKUP_EXPORT_DIR.mkdir(parents=True, exist_ok=True) - entries = [] - for p in sorted(_BACKUP_EXPORT_DIR.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True): - if not p.is_file() or p.suffix != '.zip': - continue - st = p.stat() - entries.append({ - 'filename': p.name, - 'size': st.st_size, - 'created_at': datetime.fromtimestamp(st.st_mtime).strftime('%Y-%m-%d %H:%M:%S'), - }) - return jsonify({'status': 'success', 'data': entries}) - except Exception as e: - logger.error("backup_list failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 - - -@api_v3.route('/backup/export', methods=['POST']) -def backup_export(): - """Create a new backup ZIP and return its filename.""" - try: - from src.backup_manager import create_backup - zip_path = create_backup(PROJECT_ROOT, output_dir=_BACKUP_EXPORT_DIR) - return jsonify({'status': 'success', 'filename': zip_path.name}) - except Exception as e: - logger.error("backup_export failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 - - -@api_v3.route('/backup/validate', methods=['POST']) -def backup_validate(): - """Validate an uploaded backup ZIP and return its manifest.""" - try: - from src.backup_manager import validate_backup - if 'backup_file' not in request.files: - return jsonify({'status': 'error', 'message': 'No backup_file in request'}), 400 - f = request.files['backup_file'] - with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp: - tmp_path = tmp.name - f.save(tmp_path) - try: - ok, err_msg, manifest = validate_backup(Path(tmp_path)) - finally: - try: - os.unlink(tmp_path) - except OSError: - pass - if not ok: - logger.warning("Backup validation failed: %s", err_msg) - return jsonify({'status': 'error', 'message': 'Invalid or corrupted backup file'}), 400 - safe_manifest = { - 'schema_version': manifest.get('schema_version'), - 'created_at': manifest.get('created_at'), - 'ledmatrix_version': manifest.get('ledmatrix_version'), - 'hostname': manifest.get('hostname'), - 'contents': manifest.get('contents', []), - 'detected_contents': manifest.get('detected_contents', []), - 'plugins': manifest.get('plugins', []), - 'total_uncompressed': manifest.get('total_uncompressed'), - 'file_count': manifest.get('file_count'), - } - return jsonify({'status': 'success', 'data': safe_manifest}) - except Exception as e: - logger.error("backup_validate failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 - - -@api_v3.route('/backup/restore', methods=['POST']) -def backup_restore(): - """Restore a backup ZIP with optional RestoreOptions.""" - try: - from src.backup_manager import restore_backup, RestoreOptions - if 'backup_file' not in request.files: - return jsonify({'status': 'error', 'message': 'No backup_file in request'}), 400 - f = request.files['backup_file'] - options_raw = request.form.get('options', '{}') - try: - opts_dict = json.loads(options_raw) - except json.JSONDecodeError: - opts_dict = None - if not isinstance(opts_dict, dict): - # Every option defaults to True, so falling back to {} on a - # parse failure would silently perform a FULL restore — - # secrets and all — for a caller who asked for a narrow one - # and mis-serialized it. Refuse instead of guessing. - return jsonify({ - 'status': 'error', - 'message': 'Invalid options: expected a JSON object', - }), 400 - options = RestoreOptions( - restore_config=bool(opts_dict.get('restore_config', True)), - restore_secrets=bool(opts_dict.get('restore_secrets', True)), - restore_wifi=bool(opts_dict.get('restore_wifi', True)), - restore_fonts=bool(opts_dict.get('restore_fonts', True)), - restore_plugin_uploads=bool(opts_dict.get('restore_plugin_uploads', True)), - reinstall_plugins=bool(opts_dict.get('reinstall_plugins', True)), - ) - with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp: - tmp_path = tmp.name - f.save(tmp_path) - try: - result = restore_backup(Path(tmp_path), PROJECT_ROOT, options) - finally: - try: - os.unlink(tmp_path) - except OSError: - pass - - # Reinstall plugins if requested and store manager available - if options.reinstall_plugins and result.plugins_to_install: - psm = getattr(api_v3, 'plugin_store_manager', None) or plugin_store_manager - for plug in result.plugins_to_install: - pid = plug.get('plugin_id') - if not pid: - continue - try: - if psm and hasattr(psm, 'install_plugin'): - ok = psm.install_plugin(pid) - if ok: - result.plugins_installed.append(pid) - else: - result.plugins_failed.append({'plugin_id': pid, 'error': 'install_plugin returned False'}) - else: - result.plugins_failed.append({'plugin_id': pid, 'error': 'Store manager unavailable'}) - except Exception as pe: - logger.error( - "[Backup] Failed to reinstall plugin %r: %s", pid, pe, exc_info=True - ) - result.plugins_failed.append({'plugin_id': pid, 'error': 'Installation failed; see server logs'}) - - # A restore that dropped files can still report success if the only - # failures were plugin reinstalls, since those don't touch result.errors. - if result.plugins_failed: - result.success = False - - data = result.to_dict() - if not result.success: - # Name what failed, and what nonetheless landed. A restore is - # partial far more often than it is total -- a fresh install can - # leave config_secrets.json unwritable by the web service, so - # config restores and secrets do not. "Restore had errors" alone - # left the user unable to tell a wholly failed restore from one - # that quietly dropped their API keys. - failed_plugins = [ - str(p.get('plugin_id')) for p in (result.plugins_failed or []) if p.get('plugin_id') - ] - parts = [] - if result.restored: - parts.append(f"restored: {', '.join(result.restored)}") - if result.errors: - parts.append(f"failed: {'; '.join(result.errors)}") - if failed_plugins: - parts.append(f"plugins not reinstalled: {', '.join(failed_plugins)}") - message = 'Restore incomplete — ' + ('. '.join(parts) if parts else 'see logs') - return jsonify({'status': 'error', 'message': message, 'data': data}), 500 - return jsonify({'status': 'success', 'data': data}) - except Exception as e: - logger.error("backup_restore failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 - - -@api_v3.route('/backup/download/', methods=['GET']) -def backup_download(filename): - """Stream a backup ZIP to the browser.""" - from flask import send_from_directory - if _safe_backup_path(filename) is None: - return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 - try: - # send_from_directory uses werkzeug safe_join internally — CodeQL-recognized sanitizer. - return send_from_directory(_BACKUP_EXPORT_DIR, filename, as_attachment=True) - except FileNotFoundError: - return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 - - -@api_v3.route('/backup/', methods=['DELETE']) -def backup_delete(filename): - """Delete a stored backup ZIP.""" - safe = _safe_backup_path(filename) - if safe is None: - return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 - # Enumerate the export directory and match by name so the unlink target is - # a filesystem-derived path rather than one constructed from user input. - try: - for entry in _BACKUP_EXPORT_DIR.iterdir(): - if entry.is_file() and entry.name == safe.name: - entry.unlink() - return jsonify({'status': 'success'}) - except OSError as e: - logger.error("backup_delete failed: %s", e, exc_info=True) - return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 - return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 - - -# ── Starlark / Pixlet ──────────────────────────────────────────────────────── -# These routes were added by #253 and removed by #330, which rewrote this file -# and dropped all thirteen of them. Nothing else changed: the frontend still -# calls them and plugin-repos/starlark-apps still implements the work behind -# them, so every call has been landing on Flask's 404 handler and coming back -# as the generic "Resource not found" -- which is what the Pixlet install -# button reports. -# -# Restored verbatim from 1c4d5c52^ (the commit before the removal). The other -# eleven are still missing; see the PR description. - -_STARLARK_APPS_DIR = PROJECT_ROOT / 'starlark-apps' -_STARLARK_MANIFEST_FILE = _STARLARK_APPS_DIR / 'manifest.json' - -def _get_starlark_plugin() -> Optional[Any]: - """Get the starlark-apps plugin instance, or None.""" - if not api_v3.plugin_manager: - return None - return api_v3.plugin_manager.get_plugin('starlark-apps') - -def _find_pixlet_binary(explicit_path: Optional[str] = None) -> Optional[str]: - """Find pixlet binary: explicit path → bundled binary → system PATH.""" - import platform - if explicit_path and os.path.isfile(explicit_path) and os.access(explicit_path, os.X_OK): - return explicit_path - bin_dir = PROJECT_ROOT / "bin" / "pixlet" - system = platform.system().lower() - machine = platform.machine().lower() - if system == "linux": - if "aarch64" in machine or "arm64" in machine: - name = "pixlet-linux-arm64" - elif "x86_64" in machine or "amd64" in machine: - name = "pixlet-linux-amd64" - else: - name = None - elif system == "darwin": - name = "pixlet-darwin-arm64" if "arm64" in machine else "pixlet-darwin-amd64" - else: - name = None - if name: - bundled = bin_dir / name - if bundled.is_file(): - if os.access(str(bundled), os.X_OK): - return str(bundled) - try: - bundled.chmod(0o755) - except OSError: - logger.warning("Could not make pixlet bundled binary executable (%s); falling back to PATH", bundled) - else: - if os.access(str(bundled), os.X_OK): - return str(bundled) - logger.warning("Pixlet bundled binary still not executable after chmod (%s); falling back to PATH", bundled) - return shutil.which("pixlet") - -def _read_starlark_manifest() -> Dict[str, Any]: - """Read the starlark-apps manifest.json directly from disk.""" - try: - if _STARLARK_MANIFEST_FILE.exists(): - with open(_STARLARK_MANIFEST_FILE, 'r') as f: - return json.load(f) - except (json.JSONDecodeError, OSError) as e: - logger.error(f"Error reading starlark manifest: {e}") - return {'apps': {}} - -@api_v3.route('/starlark/status', methods=['GET']) -def get_starlark_status(): - """Get Starlark plugin status and Pixlet availability.""" - try: - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - info = starlark_plugin.get_info() - magnify_info = starlark_plugin.get_magnify_recommendation() - return jsonify({ - 'status': 'success', - 'pixlet_available': info.get('pixlet_available', False), - 'pixlet_version': info.get('pixlet_version'), - 'installed_apps': info.get('installed_apps', 0), - 'enabled_apps': info.get('enabled_apps', 0), - 'current_app': info.get('current_app'), - 'plugin_enabled': starlark_plugin.enabled, - 'display_info': magnify_info - }) - - # Plugin not loaded - check Pixlet availability via shared resolver - # (respects user-configured pixlet_path, bundled binary, and system PATH) - full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {} - pixlet_path = _find_pixlet_binary(full_config.get('starlark-apps', {}).get('pixlet_path')) - pixlet_available = pixlet_path is not None - - # Read app counts from manifest - manifest = _read_starlark_manifest() - apps = manifest.get('apps', {}) - installed_count = len(apps) - enabled_count = sum(1 for a in apps.values() if a.get('enabled', True)) - - return jsonify({ - 'status': 'success', - 'pixlet_available': pixlet_available, - 'pixlet_version': None, - 'installed_apps': installed_count, - 'enabled_apps': enabled_count, - 'plugin_enabled': True, - 'plugin_loaded': False, - 'display_info': {} - }) - - except Exception as e: - logger.exception("[Starlark] get_starlark_status failed") - return jsonify({'status': 'error', 'message': 'Failed to get Starlark status', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/install-pixlet', methods=['POST']) -def install_pixlet(): - """Download and install Pixlet binary.""" - try: - script_path = PROJECT_ROOT / 'scripts' / 'download_pixlet.sh' - if not script_path.exists(): - return jsonify({'status': 'error', 'message': 'Installation script not found'}), 404 - - os.chmod(script_path, 0o755) - - result = subprocess.run( - [str(script_path)], - cwd=str(PROJECT_ROOT), - capture_output=True, - text=True, - timeout=300 - ) - - if result.returncode == 0: - logger.info("Pixlet downloaded successfully") - return jsonify({'status': 'success', 'message': 'Pixlet installed successfully!', 'output': result.stdout}) - else: - return jsonify({'status': 'error', 'message': f'Failed to download Pixlet: {result.stderr}'}), 500 - - except subprocess.TimeoutExpired as err: - logger.exception("[Starlark] Pixlet download timed out") - return jsonify({'status': 'error', 'message': 'Download timed out', - 'details': describe_exception(err)}), 500 - except Exception as e: - logger.exception("[Starlark] install_pixlet failed") - return jsonify({'status': 'error', 'message': 'Failed to install Pixlet', 'details': describe_exception(e)}), 500 - - -# The remaining eleven routes #330 dropped, restored the same way: apps CRUD, -# per-app config and toggle, render, manual .star upload, and the Tronbyte -# repository browse/categories/install the app store page is built on. Without -# these the store lists nothing and installing anything answers with the same -# generic 404 the Pixlet button did. - -def _starlark_github_token() -> Optional[str]: - """The GitHub token the Starlark store should authenticate with. - - These routes used to read `github_token` off config.json, a key that is - written nowhere and offered by no setting -- so the store always ran - unauthenticated at 60 requests/hour, on the same per-IP budget every - plugin update check spends, while the token the user had actually - configured sat in config_secrets.json raising the same budget to 5000. - The store going blank was that budget running out. - - Prefer the store manager's token, which is the one the settings UI - writes and validates; keep the config.json key as a fallback so a - hand-edited config still works. - """ - token = getattr(api_v3.plugin_store_manager, 'github_token', None) - if token: - return token - - try: - config = api_v3.config_manager.load_config() if api_v3.config_manager else {} - return config.get('github_token') - except Exception: - logger.warning("[Starlark] Could not read config for a GitHub token", exc_info=True) - return None - - -def _get_tronbyte_repository_class() -> Type[Any]: - """Import TronbyteRepository from plugin-repos directory.""" - import importlib.util - import importlib - - module_path = PROJECT_ROOT / 'plugin-repos' / 'starlark-apps' / 'tronbyte_repository.py' - if not module_path.exists(): - raise ImportError(f"TronbyteRepository module not found at {module_path}") - - # If already imported, return cached class - if "tronbyte_repository" in sys.modules: - return sys.modules["tronbyte_repository"].TronbyteRepository - - spec = importlib.util.spec_from_file_location("tronbyte_repository", str(module_path)) - if spec is None: - raise ImportError(f"Failed to create module spec for tronbyte_repository at {module_path}") - - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError("Failed to create module from spec for tronbyte_repository") - - sys.modules["tronbyte_repository"] = module - try: - spec.loader.exec_module(module) - except BaseException: - # A module that failed to execute must not stay in sys.modules: the - # cache branch above would hand back the half-initialised object for - # the rest of the process, so one transient failure would disable - # this path permanently and surface as AttributeError, not ImportError. - sys.modules.pop("tronbyte_repository", None) - raise - return module.TronbyteRepository - -def _get_pixlet_renderer_class() -> Type[Any]: - """Import PixletRenderer from plugin-repos directory.""" - import importlib.util - import importlib - - module_path = PROJECT_ROOT / 'plugin-repos' / 'starlark-apps' / 'pixlet_renderer.py' - if not module_path.exists(): - raise ImportError(f"PixletRenderer module not found at {module_path}") - - # If already imported, return cached class - if "pixlet_renderer" in sys.modules: - return sys.modules["pixlet_renderer"].PixletRenderer - - spec = importlib.util.spec_from_file_location("pixlet_renderer", str(module_path)) - if spec is None: - raise ImportError(f"Failed to create module spec for pixlet_renderer at {module_path}") - - module = importlib.util.module_from_spec(spec) - if module is None: - raise ImportError("Failed to create module from spec for pixlet_renderer") - - sys.modules["pixlet_renderer"] = module - try: - spec.loader.exec_module(module) - except BaseException: - # A module that failed to execute must not stay in sys.modules: the - # cache branch above would hand back the half-initialised object for - # the rest of the process, so one transient failure would disable - # this path permanently and surface as AttributeError, not ImportError. - sys.modules.pop("pixlet_renderer", None) - raise - return module.PixletRenderer - -def _validate_and_sanitize_app_id(app_id: Optional[str], fallback_source: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]: - """Validate and sanitize app_id to a safe slug.""" - if not app_id and fallback_source: - app_id = fallback_source - if not app_id: - return None, "app_id is required" - if '..' in app_id or '/' in app_id or '\\' in app_id: - return None, "app_id contains invalid characters" - - sanitized = re.sub(r'[^a-z0-9_]', '_', app_id.lower()).strip('_') - if not sanitized: - sanitized = f"app_{hashlib.sha256(app_id.encode()).hexdigest()[:12]}" - if sanitized[0].isdigit(): - sanitized = f"app_{sanitized}" - return sanitized, None - -def _validate_timing_value(value: Any, field_name: str, min_val: int = 1, max_val: int = 86400) -> Tuple[Optional[int], Optional[str]]: - """Validate and coerce timing values.""" - if value is None: - return None, None - try: - int_value = int(value) - except (ValueError, TypeError): - return None, f"{field_name} must be an integer" - if int_value < min_val: - return None, f"{field_name} must be at least {min_val}" - if int_value > max_val: - return None, f"{field_name} must be at most {max_val}" - return int_value, None - -def _validate_starlark_app_path(app_id: str) -> Tuple[Optional[Path], Optional[str]]: - """The app's directory, or an error if app_id could escape the base dir. - - Returns the *resolved* path rather than a boolean, and every caller uses - what it returns instead of re-joining ``_STARLARK_APPS_DIR / app_id`` - afterwards. The old shape validated in one place and rebuilt the path in - another, which is two things that have to stay in step -- and is why - CodeQL reported twenty-four path-injection alerts across these handlers - even though the guard was effective: a boolean is not a sanitiser it can - follow, and the value reaching the filesystem was the raw one. - - The name is unchanged so the call sites read the same. - """ - if not isinstance(app_id, str) or not app_id: - return None, "Invalid app_id" - - # Reject the traversal characters outright before touching the filesystem. - if '..' in app_id or '/' in app_id or '\\' in app_id: - return None, "Invalid app_id: contains path traversal characters" - - # os.path.basename strips any directory component, so what is joined below - # cannot carry one. The equality check means this rejects rather than - # silently truncates -- behaviour is identical to the character test above, - # and it is the sanitiser CodeQL's path-injection query actually follows. - # relative_to() alone is a check it cannot trace, which is why twenty-four - # of these stayed flagged after the value was threaded through properly. - safe_name = os.path.basename(app_id) - if safe_name != app_id or safe_name in ('', '.', '..'): - return None, "Invalid app_id: contains path traversal characters" - - try: - base_path = _STARLARK_APPS_DIR.resolve() - app_path = (base_path / safe_name).resolve() - try: - app_path.relative_to(base_path) - except ValueError: - return None, "Invalid app_id: path traversal attempt" - return app_path, None - except OSError as e: - logger.warning("Path validation error for app_id %r: %s", app_id, e) - return None, "Invalid app_id" - - -def _standalone_render_starlark_app(app_id: str) -> Tuple[bool, int, Optional[str]]: - """Render a Starlark app via pixlet directly (no plugin required). - - Reads the .star file and config from starlark-apps/{app_id}/, runs pixlet, - and saves the output to cached_render.webp in the same directory. - This is the web-service fallback when starlark-apps plugin is not loaded. - - Returns (success, http_status_code, error_message). - """ - manifest = _read_starlark_manifest() - if not isinstance(manifest, dict): - return False, 400, "Invalid manifest shape: expected object with 'apps' mapping" - apps = manifest.get('apps', {}) - if not isinstance(apps, dict): - return False, 400, "Invalid manifest shape: expected object with 'apps' mapping" - app_data = apps.get(app_id) - if not app_data: - return False, 404, f"App not found: {app_id}" - - # Validated here as well as at the handler: this is reachable on its own, - # and a path built from app_id should never be assembled without it. - app_dir, path_error = _validate_starlark_app_path(app_id) - if path_error: - return False, 400, path_error - star_file = app_dir / app_data.get('star_file', f'{app_id}.star') - if not star_file.exists(): - return False, 404, f"Star file not found: {star_file}" - - full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {} - plugin_config = full_config.get('starlark-apps', {}) - - pixlet_path = _find_pixlet_binary(plugin_config.get('pixlet_path')) - if not pixlet_path: - return False, 503, "Pixlet binary not found — install pixlet first" - - magnify = plugin_config.get('magnify') - if magnify is None: - hw = full_config.get('display', {}).get('hardware', {}) - cols = hw.get('cols', 64) - chain = hw.get('chain_length', 1) - rows = hw.get('rows', 32) - magnify = max(1, min(8, int(min((cols * chain) / 64, rows / 32)))) - else: - try: - magnify = max(1, min(8, int(magnify))) - except (ValueError, TypeError): - magnify = 1 - - config_file = app_dir / 'config.json' - app_config: Dict[str, Any] = {} - if config_file.exists(): - try: - with open(config_file) as f: - app_config = json.load(f) - except json.JSONDecodeError as e: - logger.warning("Invalid config.json for %r at %s: %s", app_id, config_file, e) - return False, 400, f"Invalid config.json for {app_id}" - except OSError as e: - logger.warning("Cannot read config.json for %r at %s: %s", app_id, config_file, e) - return False, 400, f"Cannot read config.json for {app_id}" - if not isinstance(app_config, dict): - return False, 400, ( - f"config.json for {app_id} must be a JSON object, " - f"got {type(app_config).__name__}" - ) - - INTERNAL_KEYS = {'render_interval', 'display_duration'} - pixlet_config = {k: v for k, v in app_config.items() if k not in INTERNAL_KEYS} - - output_path = str(app_dir / 'cached_render.webp') - cmd = [pixlet_path, 'render', str(star_file)] - for key, value in pixlet_config.items(): - if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', key): - continue - value_str = 'true' if value is True else 'false' if value is False else str(value) - if re.search(r'[`$|<>&;\x00]|\$\(', value_str): - continue - cmd.append(f'{key}={value_str}') - cmd.extend(['-o', output_path, '-m', str(magnify)]) - - try: - result = subprocess.run(cmd, capture_output=True, text=True, timeout=30, cwd=str(app_dir)) - if result.returncode == 0 and os.path.isfile(output_path): - return True, 200, None - return False, 502, f"Pixlet failed (exit {result.returncode}): {result.stderr.strip()}" - except subprocess.TimeoutExpired: - return False, 504, "Render timed out after 30s" - except Exception as e: - logger.exception("Starlark render failed for %r", app_id) - return False, 500, "Render error" - -def _write_starlark_manifest(manifest: Dict[str, Any]) -> bool: - """Write the starlark-apps manifest.json to disk with atomic write.""" - temp_file = None - try: - _STARLARK_APPS_DIR.mkdir(parents=True, exist_ok=True) - - # Atomic write: unique temp file in the target directory, then rename. - # with_suffix('.tmp') gave every caller the same manifest.tmp, and - # Flask serves concurrently -- upload, uninstall, config, toggle and - # the plugin toggle all reach here. Two writers shared one file, - # interleaved their json.dump output, and both renamed it, so the - # rename was atomic over content that was a mix of two manifests. - fd, temp_name = tempfile.mkstemp( - dir=str(_STARLARK_APPS_DIR), prefix='.manifest.', suffix='.tmp') - temp_file = Path(temp_name) - with os.fdopen(fd, 'w') as f: - json.dump(manifest, f, indent=2) - f.flush() - os.fsync(f.fileno()) # Ensure data is written to disk - os.chmod(temp_name, 0o644) # mkstemp creates 0600; match a normal write - - # Atomic rename (overwrites destination) - temp_file.replace(_STARLARK_MANIFEST_FILE) - return True - except OSError as e: - logger.error(f"Error writing starlark manifest: {e}") - # Clean up temp file if it exists - if temp_file and temp_file.exists(): - try: - temp_file.unlink() - except Exception: - pass - return False - -def _install_star_file(app_id: str, star_file_path: str, metadata: Dict[str, Any], assets_dir: Optional[str] = None) -> bool: - """Install a .star file and update the manifest (standalone, no plugin needed).""" - import shutil - import json - app_dir, path_error = _validate_starlark_app_path(app_id) - if path_error: - logger.warning("Refusing to install %r: %s", app_id, path_error) - return False - app_dir.mkdir(parents=True, exist_ok=True) - dest = app_dir / f"{app_id}.star" - shutil.copy2(star_file_path, str(dest)) - - # Copy asset directories if provided (images/, sources/, etc.) - if assets_dir and Path(assets_dir).exists(): - assets_path = Path(assets_dir) - for item in assets_path.iterdir(): - if item.is_dir(): - # Copy entire directory (e.g., images/, sources/) - dest_dir = app_dir / item.name - if dest_dir.exists(): - shutil.rmtree(dest_dir) - shutil.copytree(item, dest_dir) - logger.debug(f"Copied assets directory: {item.name}") - logger.info(f"Installed assets for {app_id}") - - # Try to extract schema using PixletRenderer - schema = None - try: - PixletRenderer = _get_pixlet_renderer_class() - pixlet = PixletRenderer() - if pixlet.is_available(): - _, schema, _ = pixlet.extract_schema(str(dest)) - if schema: - schema_path = app_dir / "schema.json" - with open(schema_path, 'w') as f: - json.dump(schema, f, indent=2) - logger.info(f"Extracted schema for {app_id}") - except Exception as e: - logger.warning(f"Failed to extract schema for {app_id}: {e}") - - # Create default config — pre-populate with schema defaults - default_config = {} - if schema: - fields = schema.get('fields') or schema.get('schema') or [] - for field in fields: - if isinstance(field, dict) and 'id' in field and 'default' in field: - default_config[field['id']] = field['default'] - - # Create config.json file - config_path = app_dir / "config.json" - with open(config_path, 'w') as f: - json.dump(default_config, f, indent=2) - - manifest = _read_starlark_manifest() - manifest.setdefault('apps', {})[app_id] = { - 'name': metadata.get('name', app_id), - 'enabled': True, - 'render_interval': metadata.get('render_interval', 300), - 'display_duration': metadata.get('display_duration', 15), - 'config': metadata.get('config', {}), - # The filename, not the full path. Readers join this to the app's own - # directory and fall back to a bare '.star', so an absolute - # value gave the key two meanings -- and Path.__truediv__ discards the - # left side when the right is absolute, which pinned the manifest to - # whatever PROJECT_ROOT installed it. Moving or redeploying the install - # then left the app unable to find its own file. - 'star_file': dest.name, - } - return _write_starlark_manifest(manifest) - -@api_v3.route('/starlark/apps', methods=['GET']) -def get_starlark_apps(): - """List all installed Starlark apps.""" - try: - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - apps_list = [] - for app_id, app_instance in starlark_plugin.apps.items(): - apps_list.append({ - 'id': app_id, - 'name': app_instance.manifest.get('name', app_id), - 'enabled': app_instance.is_enabled(), - 'has_frames': app_instance.frames is not None, - 'render_interval': app_instance.get_render_interval(), - 'display_duration': app_instance.get_display_duration(), - 'config': app_instance.config, - 'has_schema': app_instance.schema is not None, - 'last_render_time': app_instance.last_render_time - }) - return jsonify({'status': 'success', 'apps': apps_list, 'count': len(apps_list)}) - - # Standalone: read manifest from disk - manifest = _read_starlark_manifest() - apps_list = [] - for app_id, app_data in manifest.get('apps', {}).items(): - apps_list.append({ - 'id': app_id, - 'name': app_data.get('name', app_id), - 'enabled': app_data.get('enabled', True), - 'has_frames': False, - 'render_interval': app_data.get('render_interval', 300), - 'display_duration': app_data.get('display_duration', 15), - 'config': app_data.get('config', {}), - 'has_schema': False, - 'last_render_time': None - }) - return jsonify({'status': 'success', 'apps': apps_list, 'count': len(apps_list)}) - - except Exception as e: - logger.exception("[Starlark] get_starlark_apps failed") - return jsonify({'status': 'error', 'message': 'Failed to get Starlark apps', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/apps/', methods=['GET']) -def get_starlark_app(app_id): - """Get details for a specific Starlark app.""" - try: - # Validate app_id before any filesystem access - app_dir, error_msg = _validate_starlark_app_path(app_id) - if error_msg: - return jsonify({'status': 'error', 'message': error_msg}), 400 - - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - app = starlark_plugin.apps.get(app_id) - if not app: - return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 - return jsonify({ - 'status': 'success', - 'app': { - 'id': app_id, - 'name': app.manifest.get('name', app_id), - 'enabled': app.is_enabled(), - 'config': app.config, - 'schema': app.schema, - 'render_interval': app.get_render_interval(), - 'display_duration': app.get_display_duration(), - 'has_frames': app.frames is not None, - 'frame_count': len(app.frames) if app.frames else 0, - 'last_render_time': app.last_render_time, - } - }) - - # Standalone: read from manifest - manifest = _read_starlark_manifest() - app_data = manifest.get('apps', {}).get(app_id) - if not app_data: - return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 - - # Load schema from schema.json if it exists (path already validated above) - schema = None - schema_file = app_dir / 'schema.json' - if schema_file.exists(): - try: - with open(schema_file, 'r') as f: - schema = json.load(f) - except (OSError, json.JSONDecodeError) as e: - logger.warning(f"Failed to load schema for {app_id}: {e}") - - return jsonify({ - 'status': 'success', - 'app': { - 'id': app_id, - 'name': app_data.get('name', app_id), - 'enabled': app_data.get('enabled', True), - 'config': app_data.get('config', {}), - 'schema': schema, - 'render_interval': app_data.get('render_interval', 300), - 'display_duration': app_data.get('display_duration', 15), - 'has_frames': False, - 'frame_count': 0, - 'last_render_time': None, - } - }) - - except Exception as e: - logger.exception("[Starlark] get_starlark_app failed") - return jsonify({'status': 'error', 'message': 'Failed to get Starlark app', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/upload', methods=['POST']) -def upload_starlark_app(): - """Upload and install a new Starlark app.""" - try: - if 'file' not in request.files: - return jsonify({'status': 'error', 'message': 'No file uploaded'}), 400 - - file = request.files['file'] - if not file.filename or not file.filename.endswith('.star'): - return jsonify({'status': 'error', 'message': 'File must have .star extension'}), 400 - - # Check file size (limit to 5MB for .star files) - file.seek(0, 2) # Seek to end - file_size = file.tell() - file.seek(0) # Reset to beginning - MAX_STAR_SIZE = 5 * 1024 * 1024 # 5MB - if file_size > MAX_STAR_SIZE: - return jsonify({'status': 'error', 'message': f'File too large (max 5MB, got {file_size/1024/1024:.1f}MB)'}), 400 - - app_name = request.form.get('name') - app_id_input = request.form.get('app_id') - filename_base = file.filename.replace('.star', '') if file.filename else None - app_id, app_id_error = _validate_and_sanitize_app_id(app_id_input, fallback_source=filename_base) - if app_id_error: - return jsonify({'status': 'error', 'message': f'Invalid app_id: {app_id_error}'}), 400 - - render_interval_input = request.form.get('render_interval') - render_interval = 300 - if render_interval_input is not None: - render_interval, err = _validate_timing_value(render_interval_input, 'render_interval') - if err: - return jsonify({'status': 'error', 'message': err}), 400 - render_interval = render_interval or 300 - - display_duration_input = request.form.get('display_duration') - display_duration = 15 - if display_duration_input is not None: - display_duration, err = _validate_timing_value(display_duration_input, 'display_duration') - if err: - return jsonify({'status': 'error', 'message': err}), 400 - display_duration = display_duration or 15 - - import tempfile - with tempfile.NamedTemporaryFile(delete=False, suffix='.star') as tmp: - file.save(tmp.name) - temp_path = tmp.name - - try: - metadata = {'name': app_name or app_id, 'render_interval': render_interval, 'display_duration': display_duration} - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - success = starlark_plugin.install_app(app_id, temp_path, metadata) - else: - success = _install_star_file(app_id, temp_path, metadata) - if success: - return jsonify({'status': 'success', 'message': f'App installed: {app_id}', 'app_id': app_id}) - else: - return jsonify({'status': 'error', 'message': 'Failed to install app'}), 500 - finally: - try: - os.unlink(temp_path) - except OSError: - pass - - except (OSError, IOError) as err: - # This used to withhold the detail because it names absolute paths on - # the device. A full disk and a bad permission are indistinguishable - # without it, though, and describe_exception redacts credentials and - # truncates -- the same trade-off every other handler here makes. - logger.exception("[Starlark] File error uploading starlark app: %s", err) - return jsonify({'status': 'error', 'message': 'File error during upload', - 'details': describe_exception(err)}), 500 - except ImportError as err: - logger.exception("[Starlark] Module load error uploading starlark app: %s", err) - return jsonify({'status': 'error', 'message': 'Failed to load app module', 'details': describe_exception(err)}), 500 - except Exception as err: - logger.exception("[Starlark] Unexpected error uploading starlark app: %s", err) - return jsonify({'status': 'error', 'message': 'Failed to upload app', 'details': describe_exception(err)}), 500 - -@api_v3.route('/starlark/apps/', methods=['DELETE']) -def uninstall_starlark_app(app_id): - """Uninstall a Starlark app.""" - try: - # Validate app_id before any filesystem access - app_dir, error_msg = _validate_starlark_app_path(app_id) - if error_msg: - return jsonify({'status': 'error', 'message': error_msg}), 400 - - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - success = starlark_plugin.uninstall_app(app_id) - else: - # Standalone: remove app dir and manifest entry. app_dir is the - # path _validate_starlark_app_path checked, not a fresh join. - import shutil - if app_dir.exists(): - shutil.rmtree(app_dir) - manifest = _read_starlark_manifest() - manifest.get('apps', {}).pop(app_id, None) - success = _write_starlark_manifest(manifest) - - if success: - return jsonify({'status': 'success', 'message': f'App uninstalled: {app_id}'}) - else: - return jsonify({'status': 'error', 'message': 'Failed to uninstall app'}), 500 - - except Exception as e: - logger.exception("[Starlark] uninstall_starlark_app failed") - return jsonify({'status': 'error', 'message': 'Failed to uninstall Starlark app', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/apps//config', methods=['GET']) -def get_starlark_app_config(app_id): - """Get configuration for a Starlark app.""" - try: - # Validate app_id before any filesystem access - app_dir, error_msg = _validate_starlark_app_path(app_id) - if error_msg: - return jsonify({'status': 'error', 'message': error_msg}), 400 - - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - app = starlark_plugin.apps.get(app_id) - if not app: - return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 - return jsonify({'status': 'success', 'config': app.config, 'schema': app.schema}) - - # Standalone: read from config.json. app_dir is the path - # _validate_starlark_app_path checked, not a fresh join. - config_file = app_dir / "config.json" - - if not app_dir.exists(): - return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 - - config = {} - if config_file.exists(): - try: - with open(config_file, 'r') as f: - config = json.load(f) - except (OSError, json.JSONDecodeError) as e: - logger.warning(f"Failed to load config for {app_id}: {e}") - - # Load schema from schema.json - schema = None - schema_file = app_dir / "schema.json" - if schema_file.exists(): - try: - with open(schema_file, 'r') as f: - schema = json.load(f) - except Exception as e: - logger.warning(f"Failed to load schema for {app_id}: {e}") - - return jsonify({'status': 'success', 'config': config, 'schema': schema}) - - except Exception as e: - logger.exception("[Starlark] get_starlark_app_config failed") - return jsonify({'status': 'error', 'message': 'Failed to get Starlark app config', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/apps//config', methods=['PUT']) -def update_starlark_app_config(app_id): - """Update configuration for a Starlark app.""" - try: - # Validate app_id before any filesystem access - app_dir, error_msg = _validate_starlark_app_path(app_id) - if error_msg: - return jsonify({'status': 'error', 'message': error_msg}), 400 - - data = request.get_json(silent=True) - if not data: - return jsonify({'status': 'error', 'message': 'No configuration provided'}), 400 - - if 'render_interval' in data: - val, err = _validate_timing_value(data['render_interval'], 'render_interval') - if err: - return jsonify({'status': 'error', 'message': err}), 400 - data['render_interval'] = val - - if 'display_duration' in data: - val, err = _validate_timing_value(data['display_duration'], 'display_duration') - if err: - return jsonify({'status': 'error', 'message': err}), 400 - data['display_duration'] = val - - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - app = starlark_plugin.apps.get(app_id) - if not app: - return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 - - # Extract timing keys from data before updating config (they belong in manifest, not config) - render_interval = data.pop('render_interval', None) - display_duration = data.pop('display_duration', None) - - # Snapshot before mutating. save_config() can fail, and the route - # answers 500 below when it does -- but the loaded app kept the new - # values anyway, so a later GET returned configuration that was - # never persisted and the plugin rendered with it. - prev_config = dict(app.config) - prev_manifest = dict(app.manifest) - - # Update config with non-timing fields only - app.config.update(data) - - # Update manifest with timing fields - timing_changed = False - if render_interval is not None: - app.manifest['render_interval'] = render_interval - timing_changed = True - if display_duration is not None: - app.manifest['display_duration'] = display_duration - timing_changed = True - saved = app.save_config() - if not saved: - app.config.clear(); app.config.update(prev_config) - app.manifest.clear(); app.manifest.update(prev_manifest) - if saved: - # Persist manifest if timing changed (same pattern as toggle endpoint) - if timing_changed: - try: - # Use safe manifest update to prevent race conditions - timing_updates = {} - if render_interval is not None: - timing_updates['render_interval'] = render_interval - if display_duration is not None: - timing_updates['display_duration'] = display_duration - - def update_fn(manifest): - manifest['apps'][app_id].update(timing_updates) - starlark_plugin._update_manifest_safe(update_fn) - except Exception as e: - logger.warning(f"Failed to persist timing to manifest for {app_id}: {e}") - starlark_plugin._render_app(app, force=True) - return jsonify({'status': 'success', 'message': 'Configuration updated', 'config': app.config}) - else: - return jsonify({'status': 'error', 'message': 'Failed to save configuration'}), 500 - - # Standalone: update both config.json and manifest - manifest = _read_starlark_manifest() - app_data = manifest.get('apps', {}).get(app_id) - if not app_data: - return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 - - # Extract timing keys (they go in manifest, not config.json) - render_interval = data.pop('render_interval', None) - display_duration = data.pop('display_duration', None) - - # Update manifest with timing values - if render_interval is not None: - app_data['render_interval'] = render_interval - if display_duration is not None: - app_data['display_duration'] = display_duration - - # Load current config from config.json. app_dir is the path - # _validate_starlark_app_path checked, not a fresh join. - config_file = app_dir / "config.json" - current_config = {} - if config_file.exists(): - try: - with open(config_file, 'r') as f: - current_config = json.load(f) - except Exception as e: - logger.warning(f"Failed to load config for {app_id}: {e}") - - # Update config with new values (excluding timing keys) - current_config.update(data) - - # Write updated config to config.json - try: - with open(config_file, 'w') as f: - json.dump(current_config, f, indent=2) - except Exception as e: - logger.error(f"Failed to save config.json for {app_id}: {e}") - logger.exception("Failed to save Starlark configuration for %r", app_id) - return jsonify({'status': 'error', 'message': 'Failed to save configuration', - 'details': describe_exception(e)}), 500 - - # Also update manifest for backward compatibility - app_data.setdefault('config', {}).update(data) - - if _write_starlark_manifest(manifest): - return jsonify({'status': 'success', 'message': 'Configuration updated', 'config': current_config}) - else: - return jsonify({'status': 'error', 'message': 'Failed to save manifest'}), 500 - - except Exception as e: - logger.exception("[Starlark] update_starlark_app_config failed") - return jsonify({'status': 'error', 'message': 'Failed to update Starlark app config', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/apps//toggle', methods=['POST']) -def toggle_starlark_app(app_id): - """Enable or disable a Starlark app.""" - try: - data = request.get_json(silent=True) or {} - - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - app = starlark_plugin.apps.get(app_id) - if not app: - return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 - enabled = data.get('enabled') - if enabled is None: - enabled = not app.is_enabled() - app.manifest['enabled'] = enabled - # Use safe manifest update to prevent race conditions - def update_fn(manifest): - manifest['apps'][app_id]['enabled'] = enabled - starlark_plugin._update_manifest_safe(update_fn) - return jsonify({'status': 'success', 'message': f"App {'enabled' if enabled else 'disabled'}", 'enabled': enabled}) - - # Standalone: update manifest directly - manifest = _read_starlark_manifest() - app_data = manifest.get('apps', {}).get(app_id) - if not app_data: - return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 - - enabled = data.get('enabled') - if enabled is None: - enabled = not app_data.get('enabled', True) - app_data['enabled'] = enabled - if _write_starlark_manifest(manifest): - return jsonify({'status': 'success', 'message': f"App {'enabled' if enabled else 'disabled'}", 'enabled': enabled}) - else: - return jsonify({'status': 'error', 'message': 'Failed to save'}), 500 - - except Exception as e: - logger.exception("[Starlark] toggle_starlark_app failed") - return jsonify({'status': 'error', 'message': 'Failed to toggle Starlark app', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/apps//render', methods=['POST']) -def render_starlark_app(app_id): - """Force render a Starlark app.""" - try: - app_dir, err = _validate_starlark_app_path(app_id) - if err: - return jsonify({'status': 'error', 'message': err}), 400 - - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - app = starlark_plugin.apps.get(app_id) - if not app: - return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 - success = starlark_plugin._render_app(app, force=True) - if success: - return jsonify({'status': 'success', 'message': 'App rendered', - 'frame_count': len(app.frames) if app.frames else 0}) - return jsonify({'status': 'error', 'message': 'Failed to render app'}), 500 - - # Web-service context: plugin not loaded, call pixlet directly - success, status_code, error = _standalone_render_starlark_app(app_id) - if success: - return jsonify({'status': 'success', 'message': 'App rendered successfully', 'frame_count': 0}), status_code - return jsonify({'status': 'error', 'message': error or 'Render failed', 'frame_count': 0}), status_code - - except Exception as e: - logger.exception("[Starlark] render_starlark_app failed") - return jsonify({'status': 'error', 'message': 'Failed to render Starlark app', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/repository/browse', methods=['GET']) -def browse_tronbyte_repository(): - """Browse all apps in the Tronbyte repository (bulk cached fetch). - - Returns ALL apps with metadata, categories, and authors. - Filtering/sorting/pagination is handled client-side. - Results are cached server-side for 2 hours. - """ - try: - TronbyteRepository = _get_tronbyte_repository_class() - - repo = TronbyteRepository(github_token=_starlark_github_token()) - - result = repo.list_all_apps_cached() - - rate_limit = repo.get_rate_limit_info() - - # An upstream failure used to arrive here as an empty app list and go - # out as 'success', so the store drew an empty grid and said nothing. - # 502: the request was fine, GitHub was not. - if result.get('error'): - return jsonify({ - 'status': 'error', - 'message': result['error'], - 'rate_limit': rate_limit, - }), 502 - - return jsonify({ - 'status': 'success', - 'apps': result['apps'], - 'categories': result['categories'], - 'authors': result['authors'], - 'count': result['count'], - 'cached': result['cached'], - 'rate_limit': rate_limit, - }) - - except Exception as e: - logger.exception("[Starlark] browse_tronbyte_repository failed") - return jsonify({'status': 'error', 'message': 'Failed to browse repository', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/repository/install', methods=['POST']) -def install_from_tronbyte_repository(): - """Install an app from the Tronbyte repository.""" - try: - data = request.get_json(silent=True) - if not data or 'app_id' not in data: - return jsonify({'status': 'error', 'message': 'app_id is required'}), 400 - - app_id, app_id_error = _validate_and_sanitize_app_id(data['app_id']) - if app_id_error: - return jsonify({'status': 'error', 'message': f'Invalid app_id: {app_id_error}'}), 400 - - TronbyteRepository = _get_tronbyte_repository_class() - import tempfile - - repo = TronbyteRepository(github_token=_starlark_github_token()) - - success, metadata, error = repo.get_app_metadata(data['app_id']) - if not success: - return jsonify({'status': 'error', 'message': f'Failed to fetch app metadata: {error}'}), 404 - - with tempfile.NamedTemporaryFile(delete=False, suffix='.star') as tmp: - temp_path = tmp.name - - try: - # Pass filename from metadata (e.g., "analog_clock.star" for analogclock app) - # Note: manifest uses 'fileName' (camelCase), not 'filename' - filename = metadata.get('fileName') if metadata else None - success, error = repo.download_star_file(data['app_id'], Path(temp_path), filename=filename) - if not success: - return jsonify({'status': 'error', 'message': f'Failed to download app: {error}'}), 500 - - # Download assets (images, sources, etc.) to a temp directory - import tempfile - temp_assets_dir = tempfile.mkdtemp() - try: - success_assets, error_assets = repo.download_app_assets(data['app_id'], Path(temp_assets_dir)) - # Asset download is non-critical - log warning but continue if it fails - if not success_assets: - logger.warning(f"Failed to download assets for {data['app_id']}: {error_assets}") - - render_interval = data.get('render_interval', 300) - ri, err = _validate_timing_value(render_interval, 'render_interval') - if err: - return jsonify({'status': 'error', 'message': err}), 400 - render_interval = ri or 300 - - display_duration = data.get('display_duration', 15) - dd, err = _validate_timing_value(display_duration, 'display_duration') - if err: - return jsonify({'status': 'error', 'message': err}), 400 - display_duration = dd or 15 - - install_metadata = { - 'name': metadata.get('name', app_id) if metadata else app_id, - 'render_interval': render_interval, - 'display_duration': display_duration - } - - starlark_plugin = _get_starlark_plugin() - if starlark_plugin: - success = starlark_plugin.install_app(app_id, temp_path, install_metadata, assets_dir=temp_assets_dir) - else: - success = _install_star_file(app_id, temp_path, install_metadata, assets_dir=temp_assets_dir) - finally: - # Clean up temp assets directory - import shutil - try: - shutil.rmtree(temp_assets_dir) - except OSError: - pass - - if success: - return jsonify({'status': 'success', 'message': f'App installed: {metadata.get("name", app_id) if metadata else app_id}', 'app_id': app_id}) - else: - return jsonify({'status': 'error', 'message': 'Failed to install app'}), 500 - finally: - try: - os.unlink(temp_path) - except OSError: - pass - - except Exception as e: - logger.exception("[Starlark] install_from_tronbyte_repository failed") - return jsonify({'status': 'error', 'message': 'Failed to install from repository', 'details': describe_exception(e)}), 500 - -@api_v3.route('/starlark/repository/categories', methods=['GET']) -def get_tronbyte_categories(): - """Get list of available app categories (uses bulk cache).""" - try: - TronbyteRepository = _get_tronbyte_repository_class() - repo = TronbyteRepository(github_token=_starlark_github_token()) - - result = repo.list_all_apps_cached() - - if result.get('error'): - return jsonify({'status': 'error', 'message': result['error']}), 502 - - return jsonify({'status': 'success', 'categories': result['categories']}) - - except Exception as e: - logger.exception("[Starlark] get_tronbyte_categories failed") - return jsonify({'status': 'error', 'message': 'Failed to fetch categories', 'details': describe_exception(e)}), 500 - - -def _starlark_virtual_plugins() -> list: - """Installed Starlark apps, shaped like plugin entries. - - #253 surfaced these alongside real plugins so an installed .star app can - be seen, enabled and disabled from the same list as everything else; #330 - dropped it with the rest of the Starlark code, which is why an app - installs successfully and then appears nowhere. - - Reads the loaded plugin when there is one and the on-disk manifest - otherwise, so the list is right before starlark-apps has been loaded too. - """ - entries = [] - base = { - 'version': 'starlark', 'category': 'Starlark App', 'tags': ['starlark'], - 'verified': False, 'last_updated': None, 'last_commit': None, - 'last_commit_message': None, 'branch': None, 'web_ui_actions': [], - 'vegas_mode': 'fixed', 'vegas_content_type': 'multi', - 'is_starlark_app': True, - } - try: - plugin = _get_starlark_plugin() - if plugin is not None and hasattr(plugin, 'apps'): - for app_id, app in plugin.apps.items(): - m = getattr(app, 'manifest', {}) or {} - entries.append({**base, - 'id': f'starlark:{app_id}', - 'name': m.get('name', app_id), - 'author': m.get('author', 'Tronbyte Community'), - 'description': m.get('summary', 'Starlark app'), - 'enabled': app.is_enabled(), - 'loaded': True}) - return entries - - for app_id, data in (_read_starlark_manifest().get('apps', {}) or {}).items(): - entries.append({**base, - 'id': f'starlark:{app_id}', - 'name': data.get('name', app_id), - 'author': data.get('author', 'Tronbyte Community'), - 'description': data.get('summary', 'Starlark app'), - 'enabled': data.get('enabled', True), - 'loaded': False}) - except Exception: - # Never let a Starlark problem empty the whole plugins list. - logger.exception('Could not build Starlark virtual plugin entries') - return entries - - -def _toggle_starlark_app(app_id: str, enabled: bool): - """Enable or disable one Starlark app, loaded or not.""" - # Check for traversal, but toggle the key that was listed. - # _starlark_virtual_plugins publishes the raw manifest key, while - # _validate_and_sanitize_app_id lowercases it and rewrites every character - # outside [a-z0-9_]: an app stored as 'My-App' was offered to the UI as - # 'starlark:My-App' and looked up here as 'my_app', so toggling an app the - # page had just drawn answered 404. Keys written by _install_star_file are - # already sanitised; ones written by the plugin, or edited by hand, are - # not. _validate_starlark_app_path rejects traversal without rewriting. - _, err = _validate_starlark_app_path(app_id) - if err: - # err already names app_id; do not prefix it a second time. - return jsonify({'status': 'error', 'message': err}), 400 - safe_id = app_id - - plugin = _get_starlark_plugin() - if plugin is not None and safe_id in getattr(plugin, 'apps', {}): - def _update(manifest): - # setdefault rather than indexing: the app is loaded, but its - # on-disk entry need not exist, and _update_manifest_safe does not - # catch KeyError -- it would escape as a 500 rather than the error - # this returns. - manifest.setdefault('apps', {}).setdefault(safe_id, {})['enabled'] = enabled - - if plugin._update_manifest_safe(_update) is False: - return jsonify({'status': 'error', - 'message': 'Failed to save app state'}), 500 - # Only now is the in-memory copy allowed to disagree with disk. - plugin.apps[safe_id].manifest['enabled'] = enabled - else: - manifest = _read_starlark_manifest() - app_data = manifest.get('apps', {}).get(safe_id) - if not app_data: - return jsonify({'status': 'error', - 'message': f'Starlark app not found: {safe_id}'}), 404 - app_data['enabled'] = enabled - if not _write_starlark_manifest(manifest): - return jsonify({'status': 'error', 'message': 'Failed to save manifest'}), 500 - - return jsonify({'status': 'success', - 'message': f"Starlark app {'enabled' if enabled else 'disabled'}", - 'enabled': enabled}) diff --git a/web_interface/blueprints/api_v3/__init__.py b/web_interface/blueprints/api_v3/__init__.py new file mode 100644 index 00000000..730ecf2e --- /dev/null +++ b/web_interface/blueprints/api_v3/__init__.py @@ -0,0 +1,1853 @@ +"""The /api/v3 blueprint. + +This was one 10,469-line module. It is now a package, but deliberately not a +set of *separate* blueprints: every route module beside this one decorates the +single `api_v3` object defined here, so endpoint names stay `api_v3.` +and the URL map is unchanged. web_interface/app.py registers it as before. + +The shared imports, constants and helpers stay in this file rather than moving +to a _common submodule, because tests monkeypatch some of them by module +attribute -- `monkeypatch.setattr(api_v3_module, "_BACKUP_EXPORT_DIR", ...)`. +Keeping them here means that keeps working exactly as it did. Route modules +read the mutable ones back through this module (see _pkg below) rather than +binding them by value, for the same reason. + +The route modules are imported at the *bottom*: they import names from here, so +everything they need has to exist first. +""" +from flask import Blueprint, request, jsonify, Response +import contextlib +import json +import os +import re +import stat +import sys +import shutil +import subprocess +import tempfile +import time +import hashlib +import uuid +import logging +from datetime import datetime +from pathlib import Path +from typing import Dict, Any, Optional, Tuple, Type +from urllib.parse import urlparse, urlunparse +logger = logging.getLogger(__name__) +# Import new infrastructure +from src.web_interface.api_helpers import success_response, error_response, validate_request_json +from src.web_interface.errors import ErrorCode +from src.web_interface.secret_helpers import (find_secret_fields, mask_all_secret_values, + merge_secrets, remove_empty_secrets, + separate_secrets, + strip_masked_values) +from src.web_interface.error_handler import describe_exception, redact_text +from src.plugin_system.operation_types import OperationType +from src.web_interface.validators import ( + validate_file_upload +) +from src.error_aggregator import get_error_aggregator +from src.common.permission_utils import install_requirements_file +_SUDO = shutil.which('sudo') +_JOURNALCTL = shutil.which('journalctl') +_GIT = shutil.which('git') +# Cap subprocess output returned to the browser — pip can produce MBs on build failures. +_MAX_OUTPUT_BYTES = 51_200 # 50 KB +def _truncate_output(stdout: str, stderr: str) -> str: + """Combine stdout+stderr and truncate to _MAX_OUTPUT_BYTES (keeping the tail).""" + combined = (stdout + stderr).strip() + if len(combined) > _MAX_OUTPUT_BYTES: + combined = '[...output truncated...]\n' + combined[-_MAX_OUTPUT_BYTES:] + return combined +def _pip_install_requirements(req_file: Path, timeout: int) -> subprocess.CompletedProcess: + """Install a requirements.txt file, preferring the vetted sudo wrapper so + the packages are visible to root-run ledmatrix.service — not just to + whichever non-root user runs this web process. Falls back to installing + for the current process only if the wrapper isn't set up yet (i.e. the + admin hasn't run scripts/install/configure_web_sudo.sh since upgrading), + so the button still does *something* useful rather than hard-failing. + + Thin wrapper around the shared implementation in permission_utils so the + Plugin Store's own dependency installation (store_manager.py) follows the + exact same root-visible install path instead of a divergent one. + """ + return install_requirements_file(req_file, timeout=timeout) +def _scrub_git_remote_url(url: str) -> str: + """Strip embedded username/password from an HTTPS remote URL before returning it to the UI.""" + try: + p = urlparse(url) + if p.scheme in ('http', 'https') and (p.username or p.password): + netloc = p.hostname or '' + if p.port: + netloc += f':{p.port}' + return urlunparse(p._replace(netloc=netloc)) + except Exception: + pass + return url +# Will be initialized when blueprint is registered +# NOTE: the managers live on the blueprint object (app.py sets +# api_v3.config_manager / api_v3.plugin_manager). Deliberately not +# mirrored as module globals: a bare `config_manager` used to resolve to +# a None that was never assigned, which silently disabled the /health +# checks and made /display/current fall back to a hardcoded 128x64. +plugin_store_manager = None +saved_repositories_manager = None +cache_manager = None +schema_manager = None +operation_queue = None +plugin_state_manager = None +operation_history = None +sync_manager = None # Optional DisplaySyncManager instance (set by app.py if available) +# Get project root directory (web_interface/../..) +# web_interface/blueprints/api_v3/_common.py -> up four to the project root. +# This was three levels when everything lived in web_interface/blueprints/api_v3.py; +# the split moved the file one directory deeper and silently pointed PROJECT_ROOT +# at web_interface/ instead. Nothing failed at import -- it surfaced as routes +# 404ing and "installation script not found", because every path built from it +# was wrong. Asserted in test_api_v3_url_map.py so the next move cannot repeat it. +PROJECT_ROOT = Path(__file__).resolve().parents[3] +# System fonts that cannot be deleted (used by catalog API and delete endpoint) +SYSTEM_FONTS = frozenset([ + 'pressstart2p-regular', 'pressstart2p', + '4x6-font', '4x6', + '5by7.regular', '5by7', '5x7', + '5x8', '6x9', '6x10', '6x12', '6x13', '6x13b', '6x13o', + '7x13', '7x13b', '7x13o', '7x14', '7x14b', + '8x13', '8x13b', '8x13o', + '9x15', '9x15b', '9x18', '9x18b', + '10x20', + 'matrixchunky8', 'matrixlight6', 'tom-thumb', + 'clr6x12', 'helvr12', 'texgyre-27' +]) +api_v3 = Blueprint('api_v3', __name__) +def _get_plugin_version(plugin_id: str) -> str: + """Read the installed version from a plugin's manifest.json. + + Returns the version string on success, or '' if the manifest + cannot be read (missing, corrupt, permission denied, etc.). + """ + manifest_path = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id / "manifest.json" + try: + with open(manifest_path, 'r', encoding='utf-8') as f: + manifest = json.load(f) + return manifest.get('version', '') + except (FileNotFoundError, PermissionError, OSError) as e: + logger.warning("[PluginVersion] Could not read manifest for %s at %s: %s", plugin_id, manifest_path, e) + except json.JSONDecodeError as e: + logger.warning("[PluginVersion] Invalid JSON in manifest for %s at %s: %s", plugin_id, manifest_path, e) + return '' +def _is_plugin_update_available(installed_version: str, latest_version: str) -> bool: + """Return True when the registry's ``latest_version`` is strictly newer + than the installed version. + + Thin alias for the shared comparator in + `src.plugin_system.compatibility.is_update_available` — the store's + `update_plugin` uses the same function, so the UI badge and the actual + reinstall decision can never disagree. + """ + from src.plugin_system.compatibility import is_update_available + return is_update_available(installed_version, latest_version) +def _ensure_cache_manager(): + """Ensure cache manager is initialized.""" + global cache_manager + if cache_manager is None: + from src.cache_manager import CacheManager + cache_manager = CacheManager() + return cache_manager +def _save_config_atomic(config_manager, config_data, create_backup=True): + """ + Save configuration using atomic save if available, fallback to regular save. + + Returns: + tuple: (success: bool, error_message: str or None) + """ + if hasattr(config_manager, 'save_config_atomic'): + result = config_manager.save_config_atomic(config_data, create_backup=create_backup) + if result.status.value != 'success': + return False, result.message + return True, None + else: + try: + config_manager.save_config(config_data) + return True, None + except Exception as e: + return False, str(e) +def _coerce_to_bool(value): + """ + Coerce a form value to a proper Python boolean. + + HTML checkboxes send string values like "true", "on", "1" when checked. + This ensures we store actual booleans in config JSON, not strings. + + Args: + value: The form value (string, bool, int, or None) + + Returns: + bool: True if value represents a truthy checkbox state, False otherwise + """ + if value is None: + return False + if isinstance(value, bool): + return value + if isinstance(value, int): + return value == 1 + if isinstance(value, str): + return value.lower() in ('true', 'on', '1', 'yes') + return False +def _get_display_service_status(): + """Return status information about the ledmatrix service.""" + try: + result = subprocess.run( + ['systemctl', 'is-active', 'ledmatrix'], + capture_output=True, + text=True, + timeout=3 + ) + return { + 'active': result.stdout.strip() == 'active', + 'returncode': result.returncode, + 'stdout': result.stdout.strip(), + 'stderr': result.stderr.strip() + } + except subprocess.TimeoutExpired: + return { + 'active': False, + 'returncode': -1, + 'stdout': '', + 'stderr': 'timeout' + } + except Exception as err: + return { + 'active': False, + 'returncode': -1, + 'stdout': '', + 'stderr': str(err) + } +def _run_systemctl_command(args): + """Run a systemctl command safely.""" + try: + result = subprocess.run( + args, + capture_output=True, + text=True, + timeout=15 + ) + return { + 'returncode': result.returncode, + 'stdout': result.stdout, + 'stderr': result.stderr + } + except subprocess.TimeoutExpired: + return { + 'returncode': -1, + 'stdout': '', + 'stderr': 'timeout' + } + except Exception as err: + return { + 'returncode': -1, + 'stdout': '', + 'stderr': str(err) + } +def _ensure_display_service_running(): + """Ensure the ledmatrix display service is running.""" + status = _get_display_service_status() + if status.get('active'): + status['started'] = False + return status + result = _run_systemctl_command(['sudo', 'systemctl', 'start', 'ledmatrix.service']) + service_status = _get_display_service_status() + result['started'] = result.get('returncode') == 0 + result['active'] = service_status.get('active') + result['status'] = service_status + return result +def _stop_display_service(): + """Stop the ledmatrix display service.""" + result = _run_systemctl_command(['sudo', 'systemctl', 'stop', 'ledmatrix.service']) + status = _get_display_service_status() + result['active'] = status.get('active') + result['status'] = status + return result +#: Field names whose value is a credential. Matched by name because this +#: endpoint returns the whole config, core keys included, and core config has +#: no schema to carry x-secret markers. +_CREDENTIAL_NAME_PARTS = ("password", "passwd", "secret", "token", "api_key", + "apikey", "access_key", "private_key", "client_secret") +def _looks_like_a_credential(name: str) -> bool: + lowered = name.lower() + return any(part in lowered for part in _CREDENTIAL_NAME_PARTS) +def _redact_credentials(value): + """A copy of `value` with credential-named fields blanked. + + /config/main returned the raw config to anyone who could reach the port, + and this interface has no authentication. On one rig that meant a 40-char + GitHub token, a 183-char Home Assistant token and five API keys were + readable by anything on the LAN. + + The x-secret masking used by the plugin config endpoints does not help + here: this endpoint never consults a schema, and core keys such as + github.api_token have no schema to mark. Matching on the field name is + blunt, but for a whole-config dump the right default is that anything + named like a credential does not leave the process. + + Blanked rather than removed, and safe to blank: POST /config/main merges + into the loaded config and only writes the keys it was given, so a client + that round-trips this response cannot erase a secret it never saw. + """ + if isinstance(value, dict): + return {k: (_blank_credential_value(v) if _looks_like_a_credential(k) + else _redact_credentials(v)) + for k, v in value.items()} + if isinstance(value, list): + return [_redact_credentials(item) for item in value] + return value +def _blank_all_scalars(value): + """Blank every scalar reached from `value`, at any depth. + + Unlike `_blank_credential_value`, this never delegates back to + `_redact_credentials`'s name-based walk: an object reached through a + credential-owned list (e.g. `tokens: [{"value": "secret"}]`) has no + field name of its own to test, so every scalar inside it is blanked + regardless of what its keys are called. + """ + if isinstance(value, dict): + return {k: _blank_all_scalars(v) for k, v in value.items()} + if isinstance(value, list): + return [_blank_all_scalars(item) for item in value] + return "" +def _blank_credential_value(value): + """Blank a value that sits under a credential-shaped key. + + A dict is still walked -- `secrets: {api_key: ..., note: ...}` is a + section name, not a value to blank in one go, so sub-fields that are not + themselves credential-named survive (see + test_a_credential_shaped_container_is_still_walked). A list has no field + names to test for its items, though, so every scalar reached through it + is blanked outright: a bare list of secrets (`tokens: ["a", "b"]`) and a + list of credential-shaped objects (`tokens: [{"value": "secret"}]`) are + both blanked at any depth via `_blank_all_scalars`. + """ + if isinstance(value, dict): + return _redact_credentials(value) + if isinstance(value, list): + return [_blank_all_scalars(item) for item in value] + return "" +def _validate_time_format(time_str): + """Validate time format is HH:MM""" + try: + datetime.strptime(time_str, '%H:%M') + return True, None + except (ValueError, TypeError): + return False, f"Invalid time format: {time_str}. Expected HH:MM format." +def _git_current_branch(project_dir): + """Current branch name, or '' when detached or git fails.""" + try: + r = subprocess.run(['git', 'branch', '--show-current'], + capture_output=True, text=True, timeout=10, cwd=str(project_dir)) + return r.stdout.strip() if r.returncode == 0 else '' + except (subprocess.TimeoutExpired, OSError): + return '' +def _git_upstream(project_dir): + """Configured upstream for the current branch (e.g. 'origin/main'), or ''.""" + try: + r = subprocess.run(['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], + capture_output=True, text=True, timeout=10, cwd=str(project_dir)) + return r.stdout.strip() if r.returncode == 0 else '' + except (subprocess.TimeoutExpired, OSError): + return '' +def _git_remote_branch_exists(project_dir, branch): + """True when origin/ exists locally as a remote-tracking ref.""" + if not branch: + return False + try: + r = subprocess.run( + ['git', 'show-ref', '--verify', '--quiet', f'refs/remotes/origin/{branch}'], + capture_output=True, text=True, timeout=10, cwd=str(project_dir)) + return r.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False +def resolve_pull_command(project_dir): + """Work out how to pull, for branches with and without an upstream. + + A plain ``git pull --rebase`` fails outright on a branch that has no + upstream ("There is no tracking information for the current branch"), + which is easy to end up on: checking out a branch by name, restoring a + backup, or following an install guide that names one. The update button + then reports a failure the user cannot act on. + + ``--autostash`` is passed for the same reason. Rebase refuses to start + when any tracked file is modified, and on these installs something always + is: first_time_install.sh chmods five scripts that git tracked as 644, so + every machine that ran the installer carries five permanent mode changes + and the update button reports "cannot pull with rebase: You have unstaged + changes". Those modes are corrected in this commit, but a user cannot pull + the correction while the pull is what is blocked, and any other local edit + would reproduce it anyway. Autostash reapplies the changes afterwards. + + Returns ``(args, note, error)``. When ``origin/`` exists the pull + is made explicit against it, so the update proceeds and the branch is + given tracking information afterwards. + """ + upstream = _git_upstream(project_dir) + if upstream: + return ['git', 'pull', '--rebase', '--autostash'], '', None + + branch = _git_current_branch(project_dir) + if not branch: + return None, '', ( + "This checkout is in a detached HEAD state, so there is no branch " + "to update. Switch to a branch first (Tools -> Switch branch)." + ) + if _git_remote_branch_exists(project_dir, branch): + return ( + ['git', 'pull', '--rebase', '--autostash', 'origin', branch], + f"Branch '{branch}' had no upstream; pulled from origin/{branch} and set it as the upstream.", + None, + ) + return None, '', ( + f"Branch '{branch}' has no upstream and there is no origin/{branch} to " + f"pull from. Use Switch branch to move to a branch that exists on the " + f"remote, or push this one first." + ) +_BRANCH_NAME_RE = re.compile(r'[A-Za-z0-9._/-]{1,200}') +def is_valid_branch_name(name): + """Accept only plain branch names. + + This value becomes a subprocess argument, so anything exotic is refused + rather than escaped. '..' is excluded because it is range syntax to git. + """ + if not name or not _BRANCH_NAME_RE.fullmatch(name): + return False + return '..' not in name and not name.startswith('-') +def checkout_branch(project_dir, target, stash=False): + """Switch the checkout to `target`, returning (payload, http_status). + + Split out of the route so it can be tested against real repositories. + Attaches tracking when the branch exists on origin, so the next + Pull Latest is a plain `git pull` rather than the no-upstream fallback. + """ + target = (target or '').strip() + if not target: + return {'status': 'error', 'message': 'Branch name required'}, 400 + if not is_valid_branch_name(target): + return {'status': 'error', 'message': 'Invalid branch name'}, 400 + + try: + subprocess.run(['git', 'fetch', 'origin', '--prune'], + capture_output=True, text=True, timeout=60, cwd=project_dir) + + local_exists = subprocess.run( + ['git', 'show-ref', '--verify', '--quiet', f'refs/heads/{target}'], + capture_output=True, text=True, timeout=10, cwd=project_dir).returncode == 0 + remote_exists = _git_remote_branch_exists(project_dir, target) + if not local_exists and not remote_exists: + return {'status': 'error', + 'message': f"No branch '{target}' locally or on origin"}, 404 + + # Local edits block a checkout. Pull Latest already stashes for the + # same reason, so offer it here too -- but only when asked, never + # silently: putting someone's edits away unasked is worse than + # refusing the switch. + stash_note = '' + if stash: + stashed = subprocess.run(['git', 'stash', 'push', '-m', f'switch to {target}'], + capture_output=True, text=True, timeout=60, cwd=project_dir) + if stashed.returncode == 0 and 'No local changes' not in stashed.stdout: + stash_note = ' Local changes were stashed (recover them with git stash list).' + + if local_exists: + co = subprocess.run(['git', 'checkout', target], + capture_output=True, text=True, timeout=60, cwd=project_dir) + else: + # -B so a stale local ref does not block the checkout. + co = subprocess.run(['git', 'checkout', '-B', target, f'origin/{target}'], + capture_output=True, text=True, timeout=60, cwd=project_dir) + + if co.returncode != 0: + logger.warning("git checkout %s failed: %s", target, co.stderr) + return { + 'status': 'error', + 'message': f"Could not switch to '{target}'.", + # Keep git's full list of blocking files: naming them is the + # difference between an error the user can act on and one they + # cannot. + 'detail': (co.stderr or '').strip(), + 'can_retry_with_stash': 'would be overwritten by checkout' in (co.stderr or ''), + }, 200 + + if remote_exists: + subprocess.run(['git', 'branch', f'--set-upstream-to=origin/{target}', target], + capture_output=True, text=True, timeout=10, cwd=project_dir) + + logger.info("Switched checkout to branch %s", target) + return { + 'status': 'success', + 'message': f"Now on '{target}'.{stash_note} Use Pull Latest to fetch its newest code.", + }, 200 + except subprocess.TimeoutExpired: + return {'status': 'error', 'message': 'Timed out talking to git'}, 504 + except OSError as exc: + logger.error("checkout_branch failed: %s", exc, exc_info=True) + return {'status': 'error', 'message': 'Could not switch branch'}, 500 +def get_git_version(project_dir=None): + """Get git version information from the repository""" + if project_dir is None: + project_dir = PROJECT_ROOT + + try: + # Try to get tag description (e.g., v2.4-10-g123456) + result = subprocess.run( + ['git', 'describe', '--tags', '--dirty'], + capture_output=True, + text=True, + timeout=5, + cwd=str(project_dir) + ) + + if result.returncode == 0: + version_str = result.stdout.strip() + if re.match(r'^[a-zA-Z0-9._\-]+$', version_str): + return version_str + + # Fallback to short commit hash + result = subprocess.run( + ['git', 'rev-parse', '--short', 'HEAD'], + capture_output=True, + text=True, + timeout=5, + cwd=str(project_dir) + ) + + if result.returncode == 0: + version_str = result.stdout.strip() + if re.match(r'^[a-zA-Z0-9._\-]+$', version_str): + return version_str + + return 'Unknown' + except Exception: + return 'Unknown' +_update_check_cache: Dict[str, Any] = {'result': None, 'ts': 0.0} +_UPDATE_CHECK_TTL = 300 # 5 minutes — avoids a git fetch on every page load +def _update_check_failed(detail: str) -> Dict[str, Any]: + """A check that could not run is not the same as being up to date. + + Reporting update_available=False on a git failure hides the banner, and + the banner is the only route to the update button -- so a checkout git + refuses to touch looks exactly like a current one, permanently. The most + common cause is an install performed as root: git then reports "dubious + ownership" and every command fails, including the fetch here. + """ + return {'update_available': False, 'remote_sha': 'unknown', + 'commits_behind': 0, 'check_failed': True, 'error': detail} +def _describe_git_failure(stderr: str) -> str: + """Turn git's stderr into something the user can act on.""" + text = (stderr or '').strip() + if 'dubious ownership' in text or 'detected dubious ownership' in text: + return ("This checkout is owned by a different user than the one " + "running the web interface, so git refuses to use it. It is " + "usually the result of installing as root. Fix the ownership " + "and the update will work: sudo chown -R $USER:$USER " + + str(PROJECT_ROOT)) + if 'could not resolve host' in text.lower() or 'network is unreachable' in text.lower(): + return "Could not reach GitHub to check for updates." + return "Could not check for updates: " + (text.splitlines()[0] if text else "git failed") +def _installed_plugin_ids(): + """Best-effort list of installed plugin IDs for the web process. + + Health/metrics state is written by the separate display service to the + shared on-disk cache, so the tracker's in-memory set is empty here. We + enumerate the installed plugins and read each one's persisted summary by ID + instead of relying on the tracker's in-memory `get_all_*` view. + """ + pm = api_v3.plugin_manager + manifests = getattr(pm, 'plugin_manifests', None) + if not manifests: + # Only pay for a discovery scan when we haven't discovered anything yet; + # subsequent polls reuse the already-populated manifest map. + try: + pm.discover_plugins() + except Exception: + logger.debug('discover_plugins failed while listing plugin ids', exc_info=True) + manifests = getattr(pm, 'plugin_manifests', None) + try: + return list(manifests.keys()) if manifests else [] + except Exception: + logger.debug('listing plugin_manifests failed while building plugin ids', exc_info=True) + return [] +def _do_transactional_uninstall(plugin_id, preserve_config): + """Execute an uninstall with snapshot-based rollback. + + Order of operations: + 1. Snapshot main config + secrets (abort on unexpected errors, proceed on expected I/O errors). + 2. Clean up plugin config (abort with 500 if this raises — avoids orphaned files). + 3. Unload plugin from runtime if loaded (rollback + 500 if this raises). + 4. Remove plugin files (rollback + 500 if this returns False or raises). + 5. Finish (remove state, invalidate caches). + + Rollback restores the config snapshot and, if the plugin had been + loaded before unload, calls load_plugin to restore runtime state. + + Returns (True, None) on success or (False, error_message) on failure. + """ + from src.exceptions import ConfigError + + # --- Step 1: snapshot main + secrets --- + main_snapshot = None + secrets_snapshot = None + try: + main_snapshot = api_v3.config_manager.get_raw_file_content('main') + except (OSError, ConfigError): + pass # Proceed without snapshot; narrow catch preserves TypeError/AttributeError + try: + secrets_snapshot = api_v3.config_manager.get_raw_file_content('secrets') + except (OSError, ConfigError): + pass + + # --- Step 2: cleanup config first (abort before touching filesystem) --- + if not preserve_config: + api_v3.config_manager.cleanup_plugin_config(plugin_id, remove_secrets=True) + + # Record whether the plugin was running before we touch anything. + was_loaded = ( + api_v3.plugin_manager is not None + and plugin_id in api_v3.plugin_manager.plugins + ) + + def _rollback(reload_plugin): + if main_snapshot is not None: + try: + api_v3.config_manager.save_raw_file_content('main', main_snapshot) + except Exception as restore_err: + logger.error("Failed to restore main config snapshot for %s: %s", plugin_id, restore_err) + if secrets_snapshot is not None: + try: + api_v3.config_manager.save_raw_file_content('secrets', secrets_snapshot) + except Exception as restore_err: + logger.error("Failed to restore secrets snapshot for %s: %s", plugin_id, restore_err) + if reload_plugin and api_v3.plugin_manager is not None: + try: + api_v3.plugin_manager.load_plugin(plugin_id) + except Exception as reload_err: + logger.error("Failed to reload plugin %s during rollback: %s", plugin_id, reload_err) + + # --- Step 3: unload --- + if was_loaded: + try: + api_v3.plugin_manager.unload_plugin(plugin_id) + except Exception as unload_err: + _rollback(reload_plugin=False) # unload failed — runtime state unchanged + return False, f"Failed to unload plugin {plugin_id}: {unload_err}" + + # --- Step 4: remove files --- + try: + success = api_v3.plugin_store_manager.uninstall_plugin(plugin_id) + except Exception as remove_err: + _rollback(reload_plugin=was_loaded) + return False, f"Failed to remove plugin {plugin_id}: {remove_err}" + + if not success: + _rollback(reload_plugin=was_loaded) + return False, f"Failed to uninstall plugin {plugin_id}" + + # --- Step 5: finish --- + if api_v3.schema_manager: + api_v3.schema_manager.invalidate_cache(plugin_id) + if api_v3.plugin_state_manager: + api_v3.plugin_state_manager.remove_plugin_state(plugin_id) + # Persistently record the uninstall so a later core `git pull` update + # cannot resurrect a built-in plugin (committed under plugin-repos/) that + # the user removed. Best-effort: never fail the uninstall over this. + try: + api_v3.plugin_store_manager.record_uninstalled_plugin(plugin_id) + except Exception as record_err: + logger.warning("Could not record uninstall for %s: %s", plugin_id, record_err) + return True, None +def deep_merge(base_dict, update_dict): + """ + Deep merge update_dict into base_dict. + For nested dicts, recursively merge. For other types, update_dict takes precedence. + + Lists are intentionally REPLACED wholesale, never index-merged: form posts + carry complete arrays, and index-merging would resurrect items the user + deleted. This also applies to the parallel secrets lists produced by + separate_secrets — a newly saved secrets list is authoritative. + """ + result = base_dict.copy() + for key, value in update_dict.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + # Recursively merge nested dicts + result[key] = deep_merge(result[key], value) + else: + # For non-dict values or new keys, use the update value + result[key] = value + return result +def _parse_form_value(value): + """ + Parse a form value into the appropriate Python type. + Handles booleans, numbers, JSON arrays/objects, and strings. + """ + import json + + if value is None: + return None + + # Handle string values + if isinstance(value, str): + stripped = value.strip() + + # Check for boolean strings + if stripped.lower() == 'true': + return True + if stripped.lower() == 'false': + return False + if stripped.lower() in ('null', 'none') or stripped == '': + return None + + # Try parsing as JSON (for arrays and objects) - do this BEFORE number parsing + # This handles RGB arrays like "[255, 0, 0]" correctly + if stripped.startswith('[') or stripped.startswith('{'): + try: + return json.loads(stripped) + except json.JSONDecodeError: + pass + + # Try parsing as number + try: + if '.' in stripped: + return float(stripped) + return int(stripped) + except ValueError: + pass + + # Return as string (original value, not stripped) + return value + + return value +def _get_schema_property(schema, key_path): + """ + Get the schema property for a given key path (supports dot notation). + + Args: + schema: The JSON schema dict + key_path: Dot-separated path like "customization.time_text.font" + + Returns: + The property schema dict or None if not found + """ + if not schema or 'properties' not in schema: + return None + + parts = key_path.split('.') + current = schema['properties'] + i = 0 + + while i < len(parts): + # Try progressively longer candidates, longest first, so schema keys that + # themselves contain dots (e.g. league keys like "fifa.world") are matched + # instead of being mistaken for nested "fifa" -> "world" objects. + matched = False + for j in range(len(parts), i, -1): + candidate = '.'.join(parts[i:j]) + if isinstance(current, dict) and candidate in current: + prop = current[candidate] + # Consumed all remaining parts — this is the target property. + if j == len(parts): + return prop + # Navigate deeper through an object with properties. + if isinstance(prop, dict) and 'properties' in prop: + current = prop['properties'] + i = j + matched = True + break + # Matched a non-object before consuming the path — can't go deeper. + return None + if not matched: + return None + + return None +def _is_field_required(key_path, schema): + """ + Check if a field is required according to the schema. + + Args: + key_path: Dot-separated path like "mqtt.username" + schema: The JSON schema dict + + Returns: + True if field is required, False otherwise + """ + if not schema or 'properties' not in schema: + return False + + parts = key_path.split('.') + if len(parts) == 1: + # Top-level field + required = schema.get('required', []) + return parts[0] in required + else: + # Nested field - navigate to parent object + parent_path = '.'.join(parts[:-1]) + field_name = parts[-1] + + # Get parent property + parent_prop = _get_schema_property(schema, parent_path) + if not parent_prop or 'properties' not in parent_prop: + return False + + # Check if field is required in parent + required = parent_prop.get('required', []) + return field_name in required +# Sentinel object to indicate a field should be skipped (not set in config) +_SKIP_FIELD = object() +def _parse_form_value_with_schema(value, key_path, schema): + """ + Parse a form value using schema information to determine correct type. + Handles arrays (comma-separated strings), objects, and other types. + + Args: + value: The form value (usually a string) + key_path: Dot-separated path like "category_order" or "customization.time_text.font" + schema: The plugin's JSON schema + + Returns: + Parsed value with correct type, or _SKIP_FIELD to indicate the field should not be set + """ + import json + + # Get the schema property for this field + prop = _get_schema_property(schema, key_path) + + # Handle None/empty values + if value is None or (isinstance(value, str) and value.strip() == ''): + # If schema says it's an array, return empty array instead of None + if prop and prop.get('type') == 'array': + return [] + # If schema says it's an object, return empty dict instead of None + if prop and prop.get('type') == 'object': + return {} + # If it's an optional string field, preserve empty string instead of None + if prop and prop.get('type') == 'string': + if not _is_field_required(key_path, schema): + return "" # Return empty string for optional string fields + # For number/integer fields, check if they have defaults or are required + if prop: + prop_type = prop.get('type') + if prop_type in ('number', 'integer'): + # If field has a default, use it + if 'default' in prop: + return prop['default'] + # If field is not required and has no default, skip setting it + if not _is_field_required(key_path, schema): + return _SKIP_FIELD + # If field is required but empty, return None (validation will fail, which is correct) + return None + return None + + # Handle string values + if isinstance(value, str): + stripped = value.strip() + + # Check for boolean strings + if stripped.lower() == 'true': + return True + if stripped.lower() == 'false': + return False + # "on"/"off" come from HTML checkboxes — only coerce when schema says boolean + if prop and prop.get('type') == 'boolean': + if stripped.lower() == 'on': + return True + if stripped.lower() == 'off': + return False + + # Handle arrays based on schema + if prop and prop.get('type') == 'array': + # Try parsing as JSON first (handles "[1,2,3]" format) + if stripped.startswith('['): + try: + return json.loads(stripped) + except json.JSONDecodeError: + pass + + # Otherwise, treat as comma-separated string + if stripped: + # Split by comma and strip each item + items = [item.strip() for item in stripped.split(',') if item.strip()] + # Try to convert items to numbers if schema items are numbers + items_schema = prop.get('items', {}) + if items_schema.get('type') in ('number', 'integer'): + try: + return [int(item) if '.' not in item else float(item) for item in items] + except ValueError: + pass + return items + return [] + + # Handle objects based on schema + if prop and prop.get('type') == 'object': + # Try parsing as JSON + if stripped.startswith('{'): + try: + return json.loads(stripped) + except json.JSONDecodeError: + pass + # If it's not JSON, return empty dict (form shouldn't send objects as strings) + return {} + + # Try parsing as JSON (for arrays and objects) - do this BEFORE number parsing + if stripped.startswith('[') or stripped.startswith('{'): + try: + return json.loads(stripped) + except json.JSONDecodeError: + pass + + # Handle numbers based on schema + if prop: + prop_type = prop.get('type') + if prop_type == 'integer': + try: + return int(stripped) + except ValueError: + return prop.get('default', 0) + elif prop_type == 'number': + try: + return float(stripped) + except ValueError: + return prop.get('default', 0.0) + + # Try parsing as number (fallback) — skip when schema explicitly says string + if not (prop and prop.get('type') == 'string'): + try: + if '.' in stripped: + return float(stripped) + return int(stripped) + except ValueError: + pass + + # Return as string + return value + + return value +def _resolve_key_segments(key_path, config): + """Split a dot-notation path into segments, greedily preserving keys that + themselves contain dots (e.g. league keys like "fifa.world"). + + At each level the longest candidate that matches a key already present in the + config wins; otherwise the path splits on the next dot (the normal + nested-create case). Because dotted keys such as ``leagues."fifa.world"`` + always exist in the saved config being updated, this routes the value to the + real league object instead of fabricating a ``leagues.fifa.world`` tree. + """ + parts = key_path.split('.') + segments = [] + node = config + i = 0 + while i < len(parts): + matched = False + if isinstance(node, dict): + for j in range(len(parts), i, -1): + candidate = '.'.join(parts[i:j]) + if candidate in node: + segments.append(candidate) + node = node[candidate] + i = j + matched = True + break + if not matched: + part = parts[i] + segments.append(part) + node = node.get(part) if isinstance(node, dict) else None + i += 1 + return segments +def _set_nested_value(config, key_path, value): + """ + Set a value in a nested dict using dot notation path. + Handles existing nested dicts correctly by merging instead of replacing. + Keys containing dots (e.g. league keys like "fifa.world") are preserved when + they already exist in the config rather than being split into nested objects. + + Args: + config: The config dict to modify + key_path: Dot-separated path (e.g., "customization.period_text.font") + value: The value to set (or _SKIP_FIELD to skip setting) + """ + # Skip setting if value is the sentinel + if value is _SKIP_FIELD: + return + + segments = _resolve_key_segments(key_path, config) + current = config + + # Navigate/create intermediate dicts + for seg in segments[:-1]: + if seg not in current: + current[seg] = {} + elif not isinstance(current[seg], dict): + # If the existing value is not a dict, replace it with a dict + current[seg] = {} + current = current[seg] + + # Set the final value (don't overwrite with empty dict if value is None and we want to preserve structure) + if value is not None or segments[-1] not in current: + current[segments[-1]] = value +def _set_missing_booleans_to_false(config, schema_props, form_keys, prefix='', config_node=None): + """Walk schema and set missing boolean form fields to False. + + HTML checkboxes don't submit values when unchecked. When saving plugin config, + the backend starts from existing config (to support partial form updates), which + means an unchecked checkbox's old ``True`` value persists. This function detects + boolean schema properties not present in the form submission and explicitly sets + them to ``False``. + + The top-level ``enabled`` field is excluded because it has its own preservation + logic in the save endpoint. + + Handles boolean fields inside nested objects and inside arrays of objects + (e.g. ``feeds.custom_feeds.0.enabled``). + + Args: + config: The root plugin config dict (used for pure-dict paths) + schema_props: Schema ``properties`` dict at the current nesting level + form_keys: Set of form field names that were submitted + prefix: Dot-notation prefix for the current nesting level + config_node: The current config subtree when inside an array item (avoids + using _set_nested_value which corrupts lists) + """ + # Determine which config node to operate on + node = config_node if config_node is not None else config + + for prop_name, prop_schema in schema_props.items(): + if not isinstance(prop_schema, dict): + continue + + full_path = f"{prefix}.{prop_name}" if prefix else prop_name + prop_type = prop_schema.get('type') + + if prop_type == 'boolean' and full_path != 'enabled': + # If this boolean wasn't submitted in the form, it's an unchecked checkbox + if full_path not in form_keys: + if config_node is not None: + # Inside an array item — set directly on the item dict + node[prop_name] = False + else: + # Pure dict path — use helper + _set_nested_value(config, full_path, False) + + elif prop_type == 'object' and 'properties' in prop_schema: + # Recurse into nested objects + if config_node is not None: + # Inside an array item — ensure nested dict exists in item + if prop_name not in node or not isinstance(node[prop_name], dict): + node[prop_name] = {} + _set_missing_booleans_to_false( + config, prop_schema['properties'], form_keys, full_path, + config_node=node[prop_name] + ) + else: + _set_missing_booleans_to_false( + config, prop_schema['properties'], form_keys, full_path + ) + + elif prop_type == 'array': + # Handle arrays of objects that may contain boolean fields + # Form keys use indexed notation: "path.0.field", "path.1.field" + items_schema = prop_schema.get('items', {}) + if isinstance(items_schema, dict) and items_schema.get('type') == 'object' and 'properties' in items_schema: + array_prefix = f"{full_path}." + # Collect unique item indices from submitted form keys + indices = set() + for k in form_keys: + if k.startswith(array_prefix): + # Extract index: "path.0.field" -> "0" + rest = k[len(array_prefix):] + idx = rest.split('.', 1)[0] + if idx.isdigit(): + indices.add(int(idx)) + + if not indices: + continue + + # Navigate to the array in the config (create if missing) + if config_node is not None: + if prop_name not in node or not isinstance(node[prop_name], list): + node[prop_name] = [] + array_list = node[prop_name] + else: + # Navigate from root config through dict keys to get the list + parts = full_path.split('.') + current = config + for part in parts[:-1]: + if part not in current or not isinstance(current[part], dict): + current[part] = {} + current = current[part] + arr_key = parts[-1] + if arr_key not in current or not isinstance(current[arr_key], list): + current[arr_key] = [] + array_list = current[arr_key] + + # Recurse into each array item so its missing booleans get set to False + for idx in indices: + # Ensure list is long enough and item is a dict + while len(array_list) <= idx: + array_list.append({}) + if not isinstance(array_list[idx], dict): + array_list[idx] = {} + item_prefix = f"{full_path}.{idx}" + _set_missing_booleans_to_false( + config, items_schema['properties'], form_keys, item_prefix, + config_node=array_list[idx] + ) +def _enhance_schema_with_core_properties(schema): + """ + Enhance schema with core plugin properties (enabled, display_duration, live_priority). + These properties are system-managed and should always be allowed even if not in the plugin's schema. + + Args: + schema: The original JSON schema dict + + Returns: + Enhanced schema dict with core properties injected + """ + import copy + + if not schema: + return schema + + # Core plugin properties that should always be allowed + # These match the definitions in SchemaManager.validate_config_against_schema() + core_properties = { + "enabled": { + "type": "boolean", + "default": True, + "description": "Enable or disable this plugin" + }, + "display_duration": { + "type": "number", + "default": 15, + "minimum": 1, + "maximum": 300, + "description": "How long to display this plugin in seconds" + }, + "live_priority": { + "type": "boolean", + "default": False, + "description": "Enable live priority takeover when plugin has live content" + } + } + + # Create a deep copy of the schema to modify (to avoid mutating the original) + enhanced_schema = copy.deepcopy(schema) + if "properties" not in enhanced_schema: + enhanced_schema["properties"] = {} + + # Inject core properties if they're not already defined in the schema + for prop_name, prop_def in core_properties.items(): + if prop_name not in enhanced_schema["properties"]: + enhanced_schema["properties"][prop_name] = copy.deepcopy(prop_def) + + return enhanced_schema +def _filter_config_by_schema(config, schema, prefix=''): + """ + Filter config to only include fields defined in the schema. + Removes fields not in schema, especially important when additionalProperties is false. + + Args: + config: The config dict to filter + schema: The JSON schema dict + prefix: Prefix for nested paths (used recursively) + + Returns: + Filtered config dict containing only schema-defined fields + """ + if not schema or 'properties' not in schema: + return config + + filtered = {} + schema_props = schema.get('properties', {}) + + for key, value in config.items(): + if key not in schema_props: + # Field not in schema, skip it + continue + + prop_schema = schema_props[key] + + # Handle nested objects recursively + if isinstance(value, dict) and prop_schema.get('type') == 'object' and 'properties' in prop_schema: + filtered[key] = _filter_config_by_schema(value, prop_schema, f"{prefix}.{key}" if prefix else key) + else: + # Keep the value as-is for non-object types + filtered[key] = value + + return filtered +_MAX_CREDENTIAL_BACKUPS = 5 +def _prune_credential_backups(plugin_dir: Path) -> None: + """Keep only the newest _MAX_CREDENTIAL_BACKUPS credential backups. + + Every re-upload copies the previous credentials.json aside. Without + pruning those accumulate for the life of the install — each one a + complete set of OAuth client credentials sitting in the plugin + directory. + """ + backups = sorted( + plugin_dir.glob('credentials.json.backup.*'), + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + for stale in backups[_MAX_CREDENTIAL_BACKUPS:]: + try: + stale.unlink() + except OSError: + logger.warning("Could not remove old credential backup %s", stale.name) +# calendarList.list pages at 250 entries maximum. Ten pages is far past any +# real account and exists only so a malformed nextPageToken cannot spin here. +_CALENDAR_LIST_MAX_PAGES = 10 +def _calendar_plugin_dir() -> Optional[Path]: + """Where the calendar plugin is installed, or None if it is not.""" + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory('calendar') + else: + plugin_dir = PROJECT_ROOT / 'plugins' / 'calendar' + if not plugin_dir: + return None + plugin_dir = Path(plugin_dir) + return plugin_dir if plugin_dir.exists() else None +def _run_calendar_registration(plugin_dir: Path, stdin_payload: str): + """Run the plugin's OAuth script and return the JSON object it prints. + + The script decides between web and terminal mode by whether stdin is a + tty, so it must be given a pipe. It emits one JSON object on stdout; the + last parsable line is taken, because an import warning or a library's + stderr redirection can land in front of it. + + Returns (payload, error_message). Exactly one is None. + """ + script = plugin_dir / 'calendar_registration.py' + if not script.exists(): + return None, 'Authentication script not found in the calendar plugin' + + try: + result = subprocess.run( # nosec B603 - fixed script path inside the plugin dir + [sys.executable, str(script)], + input=stdin_payload, + capture_output=True, + text=True, + timeout=120, + cwd=str(plugin_dir), + ) + except subprocess.TimeoutExpired: + return None, 'Authentication timed out after 120s' + except OSError as e: + logger.error('Could not run calendar_registration.py', exc_info=True) + return None, 'Could not run the authentication script: %s' % describe_exception(e) + + for line in reversed((result.stdout or '').splitlines()): + line = line.strip() + if not line: + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + return payload, None + + raw = (result.stderr or result.stdout or '').strip() + # Redacted in the log too: this is a script that handles OAuth client + # secrets, and its stderr can quote them verbatim (CWE-532). + if raw: + logger.error('calendar_registration.py failed (exit %s): %s', + result.returncode, redact_text(raw)) + return None, 'Authentication script produced no result%s' % ( + ': %s' % redact_text(raw) if raw else '') +def _resolve_backup_export_dir() -> Path: + """Where exported backups live: beside the install, not inside it. + + They used to be written to ``/config/backups/exports``. That is + inside the directory a reinstall deletes, so the documented recovery path + -- export a backup, then reinstall -- destroyed the backup it had just + told the user to make. Anyone who downloaded the ZIP was fine; anyone + relying on the on-device copy was not. + + Falls back to the old location when the parent directory is not writable, + so an unusual layout degrades to previous behaviour instead of failing to + export at all. + """ + preferred = PROJECT_ROOT.parent / "ledmatrix-backups" + fallback = PROJECT_ROOT / "config" / "backups" / "exports" + try: + preferred.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=preferred, prefix=".writetest-"): + pass + return preferred + except OSError as e: + logger.warning( + f"[Backup] Export dir {preferred} is not writable ({e}); " + f"falling back to {fallback}, which a reinstall will delete" + ) + return fallback +_BACKUP_EXPORT_DIR = _resolve_backup_export_dir() +def _safe_backup_path(filename: str) -> Path: + """Resolve a filename to an absolute path inside the export dir, + rejecting any traversal attempts. Returns None if unsafe.""" + # Use basename first (CodeQL-recognized sanitizer) then validate format + filename = os.path.basename(filename or '') + if not filename or not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9._-]{0,200}\.zip$', filename): + return None + path = (_BACKUP_EXPORT_DIR / filename).resolve() + try: + path.relative_to(_BACKUP_EXPORT_DIR.resolve()) + except ValueError: + return None + return path +_STARLARK_APPS_DIR = PROJECT_ROOT / 'starlark-apps' +_STARLARK_MANIFEST_FILE = _STARLARK_APPS_DIR / 'manifest.json' +# A dedicated, never-replaced file to flock -- see _starlark_manifest_lock. +_STARLARK_MANIFEST_LOCK_FILE = _STARLARK_APPS_DIR / 'manifest.json.lock' +def _get_starlark_plugin() -> Optional[Any]: + """Get the starlark-apps plugin instance, or None.""" + if not api_v3.plugin_manager: + return None + return api_v3.plugin_manager.get_plugin('starlark-apps') +def _find_pixlet_binary(explicit_path: Optional[str] = None) -> Optional[str]: + """Find pixlet binary: explicit path → bundled binary → system PATH.""" + import platform + if explicit_path and os.path.isfile(explicit_path) and os.access(explicit_path, os.X_OK): + return explicit_path + bin_dir = PROJECT_ROOT / "bin" / "pixlet" + system = platform.system().lower() + machine = platform.machine().lower() + if system == "linux": + if "aarch64" in machine or "arm64" in machine: + name = "pixlet-linux-arm64" + elif "x86_64" in machine or "amd64" in machine: + name = "pixlet-linux-amd64" + else: + name = None + elif system == "darwin": + name = "pixlet-darwin-arm64" if "arm64" in machine else "pixlet-darwin-amd64" + else: + name = None + if name: + bundled = bin_dir / name + if bundled.is_file(): + if os.access(str(bundled), os.X_OK): + return str(bundled) + try: + bundled.chmod(0o755) + except OSError: + logger.warning("Could not make pixlet bundled binary executable (%s); falling back to PATH", bundled) + else: + if os.access(str(bundled), os.X_OK): + return str(bundled) + logger.warning("Pixlet bundled binary still not executable after chmod (%s); falling back to PATH", bundled) + return shutil.which("pixlet") +@contextlib.contextmanager +def _starlark_manifest_lock(): + """Hold an exclusive lock across a standalone starlark-manifest read-modify-write. + + StarlarkAppsPlugin._update_manifest_safe (plugin-repos/starlark-apps/manager.py) + already does this -- fcntl.flock held for the whole read-modify-write cycle -- + when the plugin instance is loaded. These routes fall back to reading and + writing manifest.json directly when it is not, and did so with no lock: each + _write_starlark_manifest() call is atomic on its own (temp file + rename), but + two concurrent requests can each read the manifest, mutate their own copy, and + write it back, and the second write silently discards the first's change. + + Locks _STARLARK_MANIFEST_LOCK_FILE, a sidecar that is never written to or + renamed over -- not manifest.json itself. manifest.json is replaced by an + atomic rename on every write (here and in the plugin), which swaps in a + fresh inode; a lock held on the old inode does not exclude a second locker + that opens the path afresh right after the rename and gets the new inode, + so two writers could still race each other despite both "holding a lock". + A stable sidecar path always resolves to the same inode, so every locker + -- standalone or plugin-owned -- contends for the same lock. The plugin + must lock this same sidecar file for that guarantee to hold across both. + + Callers should do their read, mutation and _write_starlark_manifest() call + entirely inside the `with` block, mirroring the plugin's lock scope. + """ + # Imported here, not at module scope: fcntl is POSIX-only, and a top-level + # import made the whole api_v3 package unimportable on Windows -- which the + # monolithic blueprint never was, so the split would have broken local dev + # and the test suite there. Deliberately NOT degraded to a no-op lock off + # POSIX: the docstring above describes a real lost-update race, and silently + # not locking would be worse than failing loudly on a platform that cannot + # run the display anyway. + import fcntl + + _STARLARK_APPS_DIR.mkdir(parents=True, exist_ok=True) + lock_fd = os.open(str(_STARLARK_MANIFEST_LOCK_FILE), os.O_CREAT | os.O_RDWR, 0o644) + try: + fcntl.flock(lock_fd, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + finally: + os.close(lock_fd) +def _read_starlark_manifest() -> Dict[str, Any]: + """Read the starlark-apps manifest.json directly from disk.""" + try: + if _STARLARK_MANIFEST_FILE.exists(): + with open(_STARLARK_MANIFEST_FILE, 'r') as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + logger.error(f"Error reading starlark manifest: {e}") + return {'apps': {}} +def _starlark_github_token() -> Optional[str]: + """The GitHub token the Starlark store should authenticate with. + + These routes used to read `github_token` off config.json, a key that is + written nowhere and offered by no setting -- so the store always ran + unauthenticated at 60 requests/hour, on the same per-IP budget every + plugin update check spends, while the token the user had actually + configured sat in config_secrets.json raising the same budget to 5000. + The store going blank was that budget running out. + + Prefer the store manager's token, which is the one the settings UI + writes and validates; keep the config.json key as a fallback so a + hand-edited config still works. + """ + token = getattr(api_v3.plugin_store_manager, 'github_token', None) + if token: + return token + + try: + config = api_v3.config_manager.load_config() if api_v3.config_manager else {} + return config.get('github_token') + except Exception: + logger.warning("[Starlark] Could not read config for a GitHub token", exc_info=True) + return None +def _get_tronbyte_repository_class() -> Type[Any]: + """Import TronbyteRepository from plugin-repos directory.""" + import importlib.util + import importlib + + module_path = PROJECT_ROOT / 'plugin-repos' / 'starlark-apps' / 'tronbyte_repository.py' + if not module_path.exists(): + raise ImportError(f"TronbyteRepository module not found at {module_path}") + + # If already imported, return cached class + if "tronbyte_repository" in sys.modules: + return sys.modules["tronbyte_repository"].TronbyteRepository + + spec = importlib.util.spec_from_file_location("tronbyte_repository", str(module_path)) + if spec is None: + raise ImportError(f"Failed to create module spec for tronbyte_repository at {module_path}") + + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError("Failed to create module from spec for tronbyte_repository") + + sys.modules["tronbyte_repository"] = module + try: + spec.loader.exec_module(module) + except BaseException: + # A module that failed to execute must not stay in sys.modules: the + # cache branch above would hand back the half-initialised object for + # the rest of the process, so one transient failure would disable + # this path permanently and surface as AttributeError, not ImportError. + sys.modules.pop("tronbyte_repository", None) + raise + return module.TronbyteRepository +def _get_pixlet_renderer_class() -> Type[Any]: + """Import PixletRenderer from plugin-repos directory.""" + import importlib.util + import importlib + + module_path = PROJECT_ROOT / 'plugin-repos' / 'starlark-apps' / 'pixlet_renderer.py' + if not module_path.exists(): + raise ImportError(f"PixletRenderer module not found at {module_path}") + + # If already imported, return cached class + if "pixlet_renderer" in sys.modules: + return sys.modules["pixlet_renderer"].PixletRenderer + + spec = importlib.util.spec_from_file_location("pixlet_renderer", str(module_path)) + if spec is None: + raise ImportError(f"Failed to create module spec for pixlet_renderer at {module_path}") + + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError("Failed to create module from spec for pixlet_renderer") + + sys.modules["pixlet_renderer"] = module + try: + spec.loader.exec_module(module) + except BaseException: + # A module that failed to execute must not stay in sys.modules: the + # cache branch above would hand back the half-initialised object for + # the rest of the process, so one transient failure would disable + # this path permanently and surface as AttributeError, not ImportError. + sys.modules.pop("pixlet_renderer", None) + raise + return module.PixletRenderer +def _validate_and_sanitize_app_id(app_id: Optional[str], fallback_source: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]: + """Validate and sanitize app_id to a safe slug.""" + if not app_id and fallback_source: + app_id = fallback_source + if not app_id: + return None, "app_id is required" + if '..' in app_id or '/' in app_id or '\\' in app_id: + return None, "app_id contains invalid characters" + + sanitized = re.sub(r'[^a-z0-9_]', '_', app_id.lower()).strip('_') + if not sanitized: + sanitized = f"app_{hashlib.sha256(app_id.encode()).hexdigest()[:12]}" + if sanitized[0].isdigit(): + sanitized = f"app_{sanitized}" + return sanitized, None +def _validate_timing_value(value: Any, field_name: str, min_val: int = 1, max_val: int = 86400) -> Tuple[Optional[int], Optional[str]]: + """Validate and coerce timing values.""" + if value is None: + return None, None + try: + int_value = int(value) + except (ValueError, TypeError): + return None, f"{field_name} must be an integer" + if int_value < min_val: + return None, f"{field_name} must be at least {min_val}" + if int_value > max_val: + return None, f"{field_name} must be at most {max_val}" + return int_value, None +def _validate_starlark_app_path(app_id: str) -> Tuple[Optional[Path], Optional[str]]: + """The app's directory, or an error if app_id could escape the base dir. + + Returns the *resolved* path rather than a boolean, and every caller uses + what it returns instead of re-joining ``_STARLARK_APPS_DIR / app_id`` + afterwards. The old shape validated in one place and rebuilt the path in + another, which is two things that have to stay in step -- and is why + CodeQL reported twenty-four path-injection alerts across these handlers + even though the guard was effective: a boolean is not a sanitiser it can + follow, and the value reaching the filesystem was the raw one. + + The name is unchanged so the call sites read the same. + """ + if not isinstance(app_id, str) or not app_id: + return None, "Invalid app_id" + + # Reject the traversal characters outright before touching the filesystem. + if '..' in app_id or '/' in app_id or '\\' in app_id: + return None, "Invalid app_id: contains path traversal characters" + + # os.path.basename strips any directory component, so what is joined below + # cannot carry one. The equality check means this rejects rather than + # silently truncates -- behaviour is identical to the character test above, + # and it is the sanitiser CodeQL's path-injection query actually follows. + # relative_to() alone is a check it cannot trace, which is why twenty-four + # of these stayed flagged after the value was threaded through properly. + safe_name = os.path.basename(app_id) + if safe_name != app_id or safe_name in ('', '.', '..'): + return None, "Invalid app_id: contains path traversal characters" + + try: + base_path = _STARLARK_APPS_DIR.resolve() + app_path = (base_path / safe_name).resolve() + try: + app_path.relative_to(base_path) + except ValueError: + return None, "Invalid app_id: path traversal attempt" + return app_path, None + except OSError as e: + logger.warning("Path validation error for app_id %r: %s", app_id, e) + return None, "Invalid app_id" +def _standalone_render_starlark_app(app_id: str) -> Tuple[bool, int, Optional[str]]: + """Render a Starlark app via pixlet directly (no plugin required). + + Reads the .star file and config from starlark-apps/{app_id}/, runs pixlet, + and saves the output to cached_render.webp in the same directory. + This is the web-service fallback when starlark-apps plugin is not loaded. + + Returns (success, http_status_code, error_message). + """ + manifest = _read_starlark_manifest() + if not isinstance(manifest, dict): + return False, 400, "Invalid manifest shape: expected object with 'apps' mapping" + apps = manifest.get('apps', {}) + if not isinstance(apps, dict): + return False, 400, "Invalid manifest shape: expected object with 'apps' mapping" + app_data = apps.get(app_id) + if not app_data: + return False, 404, f"App not found: {app_id}" + + # Validated here as well as at the handler: this is reachable on its own, + # and a path built from app_id should never be assembled without it. + app_dir, path_error = _validate_starlark_app_path(app_id) + if path_error: + return False, 400, path_error + star_file = app_dir / app_data.get('star_file', f'{app_id}.star') + if not star_file.exists(): + return False, 404, f"Star file not found: {star_file}" + + full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {} + plugin_config = full_config.get('starlark-apps', {}) + + pixlet_path = _find_pixlet_binary(plugin_config.get('pixlet_path')) + if not pixlet_path: + return False, 503, "Pixlet binary not found — install pixlet first" + + magnify = plugin_config.get('magnify') + if magnify is None: + hw = full_config.get('display', {}).get('hardware', {}) + cols = hw.get('cols', 64) + chain = hw.get('chain_length', 1) + rows = hw.get('rows', 32) + magnify = max(1, min(8, int(min((cols * chain) / 64, rows / 32)))) + else: + try: + magnify = max(1, min(8, int(magnify))) + except (ValueError, TypeError): + magnify = 1 + + config_file = app_dir / 'config.json' + app_config: Dict[str, Any] = {} + if config_file.exists(): + try: + with open(config_file) as f: + app_config = json.load(f) + except json.JSONDecodeError as e: + logger.warning("Invalid config.json for %r at %s: %s", app_id, config_file, e) + return False, 400, f"Invalid config.json for {app_id}" + except OSError as e: + logger.warning("Cannot read config.json for %r at %s: %s", app_id, config_file, e) + return False, 400, f"Cannot read config.json for {app_id}" + if not isinstance(app_config, dict): + return False, 400, ( + f"config.json for {app_id} must be a JSON object, " + f"got {type(app_config).__name__}" + ) + + INTERNAL_KEYS = {'render_interval', 'display_duration'} + pixlet_config = {k: v for k, v in app_config.items() if k not in INTERNAL_KEYS} + + output_path = str(app_dir / 'cached_render.webp') + cmd = [pixlet_path, 'render', str(star_file)] + for key, value in pixlet_config.items(): + if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', key): + continue + value_str = 'true' if value is True else 'false' if value is False else str(value) + if re.search(r'[`$|<>&;\x00]|\$\(', value_str): + continue + cmd.append(f'{key}={value_str}') + cmd.extend(['-o', output_path, '-m', str(magnify)]) + + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30, cwd=str(app_dir)) + if result.returncode == 0 and os.path.isfile(output_path): + return True, 200, None + return False, 502, f"Pixlet failed (exit {result.returncode}): {result.stderr.strip()}" + except subprocess.TimeoutExpired: + return False, 504, "Render timed out after 30s" + except Exception as e: + logger.exception("Starlark render failed for %r", app_id) + return False, 500, "Render error" +def _write_starlark_manifest(manifest: Dict[str, Any]) -> bool: + """Write the starlark-apps manifest.json to disk with atomic write.""" + temp_file = None + try: + _STARLARK_APPS_DIR.mkdir(parents=True, exist_ok=True) + + # Atomic write: unique temp file in the target directory, then rename. + # with_suffix('.tmp') gave every caller the same manifest.tmp, and + # Flask serves concurrently -- upload, uninstall, config, toggle and + # the plugin toggle all reach here. Two writers shared one file, + # interleaved their json.dump output, and both renamed it, so the + # rename was atomic over content that was a mix of two manifests. + fd, temp_name = tempfile.mkstemp( + dir=str(_STARLARK_APPS_DIR), prefix='.manifest.', suffix='.tmp') + temp_file = Path(temp_name) + with os.fdopen(fd, 'w') as f: + json.dump(manifest, f, indent=2) + f.flush() + os.fsync(f.fileno()) # Ensure data is written to disk + os.chmod(temp_name, 0o644) # mkstemp creates 0600; match a normal write + + # Atomic rename (overwrites destination) + temp_file.replace(_STARLARK_MANIFEST_FILE) + return True + except OSError as e: + logger.error(f"Error writing starlark manifest: {e}") + # Clean up temp file if it exists + if temp_file and temp_file.exists(): + try: + temp_file.unlink() + except Exception: + pass + return False +def _install_star_file(app_id: str, star_file_path: str, metadata: Dict[str, Any], assets_dir: Optional[str] = None) -> bool: + """Install a .star file and update the manifest (standalone, no plugin needed).""" + import shutil + import json + app_dir, path_error = _validate_starlark_app_path(app_id) + if path_error: + logger.warning("Refusing to install %r: %s", app_id, path_error) + return False + app_dir.mkdir(parents=True, exist_ok=True) + dest = app_dir / f"{app_id}.star" + shutil.copy2(star_file_path, str(dest)) + + # Copy asset directories if provided (images/, sources/, etc.) + if assets_dir and Path(assets_dir).exists(): + assets_path = Path(assets_dir) + for item in assets_path.iterdir(): + if item.is_dir(): + # Copy entire directory (e.g., images/, sources/) + dest_dir = app_dir / item.name + if dest_dir.exists(): + shutil.rmtree(dest_dir) + shutil.copytree(item, dest_dir) + logger.debug(f"Copied assets directory: {item.name}") + logger.info(f"Installed assets for {app_id}") + + # Try to extract schema using PixletRenderer + schema = None + try: + PixletRenderer = _get_pixlet_renderer_class() + pixlet = PixletRenderer() + if pixlet.is_available(): + _, schema, _ = pixlet.extract_schema(str(dest)) + if schema: + schema_path = app_dir / "schema.json" + with open(schema_path, 'w') as f: + json.dump(schema, f, indent=2) + logger.info(f"Extracted schema for {app_id}") + except Exception as e: + logger.warning(f"Failed to extract schema for {app_id}: {e}") + + # Create default config — pre-populate with schema defaults + default_config = {} + if schema: + fields = schema.get('fields') or schema.get('schema') or [] + for field in fields: + if isinstance(field, dict) and 'id' in field and 'default' in field: + default_config[field['id']] = field['default'] + + # Create config.json file + config_path = app_dir / "config.json" + with open(config_path, 'w') as f: + json.dump(default_config, f, indent=2) + + with _starlark_manifest_lock(): + manifest = _read_starlark_manifest() + manifest.setdefault('apps', {})[app_id] = { + 'name': metadata.get('name', app_id), + 'enabled': True, + 'render_interval': metadata.get('render_interval', 300), + 'display_duration': metadata.get('display_duration', 15), + 'config': metadata.get('config', {}), + # The filename, not the full path. Readers join this to the app's own + # directory and fall back to a bare '.star', so an absolute + # value gave the key two meanings -- and Path.__truediv__ discards the + # left side when the right is absolute, which pinned the manifest to + # whatever PROJECT_ROOT installed it. Moving or redeploying the install + # then left the app unable to find its own file. + 'star_file': dest.name, + } + return _write_starlark_manifest(manifest) +def _starlark_virtual_plugins() -> list: + """Installed Starlark apps, shaped like plugin entries. + + #253 surfaced these alongside real plugins so an installed .star app can + be seen, enabled and disabled from the same list as everything else; #330 + dropped it with the rest of the Starlark code, which is why an app + installs successfully and then appears nowhere. + + Reads the loaded plugin when there is one and the on-disk manifest + otherwise, so the list is right before starlark-apps has been loaded too. + """ + entries = [] + base = { + 'version': 'starlark', 'category': 'Starlark App', 'tags': ['starlark'], + 'verified': False, 'last_updated': None, 'last_commit': None, + 'last_commit_message': None, 'branch': None, 'web_ui_actions': [], + 'vegas_mode': 'fixed', 'vegas_content_type': 'multi', + 'is_starlark_app': True, + } + try: + plugin = _get_starlark_plugin() + if plugin is not None and hasattr(plugin, 'apps'): + for app_id, app in plugin.apps.items(): + m = getattr(app, 'manifest', {}) or {} + entries.append({**base, + 'id': f'starlark:{app_id}', + 'name': m.get('name', app_id), + 'author': m.get('author', 'Tronbyte Community'), + 'description': m.get('summary', 'Starlark app'), + 'enabled': app.is_enabled(), + 'loaded': True}) + return entries + + for app_id, data in (_read_starlark_manifest().get('apps', {}) or {}).items(): + entries.append({**base, + 'id': f'starlark:{app_id}', + 'name': data.get('name', app_id), + 'author': data.get('author', 'Tronbyte Community'), + 'description': data.get('summary', 'Starlark app'), + 'enabled': data.get('enabled', True), + 'loaded': False}) + except Exception: + # Never let a Starlark problem empty the whole plugins list. + logger.exception('Could not build Starlark virtual plugin entries') + return entries +def _toggle_starlark_app(app_id: str, enabled: bool): + """Enable or disable one Starlark app, loaded or not.""" + # Check for traversal, but toggle the key that was listed. + # _starlark_virtual_plugins publishes the raw manifest key, while + # _validate_and_sanitize_app_id lowercases it and rewrites every character + # outside [a-z0-9_]: an app stored as 'My-App' was offered to the UI as + # 'starlark:My-App' and looked up here as 'my_app', so toggling an app the + # page had just drawn answered 404. Keys written by _install_star_file are + # already sanitised; ones written by the plugin, or edited by hand, are + # not. _validate_starlark_app_path rejects traversal without rewriting. + _, err = _validate_starlark_app_path(app_id) + if err: + # err already names app_id; do not prefix it a second time. + return jsonify({'status': 'error', 'message': err}), 400 + safe_id = app_id + + plugin = _get_starlark_plugin() + if plugin is not None and safe_id in getattr(plugin, 'apps', {}): + def _update(manifest): + # setdefault rather than indexing: the app is loaded, but its + # on-disk entry need not exist, and _update_manifest_safe does not + # catch KeyError -- it would escape as a 500 rather than the error + # this returns. + manifest.setdefault('apps', {}).setdefault(safe_id, {})['enabled'] = enabled + + if plugin._update_manifest_safe(_update) is False: + return jsonify({'status': 'error', + 'message': 'Failed to save app state'}), 500 + # Only now is the in-memory copy allowed to disagree with disk. + plugin.apps[safe_id].manifest['enabled'] = enabled + else: + with _starlark_manifest_lock(): + manifest = _read_starlark_manifest() + app_data = manifest.get('apps', {}).get(safe_id) + if not app_data: + return jsonify({'status': 'error', + 'message': f'Starlark app not found: {safe_id}'}), 404 + app_data['enabled'] = enabled + if not _write_starlark_manifest(manifest): + return jsonify({'status': 'error', 'message': 'Failed to save manifest'}), 500 + + return jsonify({'status': 'success', + 'message': f"Starlark app {'enabled' if enabled else 'disabled'}", + 'enabled': enabled}) + + +# Imported last, and for their side effect: each registers its routes on +# api_v3. They import names from this module, so this module must be fully +# executed before they run. +from web_interface.blueprints.api_v3 import ( # noqa: E402,F401 + backup, + config, + display, + fonts, + misc, + plugins, + starlark, + system, + wifi, +) + +__all__ = ["api_v3"] diff --git a/web_interface/blueprints/api_v3/backup.py b/web_interface/blueprints/api_v3/backup.py new file mode 100644 index 00000000..53e3a12d --- /dev/null +++ b/web_interface/blueprints/api_v3/backup.py @@ -0,0 +1,230 @@ +"""Backup creation, listing and restore. + +Routes decorate the shared `api_v3` Blueprint from ._common, so their +endpoint names are unchanged by living here. +""" +from web_interface.blueprints.api_v3 import ( + PROJECT_ROOT, Path, _coerce_to_bool, _safe_backup_path, api_v3, + datetime, json, jsonify, logger, os, plugin_store_manager, request, + tempfile, +) +import web_interface.blueprints.api_v3 as _pkg +# Read through the module rather than bound by value: tests patch these +# as module attributes, and a value binding would not see the patch. +# Several are also called from helpers that live in __init__, so the +# package is the only patch point that covers every caller. + + +@api_v3.route('/backup/preview', methods=['GET']) +def backup_preview(): + """Return a summary of what a new backup would include.""" + try: + from src.backup_manager import preview_backup_contents + data = preview_backup_contents(PROJECT_ROOT) + return jsonify({'status': 'success', 'data': data}) + except Exception as e: + logger.error("backup_preview failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 +@api_v3.route('/backup/list', methods=['GET']) +def backup_list(): + """List backup ZIPs stored in the export directory.""" + try: + _pkg._BACKUP_EXPORT_DIR.mkdir(parents=True, exist_ok=True) + entries = [] + for p in sorted(_pkg._BACKUP_EXPORT_DIR.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True): + if not p.is_file() or p.suffix != '.zip': + continue + st = p.stat() + entries.append({ + 'filename': p.name, + 'size': st.st_size, + 'created_at': datetime.fromtimestamp(st.st_mtime).strftime('%Y-%m-%d %H:%M:%S'), + }) + return jsonify({'status': 'success', 'data': entries}) + except Exception as e: + logger.error("backup_list failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 +@api_v3.route('/backup/export', methods=['POST']) +def backup_export(): + """Create a new backup ZIP and return its filename.""" + try: + from src.backup_manager import create_backup + zip_path = create_backup(PROJECT_ROOT, output_dir=_pkg._BACKUP_EXPORT_DIR) + return jsonify({'status': 'success', 'filename': zip_path.name}) + except Exception as e: + logger.error("backup_export failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 +@api_v3.route('/backup/validate', methods=['POST']) +def backup_validate(): + """Validate an uploaded backup ZIP and return its manifest.""" + try: + from src.backup_manager import validate_backup + if 'backup_file' not in request.files: + return jsonify({'status': 'error', 'message': 'No backup_file in request'}), 400 + f = request.files['backup_file'] + with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp: + tmp_path = tmp.name + f.save(tmp_path) + try: + ok, err_msg, manifest = validate_backup(Path(tmp_path)) + finally: + try: + os.unlink(tmp_path) + except OSError: + pass + if not ok: + logger.warning("Backup validation failed: %s", err_msg) + return jsonify({'status': 'error', 'message': 'Invalid or corrupted backup file'}), 400 + safe_manifest = { + 'schema_version': manifest.get('schema_version'), + 'created_at': manifest.get('created_at'), + 'ledmatrix_version': manifest.get('ledmatrix_version'), + 'hostname': manifest.get('hostname'), + 'contents': manifest.get('contents', []), + 'detected_contents': manifest.get('detected_contents', []), + 'plugins': manifest.get('plugins', []), + 'total_uncompressed': manifest.get('total_uncompressed'), + 'file_count': manifest.get('file_count'), + } + return jsonify({'status': 'success', 'data': safe_manifest}) + except Exception as e: + logger.error("backup_validate failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 +#: The only keys RestoreOptions recognizes. A typo'd or renamed key (e.g. +#: "restoreSecrets") would otherwise be silently ignored by opts_dict.get(), +#: leaving that flag at its True default -- restoring secrets a caller's +#: request clearly meant to exclude, with no indication anything was wrong. +_RESTORE_OPTION_KEYS = frozenset(( + 'restore_config', 'restore_secrets', 'restore_wifi', 'restore_fonts', + 'restore_plugin_uploads', 'reinstall_plugins', +)) +@api_v3.route('/backup/restore', methods=['POST']) +def backup_restore(): + """Restore a backup ZIP with optional RestoreOptions.""" + try: + from src.backup_manager import restore_backup, RestoreOptions + if 'backup_file' not in request.files: + return jsonify({'status': 'error', 'message': 'No backup_file in request'}), 400 + f = request.files['backup_file'] + options_raw = request.form.get('options', '{}') + try: + opts_dict = json.loads(options_raw) + except json.JSONDecodeError: + opts_dict = None + if not isinstance(opts_dict, dict): + # Every option defaults to True, so falling back to {} on a + # parse failure would silently perform a FULL restore — + # secrets and all — for a caller who asked for a narrow one + # and mis-serialized it. Refuse instead of guessing. + return jsonify({ + 'status': 'error', + 'message': 'Invalid options: expected a JSON object', + }), 400 + unknown_keys = set(opts_dict) - _RESTORE_OPTION_KEYS + if unknown_keys: + return jsonify({ + 'status': 'error', + 'message': f'Unknown restore option(s): {", ".join(sorted(unknown_keys))}', + }), 400 + # _coerce_to_bool (not bare bool()) because a request can send these + # as JSON strings: bool("false") is True in Python, so a caller who + # explicitly asked to skip secrets would have had them restored + # anyway. + options = RestoreOptions( + restore_config=_coerce_to_bool(opts_dict.get('restore_config', True)), + restore_secrets=_coerce_to_bool(opts_dict.get('restore_secrets', True)), + restore_wifi=_coerce_to_bool(opts_dict.get('restore_wifi', True)), + restore_fonts=_coerce_to_bool(opts_dict.get('restore_fonts', True)), + restore_plugin_uploads=_coerce_to_bool(opts_dict.get('restore_plugin_uploads', True)), + reinstall_plugins=_coerce_to_bool(opts_dict.get('reinstall_plugins', True)), + ) + with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as tmp: + tmp_path = tmp.name + f.save(tmp_path) + try: + result = restore_backup(Path(tmp_path), PROJECT_ROOT, options) + finally: + try: + os.unlink(tmp_path) + except OSError: + pass + + # Reinstall plugins if requested and store manager available + if options.reinstall_plugins and result.plugins_to_install: + psm = getattr(api_v3, 'plugin_store_manager', None) or plugin_store_manager + for plug in result.plugins_to_install: + pid = plug.get('plugin_id') + if not pid: + continue + try: + if psm and hasattr(psm, 'install_plugin'): + ok = psm.install_plugin(pid) + if ok: + result.plugins_installed.append(pid) + else: + result.plugins_failed.append({'plugin_id': pid, 'error': 'install_plugin returned False'}) + else: + result.plugins_failed.append({'plugin_id': pid, 'error': 'Store manager unavailable'}) + except Exception as pe: + logger.error( + "[Backup] Failed to reinstall plugin %r: %s", pid, pe, exc_info=True + ) + result.plugins_failed.append({'plugin_id': pid, 'error': 'Installation failed; see server logs'}) + + # A restore that dropped files can still report success if the only + # failures were plugin reinstalls, since those don't touch result.errors. + if result.plugins_failed: + result.success = False + + data = result.to_dict() + if not result.success: + # Name what failed, and what nonetheless landed. A restore is + # partial far more often than it is total -- a fresh install can + # leave config_secrets.json unwritable by the web service, so + # config restores and secrets do not. "Restore had errors" alone + # left the user unable to tell a wholly failed restore from one + # that quietly dropped their API keys. + failed_plugins = [ + str(p.get('plugin_id')) for p in (result.plugins_failed or []) if p.get('plugin_id') + ] + parts = [] + if result.restored: + parts.append(f"restored: {', '.join(result.restored)}") + if result.errors: + parts.append(f"failed: {'; '.join(result.errors)}") + if failed_plugins: + parts.append(f"plugins not reinstalled: {', '.join(failed_plugins)}") + message = 'Restore incomplete — ' + ('. '.join(parts) if parts else 'see logs') + return jsonify({'status': 'error', 'message': message, 'data': data}), 500 + return jsonify({'status': 'success', 'data': data}) + except Exception as e: + logger.error("backup_restore failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 +@api_v3.route('/backup/download/', methods=['GET']) +def backup_download(filename): + """Stream a backup ZIP to the browser.""" + from flask import send_from_directory + if _safe_backup_path(filename) is None: + return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 + try: + # send_from_directory uses werkzeug safe_join internally — CodeQL-recognized sanitizer. + return send_from_directory(_pkg._BACKUP_EXPORT_DIR, filename, as_attachment=True) + except FileNotFoundError: + return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 +@api_v3.route('/backup/', methods=['DELETE']) +def backup_delete(filename): + """Delete a stored backup ZIP.""" + safe = _safe_backup_path(filename) + if safe is None: + return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 + # Enumerate the export directory and match by name so the unlink target is + # a filesystem-derived path rather than one constructed from user input. + try: + for entry in _pkg._BACKUP_EXPORT_DIR.iterdir(): + if entry.is_file() and entry.name == safe.name: + entry.unlink() + return jsonify({'status': 'success'}) + except OSError as e: + logger.error("backup_delete failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'An internal error occurred; see logs for details'}), 500 + return jsonify({'status': 'error', 'message': 'Backup not found'}), 404 diff --git a/web_interface/blueprints/api_v3/config.py b/web_interface/blueprints/api_v3/config.py new file mode 100644 index 00000000..9a7742dd --- /dev/null +++ b/web_interface/blueprints/api_v3/config.py @@ -0,0 +1,1178 @@ +"""Reading and writing configuration, including schedules. + +Routes decorate the shared `api_v3` Blueprint from ._common, so their +endpoint names are unchanged by living here. +""" +from web_interface.blueprints.api_v3 import ( + ErrorCode, Optional, PROJECT_ROOT, Path, _coerce_to_bool, + _redact_credentials, _validate_time_format, api_v3, + deep_merge, describe_exception, error_response, find_secret_fields, json, + jsonify, logger, logging, mask_all_secret_values, merge_secrets, os, + remove_empty_secrets, request, separate_secrets, strip_masked_values, + success_response, +) +import web_interface.blueprints.api_v3 as _pkg +# Read through the module rather than bound by value: tests patch these +# as module attributes, and a value binding would not see the patch. +# Several are also called from helpers that live in __init__, so the +# package is the only patch point that covers every caller. + + +@api_v3.route('/config/main', methods=['GET']) +def get_main_config(): + """Get main configuration, with credentials redacted.""" + try: + if not api_v3.config_manager: + return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 + + config = api_v3.config_manager.load_config() + return jsonify({'status': 'success', 'data': _redact_credentials(config)}) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/config/schedule', methods=['GET']) +def get_schedule_config(): + """Get current schedule configuration""" + try: + if not api_v3.config_manager: + return error_response( + ErrorCode.CONFIG_LOAD_FAILED, + 'Config manager not initialized', + status_code=500 + ) + + config = api_v3.config_manager.load_config() + schedule_config = config.get('schedule', {}) + + return success_response(data=schedule_config) + except Exception as e: + logger.error("%s failed", request.path, exc_info=True) + return error_response( + ErrorCode.CONFIG_LOAD_FAILED, + "An error occurred; see logs for details", + details=describe_exception(e), + status_code=500 + ) +@api_v3.route('/config/schedule', methods=['POST']) +def save_schedule_config(): + """Save schedule configuration""" + try: + if not api_v3.config_manager: + return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 + + data = request.get_json(silent=True) + if not data: + return jsonify({'status': 'error', 'message': 'No data provided'}), 400 + + # Load current config + current_config = api_v3.config_manager.load_config() + + # Build schedule configuration + # Handle enabled checkbox - can be True, False, or 'on' + enabled_value = data.get('enabled', False) + if isinstance(enabled_value, str): + enabled_value = enabled_value.lower() in ('true', 'on', '1') + schedule_config = { + 'enabled': enabled_value + } + + mode = data.get('mode', 'global') + schedule_config['mode'] = mode + + if mode == 'global': + # Simple global schedule + start_time = data.get('start_time', '07:00') + end_time = data.get('end_time', '23:00') + + # Validate time formats + is_valid, error_msg = _validate_time_format(start_time) + if not is_valid: + return error_response( + ErrorCode.VALIDATION_ERROR, + error_msg, + status_code=400 + ) + + is_valid, error_msg = _validate_time_format(end_time) + if not is_valid: + return error_response( + ErrorCode.VALIDATION_ERROR, + error_msg, + status_code=400 + ) + + schedule_config['start_time'] = start_time + schedule_config['end_time'] = end_time + # Remove days config when switching to global mode + schedule_config.pop('days', None) + else: + # Per-day schedule + schedule_config['days'] = {} + # Remove global times when switching to per-day mode + schedule_config.pop('start_time', None) + schedule_config.pop('end_time', None) + days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] + enabled_days_count = 0 + + for day in days: + day_config = {} + enabled_key = f'{day}_enabled' + start_key = f'{day}_start' + end_key = f'{day}_end' + + # Check if day is enabled + if enabled_key in data: + enabled_val = data[enabled_key] + # Handle checkbox values that may come as 'on', True, or False + if isinstance(enabled_val, str): + day_config['enabled'] = enabled_val.lower() in ('true', 'on', '1') + else: + day_config['enabled'] = bool(enabled_val) + else: + # Default to enabled if not specified + day_config['enabled'] = True + + # Only add times if day is enabled + if day_config.get('enabled', True): + enabled_days_count += 1 + start_time = None + end_time = None + + if start_key in data and data[start_key]: + start_time = data[start_key] + else: + start_time = '07:00' + + if end_key in data and data[end_key]: + end_time = data[end_key] + else: + end_time = '23:00' + + # Validate time formats + is_valid, error_msg = _validate_time_format(start_time) + if not is_valid: + return error_response( + ErrorCode.VALIDATION_ERROR, + f"Invalid start time for {day}: {error_msg}", + status_code=400 + ) + + is_valid, error_msg = _validate_time_format(end_time) + if not is_valid: + return error_response( + ErrorCode.VALIDATION_ERROR, + f"Invalid end time for {day}: {error_msg}", + status_code=400 + ) + + day_config['start_time'] = start_time + day_config['end_time'] = end_time + + schedule_config['days'][day] = day_config + + # Validate that at least one day is enabled in per-day mode + if enabled_days_count == 0: + return error_response( + ErrorCode.VALIDATION_ERROR, + "At least one day must be enabled in per-day schedule mode", + status_code=400 + ) + + # Update and save config using atomic save + current_config['schedule'] = schedule_config + success, error_msg = _pkg._save_config_atomic(api_v3.config_manager, current_config, create_backup=True) + if not success: + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + f"Failed to save schedule configuration: {error_msg}", + status_code=500 + ) + + # Invalidate cache on config change + try: + from web_interface.cache import invalidate_cache + invalidate_cache() + except ImportError: + pass + + return success_response(message='Schedule configuration saved successfully') + except Exception as e: + import logging + logger.error("Error saving schedule config", exc_info=True) + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + "An error occurred; see logs for details", + + status_code=500, details=describe_exception(e) + ) +@api_v3.route('/config/dim-schedule', methods=['GET']) +def get_dim_schedule_config(): + """Get current dim schedule configuration""" + import logging + import json + + if not api_v3.config_manager: + logging.error("[DIM SCHEDULE] Config manager not initialized") + return error_response( + ErrorCode.CONFIG_LOAD_FAILED, + 'Config manager not initialized', + status_code=500 + ) + + try: + config = api_v3.config_manager.load_config() + dim_schedule_config = config.get('dim_schedule', { + 'enabled': False, + 'dim_brightness': 30, + 'mode': 'global', + 'start_time': '20:00', + 'end_time': '07:00', + 'days': {} + }) + + return success_response(data=dim_schedule_config) + except FileNotFoundError as e: + logging.error(f"[DIM SCHEDULE] Config file not found: {e}", exc_info=True) + return error_response( + ErrorCode.CONFIG_LOAD_FAILED, + "Configuration file not found", + status_code=500 + ) + except json.JSONDecodeError as e: + logging.error(f"[DIM SCHEDULE] Invalid JSON in config file: {e}", exc_info=True) + return error_response( + ErrorCode.CONFIG_LOAD_FAILED, + "Configuration file contains invalid JSON", + status_code=500 + ) + except (IOError, OSError) as e: + logging.error(f"[DIM SCHEDULE] Error reading config file: {e}", exc_info=True) + return error_response( + ErrorCode.CONFIG_LOAD_FAILED, + "An error occurred; see logs for details", + status_code=500, details=describe_exception(e) + ) + except Exception as e: + logging.error(f"[DIM SCHEDULE] Unexpected error loading config: {e}", exc_info=True) + return error_response( + ErrorCode.CONFIG_LOAD_FAILED, + "An error occurred; see logs for details", + status_code=500, details=describe_exception(e) + ) +@api_v3.route('/config/dim-schedule', methods=['POST']) +def save_dim_schedule_config(): + """Save dim schedule configuration""" + try: + if not api_v3.config_manager: + return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 + + data = request.get_json(silent=True) + if not data: + return jsonify({'status': 'error', 'message': 'No data provided'}), 400 + + # Load current config + current_config = api_v3.config_manager.load_config() + + # Build dim schedule configuration + enabled_value = data.get('enabled', False) + if isinstance(enabled_value, str): + enabled_value = enabled_value.lower() in ('true', 'on', '1') + + # Validate and get dim_brightness + dim_brightness_raw = data.get('dim_brightness', 30) + try: + # Handle empty string or None + if dim_brightness_raw is None or dim_brightness_raw == '': + dim_brightness = 30 + else: + dim_brightness = int(dim_brightness_raw) + except (ValueError, TypeError, OverflowError): + return error_response( + ErrorCode.VALIDATION_ERROR, + "dim_brightness must be an integer between 0 and 100", + status_code=400 + ) + + if not 0 <= dim_brightness <= 100: + return error_response( + ErrorCode.VALIDATION_ERROR, + "dim_brightness must be between 0 and 100", + status_code=400 + ) + + dim_schedule_config = { + 'enabled': enabled_value, + 'dim_brightness': dim_brightness + } + + mode = data.get('mode', 'global') + dim_schedule_config['mode'] = mode + + if mode == 'global': + # Simple global schedule + start_time = data.get('start_time', '20:00') + end_time = data.get('end_time', '07:00') + + # Validate time formats + is_valid, error_msg = _validate_time_format(start_time) + if not is_valid: + return error_response( + ErrorCode.VALIDATION_ERROR, + error_msg, + status_code=400 + ) + + is_valid, error_msg = _validate_time_format(end_time) + if not is_valid: + return error_response( + ErrorCode.VALIDATION_ERROR, + error_msg, + status_code=400 + ) + + dim_schedule_config['start_time'] = start_time + dim_schedule_config['end_time'] = end_time + # Remove days config when switching to global mode + dim_schedule_config.pop('days', None) + else: + # Per-day schedule + dim_schedule_config['days'] = {} + # Remove global times when switching to per-day mode + dim_schedule_config.pop('start_time', None) + dim_schedule_config.pop('end_time', None) + days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] + enabled_days_count = 0 + + for day in days: + day_config = {} + enabled_key = f'{day}_enabled' + start_key = f'{day}_start' + end_key = f'{day}_end' + + # Check if day is enabled + if enabled_key in data: + enabled_val = data[enabled_key] + if isinstance(enabled_val, str): + day_config['enabled'] = enabled_val.lower() in ('true', 'on', '1') + else: + day_config['enabled'] = bool(enabled_val) + else: + day_config['enabled'] = True + + # Only add times if day is enabled + if day_config.get('enabled', True): + enabled_days_count += 1 + start_time = data.get(start_key) or '20:00' + end_time = data.get(end_key) or '07:00' + + # Validate time formats + is_valid, error_msg = _validate_time_format(start_time) + if not is_valid: + return error_response( + ErrorCode.VALIDATION_ERROR, + f"Invalid start time for {day}: {error_msg}", + status_code=400 + ) + + is_valid, error_msg = _validate_time_format(end_time) + if not is_valid: + return error_response( + ErrorCode.VALIDATION_ERROR, + f"Invalid end time for {day}: {error_msg}", + status_code=400 + ) + + day_config['start_time'] = start_time + day_config['end_time'] = end_time + + dim_schedule_config['days'][day] = day_config + + # Validate that at least one day is enabled in per-day mode + if enabled_days_count == 0: + return error_response( + ErrorCode.VALIDATION_ERROR, + "At least one day must be enabled in per-day dim schedule mode", + status_code=400 + ) + + # Update and save config using atomic save + current_config['dim_schedule'] = dim_schedule_config + success, error_msg = _pkg._save_config_atomic(api_v3.config_manager, current_config, create_backup=True) + if not success: + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + f"Failed to save dim schedule configuration: {error_msg}", + status_code=500 + ) + + # Invalidate cache on config change + try: + from web_interface.cache import invalidate_cache + invalidate_cache() + except ImportError: + pass + + return success_response(message='Dim schedule configuration saved successfully') + except Exception as e: + import logging + logger.error("Error saving dim schedule config", exc_info=True) + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + "An error occurred; see logs for details", + + status_code=500, details=describe_exception(e) + ) +@api_v3.route('/config/main', methods=['POST']) +def save_main_config(): + """Save main configuration""" + try: + if not api_v3.config_manager: + return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 + + # Try to get JSON data first, fallback to form data + data = None + if request.content_type == 'application/json': + data = request.get_json() + else: + # Handle form data + data = request.form.to_dict() + # Convert checkbox values + for key in ['web_display_autostart']: + if key in data: + data[key] = data[key] == 'on' + + if not data: + return jsonify({'status': 'error', 'message': 'No data provided'}), 400 + + # What arrives here is the config itself, and the headers carry the + # session cookie -- neither belongs in the journal, least of all at + # ERROR on every save. The shape of the request is the part with + # diagnostic value, so log that, at the level it deserves. + logger.debug("save_main_config: %s, %d top-level key(s)", + request.content_type or 'no content-type', len(data)) + + # Merge with existing config (similar to original implementation) + current_config = api_v3.config_manager.load_config() + + # Handle general settings + # Note: Checkboxes don't send data when unchecked, so we need to check if we're updating general settings + # If any general setting is present, we're updating the general tab + is_general_update = any(k in data for k in ['timezone', 'city', 'state', 'country', 'web_display_autostart', + 'auto_discover', 'auto_load_enabled', 'development_mode', 'plugins_directory']) + + if is_general_update: + # For checkbox: if not present in data during general update, it means unchecked + current_config['web_display_autostart'] = _coerce_to_bool(data.get('web_display_autostart')) + + if 'timezone' in data: + current_config['timezone'] = data['timezone'] + + # Device-wide scroll frame rate, read by plugins via + # BasePlugin.global_config. Bounds match ScrollHelper.set_target_fps, + # which clamps silently -- rejecting here instead means a value that + # would have been quietly altered is reported rather than appearing to + # save and then behaving differently. + if 'target_fps' in data and data['target_fps'] not in ('', None): + raw_target_fps = data['target_fps'] + # A JSON body can carry real floats and bools, where int() would + # silently truncate: 90.5 would save as 90, and true as 1. Reject + # them rather than storing a value the user did not ask for. Form + # posts arrive as strings, so '90.5' still fails in int() below. + if isinstance(raw_target_fps, (bool, float)): + return jsonify({ + 'status': 'error', + 'message': "Invalid value for target_fps: must be an integer" + }), 400 + try: + target_fps = int(raw_target_fps) + except (ValueError, TypeError, OverflowError): + return jsonify({ + 'status': 'error', + 'message': "Invalid value for target_fps: must be an integer" + }), 400 + if not (30 <= target_fps <= 200): + return jsonify({ + 'status': 'error', + 'message': "Invalid value for target_fps: must be between 30 and 200" + }), 400 + current_config['target_fps'] = target_fps + + # Handle location settings + if 'city' in data or 'state' in data or 'country' in data: + if 'location' not in current_config: + current_config['location'] = {} + if 'city' in data: + current_config['location']['city'] = data['city'] + if 'state' in data: + current_config['location']['state'] = data['state'] + if 'country' in data: + current_config['location']['country'] = data['country'] + + # Handle plugin system settings + if 'auto_discover' in data or 'auto_load_enabled' in data or 'development_mode' in data or 'plugins_directory' in data: + if 'plugin_system' not in current_config: + current_config['plugin_system'] = {} + + # Handle plugin system checkboxes - always set to handle unchecked state + # HTML checkboxes omit the key when unchecked, so missing key = unchecked = False + for checkbox in ['auto_discover', 'auto_load_enabled', 'development_mode']: + current_config['plugin_system'][checkbox] = _coerce_to_bool(data.get(checkbox)) + + # Handle plugins_directory + if 'plugins_directory' in data: + current_config['plugin_system']['plugins_directory'] = data['plugins_directory'] + + # Handle display settings + display_fields = ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'hardware_mapping', + 'gpio_slowdown', 'rp1_rio', 'scan_mode', 'disable_hardware_pulsing', 'inverse_colors', 'show_refresh_rate', + 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', 'use_short_date_format', + 'max_dynamic_duration_seconds', 'led_rgb_sequence', 'multiplexing', 'panel_type', + 'row_address_type', 'pixel_mapper_config', 'orientation'] + + if any(k in data for k in display_fields): + if 'display' not in current_config: + current_config['display'] = {} + if 'hardware' not in current_config['display']: + current_config['display']['hardware'] = {} + if 'runtime' not in current_config['display']: + current_config['display']['runtime'] = {} + + # Allowed values for validated string fields + LED_RGB_ALLOWED = {'RGB', 'RBG', 'GRB', 'GBR', 'BRG', 'BGR'} + PANEL_TYPE_ALLOWED = {'', 'FM6126A', 'FM6127'} + + # Validate led_rgb_sequence + if 'led_rgb_sequence' in data and data['led_rgb_sequence'] not in LED_RGB_ALLOWED: + return jsonify({'status': 'error', 'message': f"Invalid LED RGB sequence '{data['led_rgb_sequence']}'. Allowed values: {', '.join(sorted(LED_RGB_ALLOWED))}"}), 400 + + # Validate panel_type + if 'panel_type' in data and data['panel_type'] not in PANEL_TYPE_ALLOWED: + return jsonify({'status': 'error', 'message': f"Invalid panel type '{data['panel_type']}'. Allowed values: Standard (empty), FM6126A, FM6127"}), 400 + + # Validate multiplexing + if 'multiplexing' in data: + try: + mux_val = int(data['multiplexing']) + if mux_val < 0 or mux_val > 22: + return jsonify({'status': 'error', 'message': f"Invalid multiplexing value '{data['multiplexing']}'. Must be an integer from 0 to 22."}), 400 + except (ValueError, TypeError, OverflowError): + return jsonify({'status': 'error', 'message': f"Invalid multiplexing value '{data['multiplexing']}'. Must be an integer from 0 to 22."}), 400 + + # Validate pixel_mapper_config (free-form mapper string, e.g. "U-mapper;Rotate:90") + if 'pixel_mapper_config' in data and not isinstance(data['pixel_mapper_config'], str): + return jsonify({'status': 'error', 'message': 'pixel_mapper_config must be a string (e.g. "U-mapper;Rotate:90" or empty)'}), 400 + + # Validate orientation (physical mounting rotation; composed onto pixel_mapper_config at runtime) + ORIENTATION_ALLOWED = {'normal', '180'} + if 'orientation' in data and data['orientation'] not in ORIENTATION_ALLOWED: + return jsonify({'status': 'error', 'message': f"Invalid orientation '{data['orientation']}'. Allowed values: {', '.join(sorted(ORIENTATION_ALLOWED))}"}), 400 + + # Validate row_address_type + if 'row_address_type' in data: + try: + rat_val = int(data['row_address_type']) + if rat_val < 0 or rat_val > 4: + return jsonify({'status': 'error', 'message': f"Invalid row_address_type '{data['row_address_type']}'. Must be an integer from 0 to 4."}), 400 + except (ValueError, TypeError, OverflowError): + return jsonify({'status': 'error', 'message': f"Invalid row_address_type '{data['row_address_type']}'. Must be an integer from 0 to 4."}), 400 + + # Handle hardware settings + for field in ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'hardware_mapping', 'scan_mode', + 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', + 'led_rgb_sequence', 'multiplexing', 'panel_type', 'row_address_type', + 'pixel_mapper_config', 'orientation']: + if field in data: + if field in ['rows', 'cols', 'chain_length', 'parallel', 'brightness', 'scan_mode', + 'pwm_bits', 'pwm_dither_bits', 'pwm_lsb_nanoseconds', 'limit_refresh_rate_hz', + 'multiplexing', 'row_address_type']: + current_config['display']['hardware'][field] = int(data[field]) + else: + current_config['display']['hardware'][field] = data[field] + + # Handle runtime settings + if 'gpio_slowdown' in data: + current_config['display']['runtime']['gpio_slowdown'] = int(data['gpio_slowdown']) + if 'rp1_rio' in data: + try: + rp1_val = int(data['rp1_rio']) + if rp1_val not in (0, 1): + return jsonify({'status': 'error', 'message': "rp1_rio must be 0 (PIO) or 1 (RIO)"}), 400 + current_config['display']['runtime']['rp1_rio'] = rp1_val + except (ValueError, TypeError, OverflowError): + return jsonify({'status': 'error', 'message': "rp1_rio must be 0 or 1"}), 400 + + # Handle checkboxes - coerce to bool to ensure proper JSON types + for checkbox in ['disable_hardware_pulsing', 'inverse_colors', 'show_refresh_rate']: + current_config['display']['hardware'][checkbox] = _coerce_to_bool(data.get(checkbox)) + + # Handle display-level checkboxes (always set to handle unchecked state) + current_config['display']['use_short_date_format'] = _coerce_to_bool(data.get('use_short_date_format')) + + # Handle dynamic duration settings + if 'max_dynamic_duration_seconds' in data: + if 'dynamic_duration' not in current_config['display']: + current_config['display']['dynamic_duration'] = {} + current_config['display']['dynamic_duration']['max_duration_seconds'] = int(data['max_dynamic_duration_seconds']) + + # Handle double-sided display settings + double_sided_fields = ['double_sided_enabled', 'double_sided_copies', 'double_sided_axis'] + if any(k in data for k in double_sided_fields): + if 'display' not in current_config: + current_config['display'] = {} + if 'double_sided' not in current_config['display']: + current_config['display']['double_sided'] = {} + ds_config = current_config['display']['double_sided'] + + # Enabled checkbox: omitted from the form when unchecked. + # The Display form posts copies/axis on every save regardless of this + # checkbox, so when the feature is off we accept the values without + # rejecting the whole save — otherwise a stale copies/chain_length + # mismatch locks the user out of every other display setting. + enabled = _coerce_to_bool(data.get('double_sided_enabled')) + ds_config['enabled'] = enabled + + def _copies_fits_hardware(copies: int) -> Optional[str]: + """Error message if copies doesn't divide the panel evenly, else None.""" + # Use axis from this request if provided, else from stored config. + hw = current_config.get('display', {}).get('hardware', {}) + effective_axis = (data.get('double_sided_axis') + or current_config.get('display', {}).get('double_sided', {}).get('axis', 'horizontal')) + if effective_axis == 'horizontal': + chain_length = int(hw.get('chain_length', 2) or 2) + if chain_length % copies != 0: + return f"Double-sided copies ({copies}) must divide chain length ({chain_length}) evenly" + elif effective_axis == 'vertical': + parallel = int(hw.get('parallel', 1) or 1) + if parallel % copies != 0: + return f"Double-sided copies ({copies}) must divide parallel ({parallel}) evenly" + return None + + if 'double_sided_copies' in data and data['double_sided_copies'] not in ('', None): + copies = None + try: + copies = int(data['double_sided_copies']) + except (ValueError, TypeError, OverflowError): + if enabled: + return jsonify({'status': 'error', 'message': "Double-sided copies must be an integer"}), 400 + if copies is not None and not (2 <= copies <= 8): + if enabled: + return jsonify({'status': 'error', 'message': "Double-sided copies must be between 2 and 8"}), 400 + # Disabled: leave the stored value alone rather than writing junk. + copies = None + if copies is not None: + # Divisibility is a hardware-relational check — only meaningful + # when the feature is actually on. + if enabled: + fit_error = _copies_fits_hardware(copies) + if fit_error: + return jsonify({'status': 'error', 'message': fit_error}), 400 + ds_config['copies'] = copies + + if 'double_sided_axis' in data: + axis = data['double_sided_axis'] + if axis not in ('horizontal', 'vertical'): + if enabled: + return jsonify({'status': 'error', 'message': "Double-sided axis must be 'horizontal' or 'vertical'"}), 400 + else: + ds_config['axis'] = axis + + # Handle Vegas scroll mode settings + vegas_fields = ['vegas_scroll_enabled', 'vegas_scroll_speed', 'vegas_separator_width', + 'vegas_target_fps', 'vegas_buffer_ahead', 'vegas_plugin_order', 'vegas_excluded_plugins', + 'vegas_auto_trim', 'vegas_trim_threshold', 'vegas_content_padding', + 'vegas_min_plugin_width', 'vegas_lead_in_width', 'vegas_plugins_per_cycle', + 'vegas_max_plugin_width_ratio', 'vegas_dynamic_duration_enabled', + 'vegas_min_cycle_duration', 'vegas_max_cycle_duration', + 'vegas_intra_plugin_gap', 'vegas_render_width_pct', + 'vegas_min_content_separation', 'vegas_min_cut_gap', + 'vegas_continuous_scroll', 'vegas_extend_threshold_screens', + 'vegas_smooth_scroll', 'vegas_overflow_mode'] + + if any(k in data for k in vegas_fields): + if 'display' not in current_config: + current_config['display'] = {} + if 'vegas_scroll' not in current_config['display']: + current_config['display']['vegas_scroll'] = {} + + vegas_config = current_config['display']['vegas_scroll'] + + # Handle enabled checkbox + # HTML checkboxes omit the key entirely when unchecked, so if the form + # was submitted (any vegas field present) but enabled key is missing, + # the checkbox was unchecked and we should set enabled=False + vegas_config['enabled'] = _coerce_to_bool(data.get('vegas_scroll_enabled')) + vegas_config['auto_trim'] = _coerce_to_bool(data.get('vegas_auto_trim')) + vegas_config['dynamic_duration_enabled'] = _coerce_to_bool( + data.get('vegas_dynamic_duration_enabled')) + vegas_config['continuous_scroll'] = _coerce_to_bool( + data.get('vegas_continuous_scroll')) + vegas_config['smooth_scroll'] = _coerce_to_bool( + data.get('vegas_smooth_scroll')) + + # max_plugin_width_ratio is the one fractional setting, so it is + # handled outside the integer loop below. + if data.get('vegas_overflow_mode') not in ('', None): + mode = str(data['vegas_overflow_mode']).strip().lower() + if mode not in ('rotate', 'truncate'): + return jsonify({ + 'status': 'error', + 'message': "Invalid value for vegas_overflow_mode: " + "must be 'rotate' or 'truncate'" + }), 400 + vegas_config['overflow_mode'] = mode + + if data.get('vegas_extend_threshold_screens') not in ('', None): + try: + screens = float(data['vegas_extend_threshold_screens']) + except (ValueError, TypeError, OverflowError): + return jsonify({ + 'status': 'error', + 'message': "Invalid value for vegas_extend_threshold_screens: " + "must be a number" + }), 400 + if not (1.0 <= screens <= 10.0): + return jsonify({ + 'status': 'error', + 'message': "Invalid value for vegas_extend_threshold_screens: " + "must be between 1.0 and 10.0" + }), 400 + vegas_config['extend_threshold_screens'] = screens + + if data.get('vegas_max_plugin_width_ratio') not in ('', None): + try: + ratio = float(data['vegas_max_plugin_width_ratio']) + except (ValueError, TypeError, OverflowError): + return jsonify({ + 'status': 'error', + 'message': "Invalid value for vegas_max_plugin_width_ratio: " + "must be a number" + }), 400 + if not (0 <= ratio <= 20): + return jsonify({ + 'status': 'error', + 'message': "Invalid value for vegas_max_plugin_width_ratio: " + "must be between 0 and 20 (0 disables the cap)" + }), 400 + vegas_config['max_plugin_width_ratio'] = ratio + + # Handle numeric settings with validation. + # + # These bounds must match VegasModeConfig.validate(), which is what + # actually gates Vegas starting. Where they were looser, a value + # saved with a 200 and then made VegasModeCoordinator.start() bail + # out with only a log line, so the ticker silently never ran. + # Where they were tighter (scroll_speed capped at 100 against a + # slider that goes to 200), a legitimate value was rejected with a + # 400. See test_vegas_api_bounds_match_validate. + numeric_fields = { + 'vegas_scroll_speed': ('scroll_speed', 1, 200), + 'vegas_separator_width': ('separator_width', 0, 128), + 'vegas_intra_plugin_gap': ('intra_plugin_gap', 0, 128), + 'vegas_render_width_pct': ('render_width_pct', 10, 100), + 'vegas_min_content_separation': ('min_content_separation', 0, 256), + 'vegas_min_cut_gap': ('min_cut_gap', 1, 128), + 'vegas_target_fps': ('target_fps', 30, 200), + 'vegas_buffer_ahead': ('buffer_ahead', 1, 5), + 'vegas_trim_threshold': ('trim_threshold', 0, 254), + 'vegas_content_padding': ('content_padding', 0, 128), + 'vegas_min_plugin_width': ('min_plugin_width', 0, 512), + 'vegas_lead_in_width': ('lead_in_width', 0, 2048), + 'vegas_plugins_per_cycle': ('plugins_per_cycle', 1, 50), + 'vegas_min_cycle_duration': ('min_cycle_duration', 5, 3600), + 'vegas_max_cycle_duration': ('max_cycle_duration', 10, 3600), + } + for field_name, (config_key, min_val, max_val) in numeric_fields.items(): + if field_name in data: + raw_value = data[field_name] + # Skip empty strings (treat as "not provided") + if raw_value == '' or raw_value is None: + continue + try: + int_value = int(raw_value) + except (ValueError, TypeError, OverflowError): + return jsonify({ + 'status': 'error', + 'message': f"Invalid value for {field_name}: must be an integer" + }), 400 + if not (min_val <= int_value <= max_val): + return jsonify({ + 'status': 'error', + 'message': f"Invalid value for {field_name}: must be between {min_val} and {max_val}" + }), 400 + vegas_config[config_key] = int_value + + # Handle plugin order and exclusions (JSON arrays) + if 'vegas_plugin_order' in data: + try: + if isinstance(data['vegas_plugin_order'], str): + parsed = json.loads(data['vegas_plugin_order']) + else: + parsed = data['vegas_plugin_order'] + # Ensure result is a list + vegas_config['plugin_order'] = list(parsed) if isinstance(parsed, (list, tuple)) else [] + except (json.JSONDecodeError, TypeError, ValueError): + vegas_config['plugin_order'] = [] + + if 'vegas_excluded_plugins' in data: + try: + if isinstance(data['vegas_excluded_plugins'], str): + parsed = json.loads(data['vegas_excluded_plugins']) + else: + parsed = data['vegas_excluded_plugins'] + # Ensure result is a list + vegas_config['excluded_plugins'] = list(parsed) if isinstance(parsed, (list, tuple)) else [] + except (json.JSONDecodeError, TypeError, ValueError): + vegas_config['excluded_plugins'] = [] + + # Handle multi-display sync settings + sync_fields = ["sync_role", "sync_port", "sync_follower_position"] + if any(k in data for k in sync_fields): + if 'sync' not in current_config: + current_config['sync'] = {} + SYNC_ROLE_ALLOWED = {'standalone', 'leader', 'follower'} + if 'sync_role' in data: + role_val = str(data['sync_role']).lower() + if role_val not in SYNC_ROLE_ALLOWED: + return jsonify({'status': 'error', 'message': f"Invalid sync role '{role_val}'. Must be one of: standalone, leader, follower"}), 400 + current_config['sync']['role'] = role_val + if 'sync_port' in data: + try: + port_val = int(data['sync_port']) + if not (1024 <= port_val <= 65535): + return jsonify({'status': 'error', 'message': "sync_port must be between 1024 and 65535"}), 400 + current_config['sync']['port'] = port_val + except (ValueError, TypeError, OverflowError): + return jsonify({'status': 'error', 'message': "sync_port must be an integer"}), 400 + + if "sync_follower_position" in data: + pos_val = str(data["sync_follower_position"]).lower() + if pos_val not in {"left", "right"}: + return jsonify({"status": "error", "message": "sync_follower_position must be left or right"}), 400 + current_config["sync"]["follower_position"] = pos_val + + # Handle primary rotation order: must be a JSON array of plugin-id + # strings. Reject anything else with a 400 rather than silently + # coercing, so a buggy client can't clear or corrupt the saved order. + if 'plugin_rotation_order' in data: + raw_order = data.pop('plugin_rotation_order') + try: + parsed = json.loads(raw_order) if isinstance(raw_order, str) else raw_order + except (json.JSONDecodeError, TypeError, ValueError): + return jsonify({'status': 'error', + 'message': 'plugin_rotation_order must be valid JSON'}), 400 + if not isinstance(parsed, list) or not all(isinstance(p, str) for p in parsed): + return jsonify({'status': 'error', + 'message': 'plugin_rotation_order must be a list of plugin-id strings'}), 400 + if 'display' not in current_config: + current_config['display'] = {} + current_config['display']['plugin_rotation_order'] = parsed + + # Handle display durations. Popped from `data` (not just read) so + # they can never also fall through to the generic "remaining keys" + # merge near the end of this function, which would otherwise write + # them AGAIN as bogus top-level config keys (e.g. "clock_duration": 30 + # sitting at config root alongside the correct + # display.display_durations.clock_duration). + duration_fields = [k for k in list(data.keys()) + if k.endswith('_duration') or k in ('default_duration', 'transition_duration')] + if duration_fields: + if 'display' not in current_config: + current_config['display'] = {} + if 'display_durations' not in current_config['display']: + current_config['display']['display_durations'] = {} + + for field in duration_fields: + raw_value = data.pop(field) + try: + int_value = int(raw_value) + except (ValueError, TypeError, OverflowError): + return jsonify({'status': 'error', + 'message': f"Invalid duration for {field}: must be an integer"}), 400 + current_config['display']['display_durations'][field] = int_value + + # Per-mode durations from the Rotation & Durations page, posted as + # duration__ (mode keys are arbitrary plugin mode names, so + # they can't use the suffix convention above). Same pop-and-validate + # treatment, for the same reason. + mode_duration_fields = [k for k in list(data.keys()) if k.startswith('duration__')] + if mode_duration_fields: + if 'display' not in current_config: + current_config['display'] = {} + if 'display_durations' not in current_config['display']: + current_config['display']['display_durations'] = {} + + for field in mode_duration_fields: + raw_value = data.pop(field) + mode_key = field[len('duration__'):] + if not mode_key: + continue + try: + int_value = int(raw_value) + except (ValueError, TypeError, OverflowError): + return jsonify({'status': 'error', + 'message': f"Invalid duration for mode '{mode_key}': must be an integer"}), 400 + current_config['display']['display_durations'][mode_key] = int_value + + # Handle plugin configurations dynamically + # Any key that matches a plugin ID should be saved as plugin config + # This includes proper secret field handling from schema + plugin_keys_to_remove = [] + for key in data: + # Check if this key is a plugin ID + if api_v3.plugin_manager and key in api_v3.plugin_manager.plugin_manifests: + plugin_id = key + plugin_config = data[key] + + # Load plugin schema to identify secret fields (same logic as save_plugin_config) + secret_fields = set() + if api_v3.plugin_manager: + plugins_dir = api_v3.plugin_manager.plugins_dir + else: + plugin_system_config = current_config.get('plugin_system', {}) + plugins_dir_name = plugin_system_config.get('plugins_directory', 'plugin-repos') + if os.path.isabs(plugins_dir_name): + plugins_dir = Path(plugins_dir_name) + else: + plugins_dir = PROJECT_ROOT / plugins_dir_name + schema_path = plugins_dir / plugin_id / 'config_schema.json' + + if schema_path.exists(): + try: + with open(schema_path, 'r', encoding='utf-8') as f: + schema = json.load(f) + if 'properties' in schema: + secret_fields = find_secret_fields(schema['properties']) + except Exception as e: + logger.debug("Error reading schema for secret detection: %s", e) + + # Separate secrets from regular config (same logic as save_plugin_config) + regular_config, secrets_config = separate_secrets(plugin_config, secret_fields) + # The config form renders secrets masked, so every save posts + # them back blank. Without this the blank is merged over the + # stored value and the credential is destroyed by the act of + # changing an unrelated setting. A blank means "unchanged". + secrets_config = remove_empty_secrets(secrets_config) + + # PRE-PROCESSING: Preserve 'enabled' state if not in regular_config + # This prevents overwriting the enabled state when saving config from a form that doesn't include the toggle + if 'enabled' not in regular_config: + try: + if plugin_id in current_config and 'enabled' in current_config[plugin_id]: + regular_config['enabled'] = current_config[plugin_id]['enabled'] + elif api_v3.plugin_manager: + # Fallback to plugin instance if config doesn't have it + plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) + if plugin_instance: + regular_config['enabled'] = plugin_instance.enabled + # Final fallback: default to True if plugin is loaded (matches BasePlugin default) + if 'enabled' not in regular_config: + regular_config['enabled'] = True + except Exception as e: + logger.debug("Error preserving enabled state: %s", e) + # Default to True on error to avoid disabling plugins + regular_config['enabled'] = True + + # Get current secrets config + current_secrets = api_v3.config_manager.get_raw_file_content('secrets') + + # Deep merge regular config into main config + if plugin_id not in current_config: + current_config[plugin_id] = {} + current_config[plugin_id] = deep_merge(current_config[plugin_id], regular_config) + + # Deep merge secrets into secrets config + if secrets_config: + if plugin_id not in current_secrets: + current_secrets[plugin_id] = {} + # Lists merge by replacement, so deep_merge here wrote a + # blanked array straight over the stored credentials. + current_secrets[plugin_id] = merge_secrets( + current_secrets[plugin_id], secrets_config) + # Save secrets file + api_v3.config_manager.save_raw_file_content('secrets', current_secrets) + + # Mark for removal from data dict (already processed) + plugin_keys_to_remove.append(key) + + # Notify plugin of config change if loaded (with merged config including secrets) + try: + if api_v3.plugin_manager: + plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) + if plugin_instance: + # Reload merged config (includes secrets) and pass the plugin-specific section + merged_config = api_v3.config_manager.load_config() + plugin_full_config = merged_config.get(plugin_id, {}) + if hasattr(plugin_instance, 'on_config_change'): + plugin_instance.on_config_change(plugin_full_config) + except Exception as hook_err: + # Don't fail the save if hook fails + logger.warning("on_config_change failed: %s", hook_err) + + # Remove processed plugin keys from data (they're already in current_config) + for key in plugin_keys_to_remove: + del data[key] + + # Handle any remaining config keys + # System settings (timezone, city, etc.) are already handled above + # Plugin configs should use /api/v3/plugins/config endpoint, but we'll handle them here too for flexibility + for key in data: + # Skip system settings that are already handled above + if key in ['timezone', 'city', 'state', 'country', + 'web_display_autostart', 'auto_discover', + 'auto_load_enabled', 'development_mode', + 'plugins_directory', 'target_fps']: + continue + # Skip fields that are already handled above in their own named sections. + # Without this, every form field name lands as a top-level config key too. + if key in display_fields: + continue + if key in sync_fields: + continue + if key in vegas_fields: + continue + if key in double_sided_fields: + continue + # For any remaining keys (including plugin keys), use deep merge to preserve existing settings + if key in current_config and isinstance(current_config[key], dict) and isinstance(data[key], dict): + # Deep merge to preserve existing settings + current_config[key] = deep_merge(current_config[key], data[key]) + else: + current_config[key] = data[key] + + # Save the merged config using atomic save + success, error_msg = _pkg._save_config_atomic(api_v3.config_manager, current_config, create_backup=True) + if not success: + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + f"Failed to save configuration: {error_msg}", + status_code=500 + ) + + # Invalidate cache on config change + try: + from web_interface.cache import invalidate_cache + invalidate_cache() + except ImportError: + pass + + return success_response(message='Configuration saved successfully') + except Exception as e: + logger.error("Error saving config", exc_info=True) + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + "An error occurred; see logs for details", + status_code=500, details=describe_exception(e) + ) +@api_v3.route('/config/secrets', methods=['GET']) +def get_secrets_config(): + """Get secrets configuration""" + try: + if not api_v3.config_manager: + return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 + + config = api_v3.config_manager.get_raw_file_content('secrets') + # This interface has no authentication, and this file is nothing but + # credentials. It was handing all of them to anyone who could reach + # the port. Values are masked; empty and YOUR_* placeholders are left + # alone so a client can still tell "set" from "not set". + return jsonify({'status': 'success', + 'data': mask_all_secret_values(config)}) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/config/raw/main', methods=['POST']) +def save_raw_main_config(): + """Save raw main configuration JSON""" + try: + if not api_v3.config_manager: + return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 + + # silent=True so a malformed body returns None instead of raising + # Werkzeug's own BadRequest, which would answer in a different + # shape than this API's. Distinguish the two causes: a body that + # was sent but does not parse is a different mistake from no body. + data = request.get_json(silent=True) + if data is None and request.get_data(): + return jsonify({'status': 'error', 'message': 'Invalid JSON in request body'}), 400 + if not data: + return jsonify({'status': 'error', 'message': 'No data provided'}), 400 + + # Save the raw config file + api_v3.config_manager.save_raw_file_content('main', data) + + return jsonify({'status': 'success', 'message': 'Main configuration saved successfully'}) + except Exception as e: + from src.exceptions import ConfigError + logger.error("Error saving raw main config", exc_info=True) + + # Extract more specific error message if it's a ConfigError + if isinstance(e, ConfigError): + error_message = 'An error occurred; see logs for details' + if hasattr(e, 'config_path') and e.config_path: + error_message = f"{error_message} (config_path: {e.config_path})" + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + error_message, + details=describe_exception(e), + + context={'config_path': e.config_path} if hasattr(e, 'config_path') and e.config_path else None, + status_code=500 + ) + else: + error_message = 'An error occurred; see logs for details' + return error_response( + ErrorCode.UNKNOWN_ERROR, + error_message, + details=describe_exception(e), + + status_code=500 + ) +@api_v3.route('/config/raw/secrets', methods=['POST']) +def save_raw_secrets_config(): + """Save raw secrets configuration JSON""" + try: + if not api_v3.config_manager: + return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 + + # See save_raw_main_config: silent parsing, with a sent-but-broken + # body reported separately from a missing one. + data = request.get_json(silent=True) + if data is None and request.get_data(): + return jsonify({'status': 'error', 'message': 'Invalid JSON in request body'}), 400 + if not data: + return jsonify({'status': 'error', 'message': 'No data provided'}), 400 + + # The GET above masks what it returns, and this endpoint's only client + # reads the whole file, edits one field and posts all of it back. So + # most of what arrives here is the mask, echoed rather than changed -- + # storing it verbatim would replace every untouched credential with + # eight bullets. Strip those, then merge onto what is already stored, + # which makes "unchanged" mean unchanged. + # + # The cost is that a secret can no longer be cleared by blanking it. + # That needs its own affordance; a control that erases credentials as + # a side effect of saving an unrelated one is not it. + current = api_v3.config_manager.get_raw_file_content('secrets') or {} + merged = deep_merge(current, strip_masked_values(data)) + api_v3.config_manager.save_raw_file_content('secrets', merged) + + # Reload GitHub token in plugin store manager if it exists + if api_v3.plugin_store_manager: + api_v3.plugin_store_manager.github_token = api_v3.plugin_store_manager._load_github_token() + + return jsonify({'status': 'success', 'message': 'Secrets configuration saved successfully'}) + except Exception as e: + from src.exceptions import ConfigError + logger.error("Error saving raw secrets config", exc_info=True) + + # Extract more specific error message if it's a ConfigError + if isinstance(e, ConfigError): + # ConfigError has a message attribute and may have context + error_message = 'An error occurred; see logs for details' + if hasattr(e, 'config_path') and e.config_path: + error_message = f"{error_message} (config_path: {e.config_path})" + else: + error_message = 'An error occurred; see logs for details' + + return jsonify({'status': 'error', 'message': error_message, + 'details': describe_exception(e)}), 500 diff --git a/web_interface/blueprints/api_v3/display.py b/web_interface/blueprints/api_v3/display.py new file mode 100644 index 00000000..ac3efc0d --- /dev/null +++ b/web_interface/blueprints/api_v3/display.py @@ -0,0 +1,326 @@ +"""Display control, on-demand playback and preview. + +Routes decorate the shared `api_v3` Blueprint from ._common, so their +endpoint names are unchanged by living here. +""" +from web_interface.blueprints.api_v3 import ( + _ensure_cache_manager, _ensure_display_service_running, + _get_display_service_status, _stop_display_service, api_v3, + describe_exception, jsonify, logger, os, request, uuid, +) +import web_interface.blueprints.api_v3 as _pkg +# Read through the module rather than bound by value: tests patch these +# as module attributes, and a value binding would not see the patch. +# Several are also called from helpers that live in __init__, so the +# package is the only patch point that covers every caller. + + +@api_v3.route('/display/current', methods=['GET']) +def get_display_current(): + """Get current display state""" + try: + import base64 + from PIL import Image + import io + + snapshot_path = "/tmp/led_matrix_preview.png" + + # Get display dimensions from config + try: + if api_v3.config_manager: + main_config = api_v3.config_manager.load_config() + hardware_config = main_config.get('display', {}).get('hardware', {}) + cols = hardware_config.get('cols', 64) + chain_length = hardware_config.get('chain_length', 2) + rows = hardware_config.get('rows', 32) + parallel = hardware_config.get('parallel', 1) + width = cols * chain_length + height = rows * parallel + else: + width = 128 + height = 64 + except Exception: + width = 128 + height = 64 + + # Try to read snapshot file + image_data = None + if os.path.exists(snapshot_path): + try: + with Image.open(snapshot_path) as img: + # Convert to PNG and encode as base64 + buffer = io.BytesIO() + img.save(buffer, format='PNG') + image_data = base64.b64encode(buffer.getvalue()).decode('utf-8') + except Exception as img_err: + # File might be being written or corrupted, return None + pass + + display_data = { + 'timestamp': _pkg.time.time(), + 'width': width, + 'height': height, + 'image': image_data # Base64 encoded image data or None if unavailable + } + return jsonify({'status': 'success', 'data': display_data}) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/display/modes', methods=['GET']) +def get_display_modes(): + """Every display mode that can be requested on-demand, with its plugin. + + /plugins/installed carries no mode information, so anything driving the + display from outside the web UI (the Home Assistant MQTT bridge, a script) + had to read each plugin's manifest.json off disk and reimplement the + fallbacks in PluginManager.get_plugin_display_modes to do it. This is the + same list the force-display dialog offers, from the source that owns it. + + Knowing each mode's plugin_id also matters because /display/on-demand/start + falls back to find_plugin_for_mode when plugin_id is omitted, and that + lookup only sees modes declared in a static manifest -- a plugin whose + modes are generated (each installed Starlark app is one) 404s there. + Sending the plugin_id from this list skips the lookup entirely. + + Query params: + include_disabled: '1' to list modes of disabled plugins too. They can + still be requested on-demand -- the controller enables the plugin + for the duration -- so they are reported with enabled: false + rather than omitted. + """ + try: + if not api_v3.plugin_manager: + return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 + + # Discovery is lazy and normally triggered by whichever endpoint runs + # first, which is a person opening the dashboard. A caller that never + # visits it would otherwise see an empty list. + api_v3.plugin_manager.discover_plugins() + + include_disabled = request.args.get('include_disabled') in ('1', 'true', 'True') + full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {} + + modes = [] + for plugin_id, manifest in sorted(api_v3.plugin_manager.plugin_manifests.items()): + # A hand-edited or migrated config.json can hold a non-dict under a + # plugin id; DisplayController._reconcile guards the same shape, so + # it happens in practice. Without this, .get() raises AttributeError, + # the loop aborts and the endpoint answers 500 with no modes at all + # -- one bad section would blank every entity the MQTT bridge builds + # from this list. + plugin_config = full_config.get(plugin_id) + if not isinstance(plugin_config, dict): + if plugin_config is not None: + logger.warning( + "Config for plugin %r is %s, not an object; treating it as disabled", + plugin_id, type(plugin_config).__name__) + plugin_config = {} + enabled = bool(plugin_config.get('enabled', False)) + if not enabled and not include_disabled: + continue + plugin_name = (manifest or {}).get('name') or plugin_id + plugin_modes = api_v3.plugin_manager.get_plugin_display_modes(plugin_id) or [plugin_id] + for mode in plugin_modes: + # A single-mode plugin's mode is the plugin, so its own name is + # the readable label. Multi-mode plugins have no per-mode name + # anywhere, so the raw mode string is the only thing to show. + modes.append({ + 'mode': mode, + 'plugin_id': plugin_id, + 'plugin_name': plugin_name, + 'name': plugin_name if len(plugin_modes) == 1 else mode, + 'enabled': enabled, + }) + + return jsonify({'status': 'success', 'data': {'modes': modes}}) + except Exception as exc: + # describe_exception, not a bare message: test_web_error_detail.py + # enforces that every handler here returns it, because a device whose + # storage is failing otherwise answers "see logs for details" from the + # log viewer too. It redacts credentials out of the exception text. + # CodeQL flags this as stack-trace exposure across all ~75 handlers; + # it is the project's deliberate, reviewed trade-off. + logger.error('Error in get_display_modes', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 +@api_v3.route('/display/on-demand/status', methods=['GET']) +def get_on_demand_status(): + """Return the current on-demand display state.""" + try: + cache = _ensure_cache_manager() + state = cache.get('display_on_demand_state', max_age=120) + if state is None: + state = { + 'active': False, + 'status': 'idle', + 'last_updated': None + } + service_status = _get_display_service_status() + return jsonify({ + 'status': 'success', + 'data': { + 'state': state, + 'service': service_status + } + }) + except Exception as exc: + logger.error('Error in get_on_demand_status', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 +@api_v3.route('/display/on-demand/start', methods=['POST']) +def start_on_demand_display(): + """Request the display controller to run a specific plugin on-demand.""" + try: + data = request.get_json(silent=True) or {} + plugin_id = data.get('plugin_id') + mode = data.get('mode') + duration = data.get('duration') + pinned = bool(data.get('pinned', False)) + start_service = data.get('start_service', True) + + if not plugin_id and not mode: + return jsonify({'status': 'error', 'message': 'plugin_id or mode is required'}), 400 + + resolved_plugin = plugin_id + resolved_mode = mode + + if api_v3.plugin_manager: + if resolved_plugin and resolved_plugin not in api_v3.plugin_manager.plugin_manifests: + return jsonify({'status': 'error', 'message': f'Plugin {resolved_plugin} not found'}), 404 + + if resolved_plugin and not resolved_mode: + modes = api_v3.plugin_manager.get_plugin_display_modes(resolved_plugin) + resolved_mode = modes[0] if modes else resolved_plugin + elif resolved_mode and not resolved_plugin: + resolved_plugin = api_v3.plugin_manager.find_plugin_for_mode(resolved_mode) + if not resolved_plugin: + return jsonify({'status': 'error', 'message': f'Mode {resolved_mode} not found'}), 404 + + # Note: On-demand can work with disabled plugins - the display controller + # will temporarily enable them during initialization if needed + # We don't block the request here, but log it for debugging + if api_v3.config_manager and resolved_plugin: + config = api_v3.config_manager.load_config() + plugin_config = config.get(resolved_plugin, {}) + if 'enabled' in plugin_config and not plugin_config.get('enabled', False): + logger.info( + "On-demand request for disabled plugin '%s' - will be temporarily enabled", + resolved_plugin, + ) + + # Set the on-demand request in cache FIRST (before starting service) + # This ensures the request is available when the service starts/restarts + cache = _ensure_cache_manager() + request_id = data.get('request_id') or str(uuid.uuid4()) + request_payload = { + 'request_id': request_id, + 'action': 'start', + 'plugin_id': resolved_plugin, + 'mode': resolved_mode, + 'duration': duration, + 'pinned': pinned, + 'timestamp': _pkg.time.time() + } + cache.set('display_on_demand_request', request_payload) + + # Check if display service is running (or will be started) + service_status = _get_display_service_status() + service_was_running = service_status.get('active', False) + + # Stop the display service first to ensure clean state when we will restart it + if service_was_running and start_service: + import time as time_module + logger.debug("Stopping display service before starting on-demand mode") + _stop_display_service() + # Wait a brief moment for the service to fully stop + time_module.sleep(1.5) + logger.debug("Display service stopped, now starting with on-demand request") + + if not service_status.get('active') and not start_service: + return jsonify({ + 'status': 'error', + 'message': 'Display service is not running. Please start the display service or enable "Start Service" option.', + 'service_status': service_status + }), 400 + + service_result = None + if start_service: + service_result = _ensure_display_service_running() + # Check if service actually started + if service_result and not service_result.get('active'): + return jsonify({ + 'status': 'error', + 'message': 'Failed to start display service. Please check service logs or start it manually.', + 'service_result': service_result + }), 500 + + # Service was restarted (or started fresh) with on-demand request in cache + # The display controller will read the request during initialization or when it polls + + response_data = { + 'request_id': request_id, + 'plugin_id': resolved_plugin, + 'mode': resolved_mode, + 'duration': duration, + 'pinned': pinned, + 'service': service_result + } + return jsonify({'status': 'success', 'data': response_data}) + except Exception as exc: + logger.error('Error in start_on_demand_display', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 +@api_v3.route('/display/on-demand/stop', methods=['POST']) +def stop_on_demand_display(): + """Request the display controller to stop on-demand mode.""" + try: + data = request.get_json(silent=True) or {} + stop_service = data.get('stop_service', False) + + # Set the stop request in cache FIRST + # The display controller will poll this and restart without the on-demand filter + cache = _ensure_cache_manager() + request_id = data.get('request_id') or str(uuid.uuid4()) + request_payload = { + 'request_id': request_id, + 'action': 'stop', + 'timestamp': _pkg.time.time() + } + cache.set('display_on_demand_request', request_payload) + + # Note: The display controller's _clear_on_demand() will handle the restart + # to restore normal operation with all plugins + + service_result = None + if stop_service: + service_result = _stop_display_service() + + return jsonify({ + 'status': 'success', + 'data': { + 'request_id': request_id, + 'service': service_result + } + }) + except Exception as exc: + logger.error('Error in stop_on_demand_display', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(exc)}), 500 +@api_v3.route('/display/current-status', methods=['GET']) +def get_current_display_status(): + """Return the display mode/plugin currently intended to be shown. + + Published by the display process (display_controller._publish_current_mode_state) + to the shared cache whenever the active mode changes, so the web UI (e.g. the + System Logs page) can show what's on screen without querying the display + process directly. + """ + try: + cache = _ensure_cache_manager() + state = cache.get('display_current_state', max_age=120) + if state is None: + state = { + 'mode': None, + 'plugin_id': None, + 'last_updated': None, + } + return jsonify({'status': 'success', 'data': state}) + except Exception as e: + logger.error('Error in get_current_display_status', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 diff --git a/web_interface/blueprints/api_v3/fonts.py b/web_interface/blueprints/api_v3/fonts.py new file mode 100644 index 00000000..ad966ae6 --- /dev/null +++ b/web_interface/blueprints/api_v3/fonts.py @@ -0,0 +1,467 @@ +"""Font catalogue, upload, preview and deletion. + +Routes decorate the shared `api_v3` Blueprint from ._common, so their +endpoint names are unchanged by living here. +""" +from web_interface.blueprints.api_v3 import ( + PROJECT_ROOT, Path, Response, SYSTEM_FONTS, api_v3, describe_exception, + jsonify, logger, os, re, request, validate_file_upload, +) + + +@api_v3.route('/fonts/catalog', methods=['GET']) +def get_fonts_catalog(): + """Get fonts catalog""" + try: + # Check cache first (5 minute TTL) + try: + from web_interface.cache import get_cached, set_cached + cached_result = get_cached('fonts_catalog', ttl_seconds=300) + if cached_result is not None: + return jsonify({'status': 'success', 'data': {'catalog': cached_result}}) + except ImportError: + # Cache not available, continue without caching + get_cached = None + set_cached = None + + # Try to import freetype, but continue without it if unavailable + try: + import freetype + freetype_available = True + except ImportError: + freetype_available = False + + # Scan assets/fonts directory for actual font files + fonts_dir = PROJECT_ROOT / "assets" / "fonts" + catalog = {} + + if fonts_dir.exists() and fonts_dir.is_dir(): + for filename in os.listdir(fonts_dir): + if filename.endswith(('.ttf', '.otf', '.bdf')): + filepath = fonts_dir / filename + # Generate family name from filename (without extension) + family_name = os.path.splitext(filename)[0] + + # Try to get font metadata using freetype (for TTF/OTF) + metadata = {} + if filename.endswith(('.ttf', '.otf')) and freetype_available: + try: + face = freetype.Face(str(filepath)) + if face.valid: + # Get font family name from font file + family_name_from_font = face.family_name.decode('utf-8') if face.family_name else family_name + metadata = { + 'family': family_name_from_font, + 'style': face.style_name.decode('utf-8') if face.style_name else 'Regular', + 'num_glyphs': face.num_glyphs, + 'units_per_em': face.units_per_EM + } + # Use font's family name if available + if family_name_from_font: + family_name = family_name_from_font + except Exception: + # If freetype fails, use filename-based name + pass + + # Store relative path from project root + relative_path = str(filepath.relative_to(PROJECT_ROOT)) + font_type = 'ttf' if filename.endswith('.ttf') else 'otf' if filename.endswith('.otf') else 'bdf' + + # Generate human-readable display name from family_name + display_name = family_name.replace('-', ' ').replace('_', ' ') + # Add space before capital letters for camelCase names + display_name = re.sub(r'([a-z])([A-Z])', r'\1 \2', display_name) + # Add space before numbers that follow letters + display_name = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', display_name) + # Clean up multiple spaces + display_name = ' '.join(display_name.split()) + + # Use filename (without extension) as unique key to avoid collisions + # when multiple files share the same family_name from font metadata + catalog_key = os.path.splitext(filename)[0] + + # Check if this is a system font (cannot be deleted) + is_system = catalog_key.lower() in SYSTEM_FONTS + + catalog[catalog_key] = { + 'filename': filename, + 'family_name': family_name, + 'display_name': display_name, + 'path': relative_path, + 'type': font_type, + 'is_system': is_system, + 'metadata': metadata if metadata else None + } + + # Cache the result (5 minute TTL) if available + if set_cached: + try: + set_cached('fonts_catalog', catalog, ttl_seconds=300) + except Exception: + logger.error("[FontCatalog] Failed to cache fonts_catalog", exc_info=True) + + return jsonify({'status': 'success', 'data': {'catalog': catalog}}) + except Exception as e: + logger.error("%s failed", request.path, exc_info=True) + return jsonify({'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e)}), 500 +@api_v3.route('/fonts/tokens', methods=['GET']) +def get_font_tokens(): + """Get font size tokens""" + try: + # This would integrate with the actual font system + # For now, return sample tokens + tokens = { + 'xs': 6, + 'sm': 8, + 'md': 10, + 'lg': 12, + 'xl': 14, + 'xxl': 16 + } + return jsonify({'status': 'success', 'data': {'tokens': tokens}}) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/fonts/overrides', methods=['GET']) +def get_fonts_overrides(): + """Get font overrides""" + try: + # This would integrate with the actual font system + # For now, return empty overrides + overrides = {} + return jsonify({'status': 'success', 'data': {'overrides': overrides}}) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/fonts/overrides', methods=['POST']) +def save_fonts_overrides(): + """Save font overrides""" + try: + data = request.get_json(silent=True) + if not data: + return jsonify({'status': 'error', 'message': 'No data provided'}), 400 + + # This would integrate with the actual font system + return jsonify({'status': 'success', 'message': 'Font overrides saved'}) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/fonts/overrides/', methods=['DELETE']) +def delete_font_override(element_key): + """Delete font override""" + try: + # This would integrate with the actual font system + return jsonify({'status': 'success', 'message': f'Font override for {element_key} deleted'}) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/fonts/upload', methods=['POST']) +def upload_font(): + """Upload font file""" + try: + if 'font_file' not in request.files: + return jsonify({'status': 'error', 'message': 'No font file provided'}), 400 + + font_file = request.files['font_file'] + if font_file.filename == '': + return jsonify({'status': 'error', 'message': 'No file selected'}), 400 + + # Validate filename. validate_file_upload takes max_size_mb but only + # checks the filename/extension with it -- it never looks at the + # actual upload size, so the size limit below is enforced separately + # before the file is saved (same pattern as the .star upload above). + MAX_FONT_SIZE_MB = 10 + is_valid, error_msg = validate_file_upload( + font_file.filename, + max_size_mb=MAX_FONT_SIZE_MB, + allowed_extensions=['.ttf', '.otf', '.bdf'] + ) + if not is_valid: + return jsonify({'status': 'error', 'message': error_msg}), 400 + + # Check file size (stated limit is MAX_FONT_SIZE_MB) + font_file.seek(0, 2) # Seek to end + file_size = font_file.tell() + font_file.seek(0) # Reset to beginning + max_font_size_bytes = MAX_FONT_SIZE_MB * 1024 * 1024 + if file_size > max_font_size_bytes: + return jsonify({ + 'status': 'error', + 'message': f'File too large (max {MAX_FONT_SIZE_MB}MB, got {file_size / 1024 / 1024:.1f}MB)' + }), 400 + + font_family = request.form.get('font_family', '') + + if not font_family: + return jsonify({'status': 'error', 'message': 'Font file and family name required'}), 400 + + # Validate font family name + if not font_family.replace('_', '').replace('-', '').isalnum(): + return jsonify({'status': 'error', 'message': 'Font family name must contain only letters, numbers, underscores, and hyphens'}), 400 + + # Save the font file to assets/fonts directory + fonts_dir = PROJECT_ROOT / "assets" / "fonts" + fonts_dir.mkdir(parents=True, exist_ok=True) + + # Create filename from family name + original_ext = os.path.splitext(font_file.filename)[1].lower() + safe_filename = f"{font_family}{original_ext}" + filepath = fonts_dir / safe_filename + + # Check if file already exists + if filepath.exists(): + return jsonify({'status': 'error', 'message': f'Font with name {font_family} already exists'}), 400 + + # Save the file + font_file.save(str(filepath)) + + # Clear font catalog cache + try: + from web_interface.cache import delete_cached + delete_cached('fonts_catalog') + except ImportError as e: + logger.warning("[FontUpload] Cache module not available: %s", e) + except Exception: + logger.error("[FontUpload] Failed to clear fonts_catalog cache", exc_info=True) + + return jsonify({ + 'status': 'success', + 'message': f'Font {font_family} uploaded successfully', + 'font_family': font_family, + 'filename': safe_filename, + 'path': f'assets/fonts/{safe_filename}' + }) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/fonts/preview', methods=['GET']) +def get_font_preview() -> tuple[Response, int] | Response: + """Generate a preview image of text rendered with a specific font""" + try: + from PIL import Image, ImageDraw, ImageFont + import io + import base64 + + # Limits to prevent DoS via large image generation on constrained devices + MAX_TEXT_CHARS = 100 + MAX_TEXT_LINES = 3 + MAX_DIM = 1024 # Max width or height in pixels + MAX_PIXELS = 500000 # Max total pixels (e.g., ~700x700) + + font_filename = request.args.get('font', '') + text = request.args.get('text', 'Sample Text 123') + bg_color = request.args.get('bg', '000000') + fg_color = request.args.get('fg', 'ffffff') + + # Validate text length and line count early + if len(text) > MAX_TEXT_CHARS: + return jsonify({'status': 'error', 'message': f'Text exceeds maximum length of {MAX_TEXT_CHARS} characters'}), 400 + if text.count('\n') >= MAX_TEXT_LINES: + return jsonify({'status': 'error', 'message': f'Text exceeds maximum of {MAX_TEXT_LINES} lines'}), 400 + + # Safe integer parsing for size + try: + size = int(request.args.get('size', 12)) + except (ValueError, TypeError, OverflowError): + return jsonify({'status': 'error', 'message': 'Invalid font size'}), 400 + + if not font_filename: + return jsonify({'status': 'error', 'message': 'Font filename required'}), 400 + + # Validate size + if size < 4 or size > 72: + return jsonify({'status': 'error', 'message': 'Font size must be between 4 and 72'}), 400 + + # Security: Validate font_filename to prevent path traversal + # Only allow alphanumeric, hyphen, underscore, and dot (for extension) + safe_name = Path(font_filename).name # Strip any directory components + if safe_name != font_filename or '..' in font_filename: + return jsonify({'status': 'error', 'message': 'Invalid font filename'}), 400 + + # Validate extension + allowed_extensions = ['.ttf', '.otf', '.bdf'] + has_valid_ext = any(safe_name.lower().endswith(ext) for ext in allowed_extensions) + name_without_ext = safe_name.rsplit('.', 1)[0] if '.' in safe_name else safe_name + + # Find the font file + fonts_dir = PROJECT_ROOT / "assets" / "fonts" + if not fonts_dir.exists(): + return jsonify({'status': 'error', 'message': 'Fonts directory not found'}), 404 + + font_path = fonts_dir / safe_name + + if not font_path.exists() and not has_valid_ext: + # Try finding by family name (without extension) + for ext in allowed_extensions: + potential_path = fonts_dir / f"{name_without_ext}{ext}" + if potential_path.exists(): + font_path = potential_path + break + + # Final security check: ensure path is within fonts_dir + try: + font_path.resolve().relative_to(fonts_dir.resolve()) + except ValueError: + return jsonify({'status': 'error', 'message': 'Invalid font path'}), 400 + + if not font_path.exists(): + return jsonify({'status': 'error', 'message': f'Font file not found: {font_filename}'}), 404 + + # Parse colors + try: + bg_rgb = tuple(int(bg_color[i:i+2], 16) for i in (0, 2, 4)) + fg_rgb = tuple(int(fg_color[i:i+2], 16) for i in (0, 2, 4)) + except (ValueError, IndexError): + bg_rgb = (0, 0, 0) + fg_rgb = (255, 255, 255) + + # Load font + font = None + if str(font_path).endswith('.bdf'): + # BDF fonts require complex per-glyph rendering via freetype + # Return explicit error rather than showing misleading preview with default font + return jsonify({ + 'status': 'error', + 'message': 'BDF font preview not supported. BDF fonts will render correctly on the LED matrix.' + }), 400 + else: + # TTF/OTF fonts + try: + font = ImageFont.truetype(str(font_path), size) + except (IOError, OSError) as e: + # IOError/OSError raised for invalid/corrupt font files + logger.warning("[FontPreview] Failed to load font %s: %s", font_path, e) + font = ImageFont.load_default() + + # Calculate text size + temp_img = Image.new('RGB', (1, 1)) + temp_draw = ImageDraw.Draw(temp_img) + bbox = temp_draw.textbbox((0, 0), text, font=font) + text_width = bbox[2] - bbox[0] + text_height = bbox[3] - bbox[1] + + # Create image with padding + padding = 10 + img_width = max(text_width + padding * 2, 100) + img_height = max(text_height + padding * 2, 30) + + # Validate resulting image size to prevent memory/CPU spikes + if img_width > MAX_DIM or img_height > MAX_DIM: + return jsonify({'status': 'error', 'message': 'Requested image too large'}), 400 + if img_width * img_height > MAX_PIXELS: + return jsonify({'status': 'error', 'message': 'Requested image too large'}), 400 + + img = Image.new('RGB', (img_width, img_height), bg_rgb) + draw = ImageDraw.Draw(img) + + # Center text + x = (img_width - text_width) // 2 + y = (img_height - text_height) // 2 + + draw.text((x, y), text, font=font, fill=fg_rgb) + + # Convert to base64 + buffer = io.BytesIO() + img.save(buffer, format='PNG') + buffer.seek(0) + img_base64 = base64.b64encode(buffer.getvalue()).decode('utf-8') + + return jsonify({ + 'status': 'success', + 'data': { + 'image': f'data:image/png;base64,{img_base64}', + 'width': img_width, + 'height': img_height + } + }) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/fonts/', methods=['DELETE']) +def delete_font(font_family: str) -> tuple[Response, int] | Response: + """Delete a user-uploaded font file""" + try: + # Security: Validate font_family to prevent path traversal + # Reject if it contains path separators or .. + if '..' in font_family or '/' in font_family or '\\' in font_family: + return jsonify({'status': 'error', 'message': 'Invalid font family name'}), 400 + + # Only allow safe characters: alphanumeric, hyphen, underscore, dot + if not re.match(r'^[a-zA-Z0-9_\-\.]+$', font_family): + return jsonify({'status': 'error', 'message': 'Invalid font family name'}), 400 + + # Check if this is a system font (uses module-level SYSTEM_FONTS frozenset) + if font_family.lower() in SYSTEM_FONTS: + return jsonify({'status': 'error', 'message': 'Cannot delete system fonts'}), 403 + + # Find and delete the font file + fonts_dir = PROJECT_ROOT / "assets" / "fonts" + + # Ensure fonts directory exists + if not fonts_dir.exists() or not fonts_dir.is_dir(): + return jsonify({'status': 'error', 'message': 'Fonts directory not found'}), 404 + + deleted = False + deleted_filename = None + + # Only try valid font extensions (no empty string to avoid matching directories) + for ext in ['.ttf', '.otf', '.bdf']: + potential_path = fonts_dir / f"{font_family}{ext}" + + # Security: Verify path is within fonts_dir + try: + potential_path.resolve().relative_to(fonts_dir.resolve()) + except ValueError: + continue # Path escapes fonts_dir, skip + + if potential_path.exists() and potential_path.is_file(): + potential_path.unlink() + deleted = True + deleted_filename = f"{font_family}{ext}" + break + + if not deleted: + # Try case-insensitive match within fonts directory + font_family_lower = font_family.lower() + for filename in os.listdir(fonts_dir): + # Only consider files with valid font extensions + if not any(filename.lower().endswith(ext) for ext in ['.ttf', '.otf', '.bdf']): + continue + + name_without_ext = os.path.splitext(filename)[0] + if name_without_ext.lower() == font_family_lower: + filepath = fonts_dir / filename + + # Security: Verify path is within fonts_dir + try: + filepath.resolve().relative_to(fonts_dir.resolve()) + except ValueError: + continue # Path escapes fonts_dir, skip + + if filepath.is_file(): + filepath.unlink() + deleted = True + deleted_filename = filename + break + + if not deleted: + return jsonify({'status': 'error', 'message': f'Font not found: {font_family}'}), 404 + + # Clear font catalog cache + try: + from web_interface.cache import delete_cached + delete_cached('fonts_catalog') + except ImportError as e: + logger.warning("[FontDelete] Cache module not available: %s", e) + except Exception: + logger.error("[FontDelete] Failed to clear fonts_catalog cache", exc_info=True) + + return jsonify({ + 'status': 'success', + 'message': f'Font {deleted_filename} deleted successfully' + }) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 diff --git a/web_interface/blueprints/api_v3/misc.py b/web_interface/blueprints/api_v3/misc.py new file mode 100644 index 00000000..094ddecc --- /dev/null +++ b/web_interface/blueprints/api_v3/misc.py @@ -0,0 +1,398 @@ +"""Routes with no larger group of their own: errors, integrations, +cache, sync, skins, logs, health and hardware. + +Routes decorate the shared `api_v3` Blueprint from ._common, so their +endpoint names are unchanged by living here. +""" +from web_interface.blueprints.api_v3 import ( + ErrorCode, Path, _JOURNALCTL, _SUDO, _get_display_service_status, api_v3, + describe_exception, error_response, get_error_aggregator, json, jsonify, + logger, os, request, subprocess, success_response, +) +import web_interface.blueprints.api_v3 as _pkg +# Read through the module rather than bound by value: tests patch these +# as module attributes, and a value binding would not see the patch. +# Several are also called from helpers that live in __init__, so the +# package is the only patch point that covers every caller. + + +@api_v3.route('/health', methods=['GET']) +def get_health(): + """Get system health status""" + try: + health_status = { + 'status': 'healthy', + 'timestamp': _pkg.time.time(), + 'services': {}, + 'checks': {} + } + + # Check web interface service + # Stamp the start _pkg.time before measuring against it -- reading it with a + # fallback of _pkg.time.time() and only assigning afterwards made the very + # first call subtract two separate clock reads, reporting a small + # negative uptime. + if not hasattr(get_health, '_start_time'): + get_health._start_time = _pkg.time.time() + health_status['services']['web_interface'] = { + 'status': 'running', + 'uptime_seconds': _pkg.time.time() - get_health._start_time + } + + # Check display service + display_service_status = _get_display_service_status() + health_status['services']['display_service'] = { + 'status': 'active' if display_service_status.get('active') else 'inactive', + 'details': display_service_status + } + + # Check config file accessibility + try: + if api_v3.config_manager: + test_config = api_v3.config_manager.load_config() + health_status['checks']['config_file'] = { + 'status': 'accessible', + 'readable': True + } + else: + health_status['checks']['config_file'] = { + 'status': 'unknown', + 'readable': False + } + except Exception as e: + health_status['checks']['config_file'] = { + 'status': 'error', + 'readable': False, + 'error': 'see logs for details' + } + + # Check plugin system + try: + if api_v3.plugin_manager: + # Try to discover plugins (lightweight check) + plugin_count = len(api_v3.plugin_manager.get_available_plugins()) if hasattr(api_v3.plugin_manager, 'get_available_plugins') else 0 + health_status['checks']['plugin_system'] = { + 'status': 'operational', + 'plugin_count': plugin_count + } + else: + health_status['checks']['plugin_system'] = { + 'status': 'not_initialized' + } + except Exception as e: + health_status['checks']['plugin_system'] = { + 'status': 'error', + 'error': 'see logs for details' + } + + # Check hardware connectivity (if display manager available) + try: + snapshot_path = "/tmp/led_matrix_preview.png" + if os.path.exists(snapshot_path): + # Check if snapshot is recent (updated in last 60 seconds) + mtime = os.path.getmtime(snapshot_path) + age_seconds = _pkg.time.time() - mtime + health_status['checks']['hardware'] = { + 'status': 'connected' if age_seconds < 60 else 'stale', + 'snapshot_age_seconds': round(age_seconds, 1) + } + else: + health_status['checks']['hardware'] = { + 'status': 'no_snapshot', + 'note': 'Display service may not be running' + } + except Exception as e: + health_status['checks']['hardware'] = { + 'status': 'unknown', + 'error': 'see logs for details' + } + + # Determine overall health + all_healthy = all( + check.get('status') in ['accessible', 'operational', 'connected', 'running', 'active'] + for check in health_status['checks'].values() + ) + + if not all_healthy: + health_status['status'] = 'degraded' + + return jsonify({'status': 'success', 'data': health_status}) + except Exception as e: + logger.error("%s failed", request.path, exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e), + 'data': {'status': 'unhealthy'} + }), 500 +@api_v3.route('/hardware/status', methods=['GET']) +def get_hardware_status(): + """Return LED matrix hardware initialization status written by display_manager at startup.""" + status_path = "/tmp/led_matrix_hw_status.json" # nosec B108 + try: + with open(status_path) as f: + hw_data = json.load(f) + return jsonify({"status": "success", "data": hw_data}) + except FileNotFoundError: + return jsonify({"status": "success", "data": {"ok": None, "error": "Display service not yet started"}}) + except PermissionError: + logger.warning("Permission denied reading hardware status file; display service may be running as a different user") + return jsonify({"status": "success", "data": {"ok": False, "error": "Hardware status temporarily unavailable"}}) + except json.JSONDecodeError: + logger.error("Failed to parse hardware status file", exc_info=True) + return jsonify({"status": "success", "data": {"ok": False, "error": "Hardware status file corrupted"}}) + except Exception: + logger.error("Unexpected error reading hardware status", exc_info=True) + return jsonify({"status": "error", "message": "Unable to read hardware status"}), 500 +@api_v3.route('/skins', methods=['GET']) +def list_skins(): + """List installed visual skins (docs/SKIN_SYSTEM.md). + + Optional ?plugin_id=... filters to skins matching that plugin. + """ + try: + from src.skin_system import skin_runtime + + plugin_id = request.args.get('plugin_id') + if plugin_id: + skins = skin_runtime.skins_for_plugin(plugin_id) + else: + # The discovery cache self-invalidates on directory/manifest + # mtime changes, so no force_refresh — keeps Pi disk I/O down. + skins = skin_runtime.discover_skins() + + payload = [] + for skin_id, manifest in sorted(skins.items()): + skin_dir = Path(manifest['_skin_dir']) + preview = manifest.get('preview') + payload.append({ + 'id': skin_id, + 'name': manifest.get('name', skin_id), + 'version': manifest.get('version'), + 'author': manifest.get('author'), + 'description': manifest.get('description', ''), + 'skin_api_version': manifest.get('skin_api_version'), + 'targets': manifest.get('targets', {}), + 'modes': manifest.get('modes', []), + 'has_preview': bool(preview and (skin_dir / preview).is_file()), + }) + return jsonify({'status': 'success', 'data': {'skins': payload}}) + except Exception as e: + logger.error('Error in list_skins', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/logs', methods=['GET']) +def get_logs(): + """Get system logs from journalctl""" + try: + if not _JOURNALCTL: + return jsonify({'status': 'error', 'message': 'journalctl not found on this system'}), 503 + # Get recent logs from journalctl + _cmd = ([_SUDO, _JOURNALCTL] if _SUDO else [_JOURNALCTL]) + [ + '-u', 'ledmatrix.service', '-u', 'ledmatrix-web.service', + '-n', '100', '--no-pager', '--output=short-iso'] + result = subprocess.run( + _cmd, + capture_output=True, + text=True, + timeout=5 + ) + + if result.returncode == 0: + logs_text = result.stdout.strip() + return jsonify({ + 'status': 'success', + 'data': { + 'logs': logs_text if logs_text else 'No logs available from ledmatrix or ledmatrix-web service' + } + }) + else: + return jsonify({ + 'status': 'error', + 'message': f'Failed to get logs: {result.stderr}' + }), 500 + + except subprocess.TimeoutExpired: + return jsonify({ + 'status': 'error', + 'message': 'Timeout while fetching logs' + }), 500 + except Exception as e: + logger.error("%s failed", request.path, exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) + }), 500 +# Multi-Display Sync Endpoints +@api_v3.route('/sync/status', methods=['GET']) +def get_sync_status(): + """Return live multi-display sync status written by the display process.""" + import os as _os + status_file = "/tmp/led_matrix_sync_status.json" + # Also surface config so the UI can show the configured role even before + # the display process has written a status file. + cfg_role = "standalone" + cfg_port = 5765 + if api_v3.config_manager: + try: + cfg = api_v3.config_manager.load_config().get("sync", {}) + cfg_role = cfg.get("role", "standalone") + cfg_port = int(cfg.get("port", 5765)) + except Exception: + pass + + if _os.path.exists(status_file): + try: + with open(status_file) as f: + live = json.load(f) + return jsonify({"status": "success", "data": live}) + except Exception: + pass + + # Status file not yet written — return config-only placeholder + return jsonify({ + "status": "success", + "data": { + "role": cfg_role, + "port": cfg_port, + "state": "starting", + } + }) +@api_v3.route('/cache/list', methods=['GET']) +def list_cache_files(): + """List all cache files with metadata""" + try: + if not api_v3.cache_manager: + # Initialize cache manager if not already initialized + from src.cache_manager import CacheManager + api_v3.cache_manager = CacheManager() + + cache_files = api_v3.cache_manager.list_cache_files() + cache_dir = api_v3.cache_manager.get_cache_dir() + + return jsonify({ + 'status': 'success', + 'data': { + 'cache_files': cache_files, + 'cache_dir': cache_dir, + 'total_files': len(cache_files) + } + }) + except Exception as e: + logger.error('Error in list_cache_files', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/cache/delete', methods=['POST']) +def delete_cache_file(): + """Delete a specific cache file by key""" + try: + if not api_v3.cache_manager: + # Initialize cache manager if not already initialized + from src.cache_manager import CacheManager + api_v3.cache_manager = CacheManager() + + data = request.get_json(silent=True) + if not data or 'key' not in data: + return jsonify({'status': 'error', 'message': 'cache key is required'}), 400 + + cache_key = data['key'] + + # Delete the cache file + api_v3.cache_manager.clear_cache(cache_key) + + return jsonify({ + 'status': 'success', + 'message': f'Cache file for key "{cache_key}" deleted successfully' + }) + except Exception as e: + logger.error('Error in delete_cache_file', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/errors/summary', methods=['GET']) +def get_error_summary(): + """ + Get summary of all errors for monitoring and debugging. + + Returns error counts, detected patterns, and recent errors. + """ + try: + aggregator = get_error_aggregator() + summary = aggregator.get_error_summary() + return success_response(data=summary, message="Error summary retrieved") + except Exception as e: + logger.error(f"Error getting error summary: {e}", exc_info=True) + return error_response( + error_code=ErrorCode.SYSTEM_ERROR, + message="Failed to retrieve error summary", + status_code=500 + ) +@api_v3.route('/errors/plugin/', methods=['GET']) +def get_plugin_errors(plugin_id): + """ + Get error health status for a specific plugin. + + Args: + plugin_id: Plugin identifier + + Returns health status and error statistics for the plugin. + """ + try: + aggregator = get_error_aggregator() + health = aggregator.get_plugin_health(plugin_id) + return success_response(data=health, message="Plugin health retrieved") + except Exception as e: + logger.error(f"Error getting plugin health for {plugin_id}: {e}", exc_info=True) + return error_response( + error_code=ErrorCode.SYSTEM_ERROR, + message=f"Failed to retrieve health for plugin {plugin_id}", + status_code=500 + ) +@api_v3.route('/errors/clear', methods=['POST']) +def clear_old_errors(): + """ + Clear error records older than specified age. + + Request body (optional): + max_age_hours: Maximum age in hours (default: 24, max: 8760 = 1 year) + """ + try: + data = request.get_json(silent=True) or {} + raw_max_age = data.get('max_age_hours', 24) + + # Validate and coerce max_age_hours + try: + max_age_hours = int(raw_max_age) + if max_age_hours < 1: + return error_response( + error_code=ErrorCode.INVALID_INPUT, + message="max_age_hours must be at least 1", + context={'provided_value': raw_max_age}, + status_code=400 + ) + if max_age_hours > 8760: # 1 year max + return error_response( + error_code=ErrorCode.INVALID_INPUT, + message="max_age_hours cannot exceed 8760 (1 year)", + context={'provided_value': raw_max_age}, + status_code=400 + ) + except (ValueError, TypeError, OverflowError): + return error_response( + error_code=ErrorCode.INVALID_INPUT, + message="max_age_hours must be a valid integer", + context={'provided_value': str(raw_max_age)}, + status_code=400 + ) + + aggregator = get_error_aggregator() + cleared_count = aggregator.clear_old_records(max_age_hours=max_age_hours) + + return success_response( + data={'cleared_count': cleared_count}, + message=f"Cleared {cleared_count} error records older than {max_age_hours} hours" + ) + except Exception as e: + logger.error(f"Error clearing old errors: {e}", exc_info=True) + return error_response( + error_code=ErrorCode.SYSTEM_ERROR, + message="Failed to clear old errors", + status_code=500 + ) diff --git a/web_interface/blueprints/api_v3/plugins.py b/web_interface/blueprints/api_v3/plugins.py new file mode 100644 index 00000000..b6bb9f11 --- /dev/null +++ b/web_interface/blueprints/api_v3/plugins.py @@ -0,0 +1,3902 @@ +"""Plugin install, update, enable/disable, config and store routes. + +Routes decorate the shared `api_v3` Blueprint from ._common, so their +endpoint names are unchanged by living here. +""" +from web_interface.blueprints.api_v3 import ( + ErrorCode, OperationType, PROJECT_ROOT, Path, Response, + _CALENDAR_LIST_MAX_PAGES, _SKIP_FIELD, + _coerce_to_bool, _do_transactional_uninstall, + _enhance_schema_with_core_properties, _filter_config_by_schema, + _get_plugin_version, _get_schema_property, _installed_plugin_ids, + _is_plugin_update_available, _parse_form_value_with_schema, + _prune_credential_backups, _run_calendar_registration, _set_missing_booleans_to_false, _set_nested_value, + _starlark_virtual_plugins, _toggle_starlark_app, api_v3, datetime, + deep_merge, describe_exception, error_response, find_secret_fields, + hashlib, json, jsonify, logger, logging, merge_secrets, os, redact_text, + remove_empty_secrets, request, separate_secrets, shutil, stat, subprocess, + success_response, sys, tempfile, uuid, validate_request_json, +) +import web_interface.blueprints.api_v3 as _pkg +# Read through the module rather than bound by value: tests patch these +# as module attributes, and a value binding would not see the patch. +# Several are also called from helpers that live in __init__, so the +# package is the only patch point that covers every caller. + + +@api_v3.route('/plugins/installed', methods=['GET']) +def get_installed_plugins(): + """Get installed plugins""" + try: + if not api_v3.plugin_manager or not api_v3.plugin_store_manager: + return jsonify({'status': 'error', 'message': 'Plugin managers not initialized'}), 500 + + import json + from pathlib import Path + + # Re-discover plugins to ensure we have the latest list + # This handles cases where plugins are added/removed after app startup + api_v3.plugin_manager.discover_plugins() + + # Get all installed plugin info from the plugin manager + all_plugin_info = api_v3.plugin_manager.get_all_plugin_info() + + # Load config once before the loop (not per-plugin) + full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {} + + def _build_plugin_entry(plugin_info): + plugin_id = plugin_info.get('id') + try: + return _build_plugin_entry_inner(plugin_info, plugin_id) + except Exception: + logger.exception("Error building plugin entry for %s — skipping", plugin_id) + return None + + def _build_plugin_entry_inner(plugin_info, plugin_id): + # Capture runtime state (state machine + error context) before the + # manifest merge below can shadow the 'state' key. get_all_plugin_info + # attaches this via PluginStateManager.get_state_info(); surfacing it + # lets the UI show *why* a plugin isn't running instead of just + # 'loaded: false'. + state_info = plugin_info.get('state') + plugin_state = None + plugin_error_info = None + if isinstance(state_info, dict): + plugin_state = state_info.get('state') + plugin_error_info = state_info.get('error_info') + + # Re-read manifest from disk to ensure we have the latest metadata + manifest_path = Path(api_v3.plugin_manager.plugins_dir) / plugin_id / "manifest.json" + if manifest_path.exists(): + try: + with open(manifest_path, 'r', encoding='utf-8') as f: + fresh_manifest = json.load(f) + if isinstance(fresh_manifest, dict): + plugin_info.update(fresh_manifest) + else: + logger.debug("Manifest for %s is not a dict (%s) — skipping merge", + plugin_id, type(fresh_manifest).__name__) + except (FileNotFoundError, PermissionError, json.JSONDecodeError) as e: + logger.debug("Could not read fresh manifest for %s: %s", plugin_id, e) + + # Enabled status: config is source of truth, fall back to instance + enabled = None + plugin_config = full_config.get(plugin_id, {}) + if 'enabled' in plugin_config: + enabled = bool(plugin_config['enabled']) + + # Single get_plugin() call shared for both enabled fallback and Vegas mode + plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) + if enabled is None: + enabled = plugin_instance.enabled if plugin_instance else True + + # Verified + latest published version from registry (no network call) + store_info = api_v3.plugin_store_manager.get_registry_info(plugin_id) + verified = store_info.get('verified', False) if store_info else False + latest_version = store_info.get('latest_version', '') if store_info else '' + installed_version = plugin_info.get('version', '') + update_available = _is_plugin_update_available(installed_version, latest_version) + + # Local git info (single subprocess on cache miss, zero on hit) + plugin_path = Path(api_v3.plugin_manager.plugins_dir) / plugin_id + local_git_info = api_v3.plugin_store_manager._get_local_git_info(plugin_path) if plugin_path.exists() else None + + if local_git_info: + sha = local_git_info.get('sha', '') + last_commit = local_git_info.get('short_sha') or (sha[:7] if sha else None) + branch = local_git_info.get('branch') + last_updated = local_git_info.get('date_iso') or local_git_info.get('date') + else: + last_updated = plugin_info.get('last_updated') + last_commit = plugin_info.get('last_commit') or plugin_info.get('last_commit_sha') + branch = plugin_info.get('branch') + if store_info: + last_updated = last_updated or store_info.get('last_updated') or store_info.get('last_updated_iso') + last_commit = last_commit or store_info.get('last_commit') or store_info.get('last_commit_sha') + branch = branch or store_info.get('branch') or store_info.get('default_branch') + + last_commit_message = plugin_info.get('last_commit_message') + if store_info and not last_commit_message: + last_commit_message = store_info.get('last_commit_message') + + # Vegas mode from instance, overridden by explicit config value + vegas_mode = None + vegas_content_type = None + if plugin_instance: + try: + if hasattr(plugin_instance, 'get_vegas_display_mode'): + mode = plugin_instance.get_vegas_display_mode() + vegas_mode = mode.value if hasattr(mode, 'value') else str(mode) + except (AttributeError, TypeError, ValueError) as e: + logger.debug("[%s] Failed to get vegas_display_mode: %s", plugin_id, e) + try: + if hasattr(plugin_instance, 'get_vegas_content_type'): + vegas_content_type = plugin_instance.get_vegas_content_type() + except (AttributeError, TypeError, ValueError) as e: + logger.debug("[%s] Failed to get vegas_content_type: %s", plugin_id, e) + + if 'vegas_mode' in plugin_config: + vegas_mode = plugin_config['vegas_mode'] + + return { + 'id': plugin_id, + 'name': plugin_info.get('name', plugin_id), + 'version': plugin_info.get('version', ''), + 'latest_version': latest_version, + 'update_available': update_available, + 'author': plugin_info.get('author', 'Unknown'), + 'category': plugin_info.get('category', 'General'), + 'description': plugin_info.get('description', 'No description available'), + 'tags': plugin_info.get('tags', []), + 'enabled': enabled, + 'verified': verified, + 'loaded': plugin_info.get('loaded', False), + 'state': plugin_state, + 'error_info': plugin_error_info, + 'last_updated': last_updated, + 'last_commit': last_commit, + 'last_commit_message': last_commit_message, + 'branch': branch, + 'web_ui_actions': plugin_info.get('web_ui_actions', []), + 'vegas_mode': vegas_mode, + 'vegas_content_type': vegas_content_type, + } + + from concurrent.futures import ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=8) as executor: + results = list(executor.map(_build_plugin_entry, all_plugin_info)) + plugins = [r for r in results if r is not None] + plugins.extend(_starlark_virtual_plugins()) + + return jsonify({'status': 'success', 'data': {'plugins': plugins}}) + except Exception as e: + logger.error('Error in get_installed_plugins', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/health', methods=['GET']) +def get_plugin_health(): + """Get health metrics for all plugins""" + try: + if not api_v3.plugin_manager: + return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 + + # Check if health tracker is available + if not hasattr(api_v3.plugin_manager, 'health_tracker') or not api_v3.plugin_manager.health_tracker: + return jsonify({ + 'status': 'success', + 'data': {}, + 'message': 'Health tracking not available' + }) + + tracker = api_v3.plugin_manager.health_tracker + # Build per-plugin summaries by ID so persisted (cross-process) health + # is included, then fold in any in-memory-only entries. + health_summaries = {} + for pid in _installed_plugin_ids(): + try: + # force_reload: this process only reads; bypass the in-memory + # snapshot so each poll reflects the display service's latest + # persisted state. + health_summaries[pid] = tracker.get_health_summary(pid, force_reload=True) + except Exception: + logger.debug('Could not read health summary for %s', pid, exc_info=True) + try: + for pid, summary in tracker.get_all_health_summaries().items(): + health_summaries.setdefault(pid, summary) + except Exception: + logger.debug('get_all_health_summaries failed', exc_info=True) + + return jsonify({ + 'status': 'success', + 'data': health_summaries + }) + except Exception as e: + logger.error('Error in get_plugin_health', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/health/', methods=['GET']) +def get_plugin_health_single(plugin_id): + """Get health metrics for a specific plugin""" + try: + if not api_v3.plugin_manager: + return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 + + # Check if health tracker is available + if not hasattr(api_v3.plugin_manager, 'health_tracker') or not api_v3.plugin_manager.health_tracker: + return jsonify({ + 'status': 'error', + 'message': 'Health tracking not available' + }), 503 + + # Get health summary for specific plugin + health_summary = api_v3.plugin_manager.health_tracker.get_health_summary(plugin_id) + + return jsonify({ + 'status': 'success', + 'data': health_summary + }) + except Exception as e: + logger.error('Error in get_plugin_health_single', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/health//reset', methods=['POST']) +def reset_plugin_health(plugin_id): + """Reset health state for a plugin (manual recovery)""" + try: + if not api_v3.plugin_manager: + return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 + + # Check if health tracker is available + if not hasattr(api_v3.plugin_manager, 'health_tracker') or not api_v3.plugin_manager.health_tracker: + return jsonify({ + 'status': 'error', + 'message': 'Health tracking not available' + }), 503 + + # Reset health state + api_v3.plugin_manager.health_tracker.reset_health(plugin_id) + + return jsonify({ + 'status': 'success', + 'message': f'Health state reset for plugin {plugin_id}' + }) + except Exception as e: + logger.error('Error in reset_plugin_health', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/metrics', methods=['GET']) +def get_plugin_metrics(): + """Get resource metrics for all plugins""" + try: + if not api_v3.plugin_manager: + return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 + + # Check if resource monitor is available + if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: + return jsonify({ + 'status': 'success', + 'data': {}, + 'message': 'Resource monitoring not available' + }) + + monitor = api_v3.plugin_manager.resource_monitor + # Build per-plugin summaries by ID so persisted (cross-process) metrics + # are included, then fold in any in-memory-only entries. + metrics_summaries = {} + for pid in _installed_plugin_ids(): + try: + # force_reload: read-only path — bypass the in-memory snapshot so + # each poll reflects the display service's latest persisted metrics. + metrics_summaries[pid] = monitor.get_metrics_summary(pid, force_reload=True) + except Exception: + logger.debug('Could not read metrics summary for %s', pid, exc_info=True) + try: + for pid, summary in monitor.get_all_metrics_summaries().items(): + metrics_summaries.setdefault(pid, summary) + except Exception: + logger.debug('get_all_metrics_summaries failed', exc_info=True) + + return jsonify({ + 'status': 'success', + 'data': metrics_summaries + }) + except Exception as e: + logger.error('Error in get_plugin_metrics', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/metrics/', methods=['GET']) +def get_plugin_metrics_single(plugin_id): + """Get resource metrics for a specific plugin""" + try: + if not api_v3.plugin_manager: + return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 + + # Check if resource monitor is available + if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: + return jsonify({ + 'status': 'error', + 'message': 'Resource monitoring not available' + }), 503 + + # Get metrics summary for specific plugin + metrics_summary = api_v3.plugin_manager.resource_monitor.get_metrics_summary(plugin_id) + + return jsonify({ + 'status': 'success', + 'data': metrics_summary + }) + except Exception as e: + logger.error('Error in get_plugin_metrics_single', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/metrics//reset', methods=['POST']) +def reset_plugin_metrics(plugin_id): + """Reset metrics for a plugin""" + try: + if not api_v3.plugin_manager: + return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 + + # Check if resource monitor is available + if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: + return jsonify({ + 'status': 'error', + 'message': 'Resource monitoring not available' + }), 503 + + # Reset metrics + api_v3.plugin_manager.resource_monitor.reset_metrics(plugin_id) + + return jsonify({ + 'status': 'success', + 'message': f'Metrics reset for plugin {plugin_id}' + }) + except Exception as e: + logger.error('Error in reset_plugin_metrics', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/limits/', methods=['GET', 'POST']) +def manage_plugin_limits(plugin_id): + """Get or set resource limits for a plugin""" + try: + if not api_v3.plugin_manager: + return jsonify({'status': 'error', 'message': 'Plugin manager not initialized'}), 500 + + # Check if resource monitor is available + if not hasattr(api_v3.plugin_manager, 'resource_monitor') or not api_v3.plugin_manager.resource_monitor: + return jsonify({ + 'status': 'error', + 'message': 'Resource monitoring not available' + }), 503 + + if request.method == 'GET': + # Get limits + limits = api_v3.plugin_manager.resource_monitor.get_limits(plugin_id) + if limits: + return jsonify({ + 'status': 'success', + 'data': { + 'max_memory_mb': limits.max_memory_mb, + 'max_cpu_percent': limits.max_cpu_percent, + 'max_execution_time': limits.max_execution_time, + 'warning_threshold': limits.warning_threshold + } + }) + else: + return jsonify({ + 'status': 'success', + 'data': None, + 'message': 'No limits configured for this plugin' + }) + else: + # POST - Set limits + data = request.get_json(silent=True) or {} + from src.plugin_system.resource_monitor import ResourceLimits + + limits = ResourceLimits( + max_memory_mb=data.get('max_memory_mb'), + max_cpu_percent=data.get('max_cpu_percent'), + max_execution_time=data.get('max_execution_time'), + warning_threshold=data.get('warning_threshold', 0.8) + ) + + api_v3.plugin_manager.resource_monitor.set_limits(plugin_id, limits) + + return jsonify({ + 'status': 'success', + 'message': f'Resource limits updated for plugin {plugin_id}' + }) + except Exception as e: + logger.error('Error in manage_plugin_limits', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/toggle', methods=['POST']) +def toggle_plugin(): + """Toggle plugin enabled/disabled""" + try: + if not api_v3.plugin_manager or not api_v3.config_manager: + return jsonify({'status': 'error', 'message': 'Plugin or config manager not initialized'}), 500 + + # Support both JSON and form data (for HTMX submissions) + content_type = request.content_type or '' + + if 'application/json' in content_type: + data = request.get_json(silent=True) + if not data or 'plugin_id' not in data or 'enabled' not in data: + return jsonify({'status': 'error', 'message': 'plugin_id and enabled required'}), 400 + plugin_id = data['plugin_id'] + enabled = data['enabled'] + else: + # Form data or query string (HTMX submission) + plugin_id = request.args.get('plugin_id') or request.form.get('plugin_id') + if not plugin_id: + return jsonify({'status': 'error', 'message': 'plugin_id required'}), 400 + + # For checkbox toggle, if form was submitted, the checkbox was checked (enabled) + # If using HTMX with hx-trigger="change", we need to check if checkbox is checked + # The checkbox value or 'enabled' form field indicates the state + enabled_str = request.form.get('enabled', request.args.get('enabled', '')) + + # Handle various truthy/falsy values + if enabled_str.lower() in ('true', '1', 'on', 'yes'): + enabled = True + elif enabled_str.lower() in ('false', '0', 'off', 'no', ''): + # Empty string means checkbox was unchecked (toggle off) + enabled = False + else: + # Default: toggle based on current state + config = api_v3.config_manager.load_config() + current_enabled = config.get(plugin_id, {}).get('enabled', False) + enabled = not current_enabled + + # A Starlark app is not a plugin in plugin_manager's sense -- it is an + # entry in starlark-apps' own manifest -- so its enable/disable is + # handled here rather than falling through to the check below, which + # would answer "Plugin not found". + if plugin_id.startswith('starlark:'): + return _toggle_starlark_app(plugin_id[len('starlark:'):], enabled) + + # Check if plugin exists in manifests (discovered but may not be loaded) + if plugin_id not in api_v3.plugin_manager.plugin_manifests: + return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 + + # Update config (this is what the display controller reads) + config = api_v3.config_manager.load_config() + if plugin_id not in config: + config[plugin_id] = {} + config[plugin_id]['enabled'] = enabled + + # Use atomic save if available + if hasattr(api_v3.config_manager, 'save_config_atomic'): + result = api_v3.config_manager.save_config_atomic(config, create_backup=True) + if result.status.value != 'success': + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + f"Failed to save configuration: {result.message}", + status_code=500 + ) + else: + api_v3.config_manager.save_config(config) + + # Update state manager if available + if api_v3.plugin_state_manager: + api_v3.plugin_state_manager.set_plugin_enabled(plugin_id, enabled) + + # Log operation + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "enable" if enabled else "disable", + plugin_id=plugin_id, + status="success" + ) + + # If plugin is loaded, also call its lifecycle methods + # Wrap in try/except to prevent lifecycle errors from failing the toggle + plugin = api_v3.plugin_manager.get_plugin(plugin_id) + if plugin: + try: + if enabled: + if hasattr(plugin, 'on_enable'): + plugin.on_enable() + else: + if hasattr(plugin, 'on_disable'): + plugin.on_disable() + except Exception as lifecycle_error: + # Log the error but don't fail the toggle - config is already saved + import logging + logging.warning(f"Lifecycle method error for {plugin_id}: {lifecycle_error}", exc_info=True) + + return success_response( + message=f"Plugin {plugin_id} {'enabled' if enabled else 'disabled'} successfully" + ) + except Exception as e: + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.PLUGIN_OPERATION_CONFLICT) + if api_v3.operation_history: + toggle_type = "enable" if ('data' in locals() and data.get('enabled')) else "disable" + api_v3.operation_history.record_operation( + toggle_type, + plugin_id=data.get('plugin_id') if 'data' in locals() else None, + status="failed", + error=str(e) + ) + return error_response( + error.error_code, + error.message, + details=error.details, + context=error.context, + status_code=500 + ) +@api_v3.route('/plugins/operation/', methods=['GET']) +def get_operation_status(operation_id): + """Get status of a plugin operation""" + try: + if not api_v3.operation_queue: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'Operation queue not initialized', + status_code=500 + ) + + operation = api_v3.operation_queue.get_operation_status(operation_id) + if not operation: + return error_response( + ErrorCode.PLUGIN_NOT_FOUND, + f'Operation {operation_id} not found', + status_code=404 + ) + + return success_response(data=operation.to_dict()) + except Exception as e: + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) + return error_response( + error.error_code, + error.message, + details=error.details, + status_code=500 + ) +@api_v3.route('/plugins/operation/history', methods=['GET']) +def get_operation_history() -> Response: + """Get operation history from the audit log.""" + if not api_v3.operation_history: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'Operation history not initialized', + status_code=500 + ) + + try: + limit = request.args.get('limit', 50, type=int) + plugin_id = request.args.get('plugin_id') + operation_type = request.args.get('operation_type') + except (ValueError, TypeError) as e: + return error_response(ErrorCode.INVALID_INPUT, f'Invalid query parameter: {e}', status_code=400) + + try: + history = api_v3.operation_history.get_history( + limit=limit, + plugin_id=plugin_id, + operation_type=operation_type + ) + except (AttributeError, RuntimeError) as e: + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) + return error_response(error.error_code, error.message, details=error.details, status_code=500) + + return success_response(data=[record.to_dict() for record in history]) +@api_v3.route('/plugins/operation/history', methods=['DELETE']) +def clear_operation_history() -> Response: + """Clear operation history.""" + if not api_v3.operation_history: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'Operation history not initialized', + status_code=500 + ) + + try: + api_v3.operation_history.clear_history() + except (OSError, RuntimeError) as e: + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) + return error_response(error.error_code, error.message, details=error.details, status_code=500) + + return success_response(message='Operation history cleared') +@api_v3.route('/plugins/state', methods=['GET']) +def get_plugin_state(): + """Get plugin state from state manager""" + try: + if not api_v3.plugin_state_manager: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'State manager not initialized', + status_code=500 + ) + + plugin_id = request.args.get('plugin_id') + + if plugin_id: + # Get state for specific plugin + state = api_v3.plugin_state_manager.get_plugin_state(plugin_id) + if not state: + return error_response( + ErrorCode.PLUGIN_NOT_FOUND, + f'Plugin {plugin_id} not found in state manager', + context={'plugin_id': plugin_id}, + status_code=404 + ) + return success_response(data=state.to_dict()) + else: + # Get all plugin states + all_states = api_v3.plugin_state_manager.get_all_states() + return success_response(data={ + plugin_id: state.to_dict() + for plugin_id, state in all_states.items() + }) + except Exception as e: + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) + return error_response( + error.error_code, + error.message, + details=error.details, + context=error.context, + status_code=500 + ) +@api_v3.route('/plugins/state/reconcile', methods=['POST']) +def reconcile_plugin_state(): + """Reconcile plugin state across all sources""" + try: + if not api_v3.plugin_state_manager or not api_v3.plugin_manager: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'State manager or plugin manager not initialized', + status_code=500 + ) + + from src.plugin_system.state_reconciliation import StateReconciliation + + # Parse optional `force` flag from request body, guarding against + # non-dict bodies (bare string, array, null) that would raise AttributeError. + payload = request.get_json(silent=True) + if not isinstance(payload, dict): + payload = {} + force = _coerce_to_bool(payload.get('force', False)) + + reconciler = StateReconciliation( + state_manager=api_v3.plugin_state_manager, + config_manager=api_v3.config_manager, + plugin_manager=api_v3.plugin_manager, + plugins_dir=Path(api_v3.plugin_manager.plugins_dir) + ) + + result = reconciler.reconcile_state(force=force) + + return success_response( + data={ + 'inconsistencies_found': len(result.inconsistencies_found), + 'inconsistencies_fixed': len(result.inconsistencies_fixed), + 'inconsistencies_manual': len(result.inconsistencies_manual), + 'inconsistencies': [ + { + 'plugin_id': inc.plugin_id, + 'type': inc.inconsistency_type.value, + 'description': inc.description, + 'fix_action': inc.fix_action.value + } + for inc in result.inconsistencies_found + ], + 'fixed': [ + { + 'plugin_id': inc.plugin_id, + 'type': inc.inconsistency_type.value, + 'description': inc.description + } + for inc in result.inconsistencies_fixed + ], + 'manual_fix_required': [ + { + 'plugin_id': inc.plugin_id, + 'type': inc.inconsistency_type.value, + 'description': inc.description + } + for inc in result.inconsistencies_manual + ] + }, + message=result.message + ) + except Exception as e: + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.SYSTEM_ERROR) + return error_response( + error.error_code, + error.message, + details=error.details, + context=error.context, + status_code=500 + ) +def _drop_stale_reconciliation_findings(unresolved): + """Re-check a stored reconciliation verdict against current state. + + The verdict is a snapshot written once per run, and a run that could not + apply a fix also refuses to retry -- so a resolved condition was reported + indefinitely. Best-effort: any failure here returns the list untouched, + because showing a stale warning beats failing the endpoint. + + Both sets come from the reconciliation module's own extractors rather than + being re-derived here. That matters for correctness, not tidiness: a plain + set(load_config()) also contains system keys, the secrets-file keys merged + in by load_config(), and non-dict values, and any directory holding a + manifest.json would count as installed even if that manifest does not + parse. Either looseness clears findings that are still true -- and a secrets + key read as a plugin is the very bug the filter exists to stop reporting. + """ + try: + from src.plugin_system.state_reconciliation import ( + config_plugin_ids, disk_plugin_ids, ignored_config_keys, + still_unresolved, + ) + + cm = api_v3.config_manager + config_keys = config_plugin_ids(cm.load_config() or {}, + ignored_config_keys(cm)) + plugins_dir = getattr(api_v3.plugin_manager, 'plugins_dir', None) + installed = disk_plugin_ids(plugins_dir) if plugins_dir else set() + return still_unresolved(unresolved, config_keys, installed) + except Exception: + logger.debug("[Reconciliation] Could not re-check stored findings", exc_info=True) + return unresolved + + +@api_v3.route('/plugins/reconciliation-status', methods=['GET']) +def get_reconciliation_status(): + """Return the result of the last startup reconciliation from /tmp status file.""" + _recon_path = os.path.join(tempfile.gettempdir(), "ledmatrix_reconciliation.json") + try: + st = os.lstat(_recon_path) + except FileNotFoundError: + return jsonify({'status': 'success', 'data': {'done': False, 'unresolved': []}}) + if stat.S_ISLNK(st.st_mode) or not stat.S_ISREG(st.st_mode): + logger.warning("[Reconciliation] Status file is not a regular file: %s", _recon_path) + return jsonify({'status': 'success', 'data': {'done': False, 'unresolved': []}}) + try: + with open(_recon_path) as _f: + data = json.load(_f) + if data.get('unresolved'): + data['unresolved'] = _drop_stale_reconciliation_findings(data['unresolved']) + return jsonify({'status': 'success', 'data': data}) + except json.JSONDecodeError: + logger.exception("[Reconciliation] Failed to parse status file: %s", _recon_path) + return jsonify({'status': 'success', 'data': {'done': False, 'unresolved': []}}) + except PermissionError: + logger.exception("[Reconciliation] Permission denied reading status file: %s", _recon_path) + return jsonify({'status': 'success', 'data': {'done': False, 'unresolved': []}}) +@api_v3.route('/plugins/config', methods=['GET']) +def get_plugin_config(): + """Get plugin configuration""" + try: + if not api_v3.config_manager: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'Config manager not initialized', + status_code=500 + ) + + plugin_id = request.args.get('plugin_id') + if not plugin_id: + return error_response( + ErrorCode.INVALID_INPUT, + 'plugin_id required', + context={'missing_params': ['plugin_id']}, + status_code=400 + ) + + # Get plugin configuration from config manager + main_config = api_v3.config_manager.load_config() + plugin_config = main_config.get(plugin_id, {}) + + # Merge with defaults from schema so form shows default values for missing fields + schema_mgr = api_v3.schema_manager + if schema_mgr: + try: + defaults = schema_mgr.generate_default_config(plugin_id, use_cache=True) + plugin_config = schema_mgr.merge_with_defaults(plugin_config, defaults) + except Exception as e: + # Log but don't fail - defaults merge is best effort + import logging + logging.warning(f"Could not merge defaults for {plugin_id}: {e}") + + # Special handling for of-the-day plugin: populate uploaded_files and categories from disk + if plugin_id == 'of-the-day' or plugin_id == 'ledmatrix-of-the-day': + # Get plugin directory - plugin_id in manifest is 'of-the-day', but directory is 'ledmatrix-of-the-day' + plugin_dir_name = 'ledmatrix-of-the-day' + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_dir_name) + # If not found, try with the plugin_id + if not plugin_dir or not Path(plugin_dir).exists(): + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + else: + plugin_dir = PROJECT_ROOT / 'plugins' / plugin_dir_name + if not plugin_dir.exists(): + plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id + + if plugin_dir and Path(plugin_dir).exists(): + data_dir = Path(plugin_dir) / 'of_the_day' + if data_dir.exists(): + # Scan for JSON files + uploaded_files = [] + categories_from_files = {} + + for json_file in data_dir.glob('*.json'): + try: + # Get file stats + stat = json_file.stat() + + # Read JSON to count entries + with open(json_file, 'r', encoding='utf-8') as f: + json_data = json.load(f) + entry_count = len(json_data) if isinstance(json_data, dict) else 0 + + # Extract category name from filename + category_name = json_file.stem + filename = json_file.name + + # Create file entry + file_entry = { + 'id': category_name, + 'category_name': category_name, + 'filename': filename, + 'original_filename': filename, + 'path': f'of_the_day/{filename}', + 'size': stat.st_size, + 'uploaded_at': datetime.fromtimestamp(stat.st_mtime).isoformat() + 'Z', + 'entry_count': entry_count + } + uploaded_files.append(file_entry) + + # Create/update category entry if not in config + if category_name not in plugin_config.get('categories', {}): + display_name = category_name.replace('_', ' ').title() + categories_from_files[category_name] = { + 'enabled': False, # Default to disabled, user can enable + 'data_file': f'of_the_day/{filename}', + 'display_name': display_name + } + else: + # Update with file info if needed + categories_from_files[category_name] = plugin_config['categories'][category_name] + # Ensure data_file is correct + categories_from_files[category_name]['data_file'] = f'of_the_day/{filename}' + + except Exception as e: + logger.debug("Could not read json file: %s", e) + continue + + # Update plugin_config with scanned files + if uploaded_files: + plugin_config['uploaded_files'] = uploaded_files + + # Merge categories from files with existing config + # Start with existing categories (preserve user settings like enabled/disabled) + existing_categories = plugin_config.get('categories', {}).copy() + + # Update existing categories with file info, add new ones from files + for cat_name, cat_data in categories_from_files.items(): + if cat_name in existing_categories: + # Preserve existing enabled state and display_name, but update data_file path + existing_categories[cat_name]['data_file'] = cat_data['data_file'] + if 'display_name' not in existing_categories[cat_name] or not existing_categories[cat_name]['display_name']: + existing_categories[cat_name]['display_name'] = cat_data['display_name'] + else: + # Add new category from file (default to disabled) + existing_categories[cat_name] = cat_data + + if existing_categories: + plugin_config['categories'] = existing_categories + + # Update category_order to include all categories + category_order = plugin_config.get('category_order', []).copy() + all_category_names = set(existing_categories.keys()) + for cat_name in all_category_names: + if cat_name not in category_order: + category_order.append(cat_name) + if category_order: + plugin_config['category_order'] = category_order + + # If no config exists, return defaults + if not plugin_config: + plugin_config = { + 'enabled': True, + 'display_duration': 30 + } + + return success_response(data=plugin_config) + except Exception as e: + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.CONFIG_LOAD_FAILED) + return error_response( + error.error_code, + error.message, + details=error.details, + context=error.context, + status_code=500 + ) +@api_v3.route('/plugins/update', methods=['POST']) +def update_plugin(): + """Update plugin""" + try: + # Support both JSON and form data + content_type = request.content_type or '' + + if 'application/json' in content_type: + # JSON request + data, error = validate_request_json(['plugin_id']) + if error: + logger.debug("[UPDATE] JSON validation failed. Content-Type: %s", content_type) + return error + else: + # Form data or query string + plugin_id = request.args.get('plugin_id') or request.form.get('plugin_id') + if not plugin_id: + logger.debug("[UPDATE] Missing plugin_id. Content-Type: %s", content_type) + return error_response( + ErrorCode.INVALID_INPUT, + 'plugin_id required', + status_code=400 + ) + data = {'plugin_id': plugin_id} + + if not api_v3.plugin_store_manager: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'Plugin store manager not initialized', + status_code=500 + ) + + plugin_id = data['plugin_id'] + + # Always do direct updates (they're fast git pull operations) + # Operation queue is reserved for longer operations like install/uninstall + plugin_dir = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id + manifest_path = plugin_dir / "manifest.json" + + current_last_updated = None + current_commit = None + current_branch = None + + if manifest_path.exists(): + try: + import json + with open(manifest_path, 'r', encoding='utf-8') as f: + manifest = json.load(f) + current_last_updated = manifest.get('last_updated') + if manifest.get('local_only'): + logger.debug("Skipping update for local-only plugin: %s", plugin_id) + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "update", + plugin_id=plugin_id, + status="skipped", + details={"reason": "local_only"} + ) + return success_response(message=f'Plugin {plugin_id} is managed locally and does not receive registry updates') + except Exception as e: + logger.debug("Could not read local manifest for plugin: %s", e) + + if api_v3.plugin_store_manager: + git_info_before = api_v3.plugin_store_manager._get_local_git_info(plugin_dir) + if git_info_before: + current_commit = git_info_before.get('sha') + current_branch = git_info_before.get('branch') + + # Check if plugin is a git repo first (for better error messages) + plugin_path_dir = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id + is_git_repo = False + if plugin_path_dir.exists(): + git_info = api_v3.plugin_store_manager._get_local_git_info(plugin_path_dir) + is_git_repo = git_info is not None + if is_git_repo: + logger.debug("Plugin is a git repository, will update via git pull") + + remote_info = api_v3.plugin_store_manager.get_plugin_info(plugin_id, fetch_latest_from_github=True) + remote_commit = remote_info.get('last_commit_sha') if remote_info else None + remote_branch = remote_info.get('branch') if remote_info else None + + # Update the plugin + success = api_v3.plugin_store_manager.update_plugin(plugin_id) + + if success: + updated_last_updated = current_last_updated + try: + if manifest_path.exists(): + import json + with open(manifest_path, 'r', encoding='utf-8') as f: + manifest = json.load(f) + updated_last_updated = manifest.get('last_updated', current_last_updated) + except Exception as e: + logger.debug("Could not read updated manifest after update: %s", e) + + updated_commit = None + updated_branch = remote_branch or current_branch + if api_v3.plugin_store_manager: + git_info_after = api_v3.plugin_store_manager._get_local_git_info(plugin_dir) + if git_info_after: + updated_commit = git_info_after.get('sha') + updated_branch = git_info_after.get('branch') or updated_branch + + message = f'Plugin {plugin_id} updated successfully' + if current_commit and updated_commit and current_commit == updated_commit: + message = f'Plugin {plugin_id} already up to date (commit {updated_commit[:7]})' + elif updated_commit: + message = f'Plugin {plugin_id} updated to commit {updated_commit[:7]}' + if updated_branch: + message += f' on branch {updated_branch}' + elif updated_last_updated and updated_last_updated != current_last_updated: + message = f'Plugin {plugin_id} refreshed (Last Updated {updated_last_updated})' + + remote_commit_short = remote_commit[:7] if remote_commit else None + if remote_commit_short and updated_commit and remote_commit_short != updated_commit[:7]: + message += f' (remote latest {remote_commit_short})' + + # Invalidate schema cache + if api_v3.schema_manager: + api_v3.schema_manager.invalidate_cache(plugin_id) + + # Rediscover plugins + if api_v3.plugin_manager: + api_v3.plugin_manager.discover_plugins() + if plugin_id in api_v3.plugin_manager.plugins: + api_v3.plugin_manager.reload_plugin(plugin_id) + + # Update state and history + if api_v3.plugin_state_manager: + api_v3.plugin_state_manager.update_plugin_state( + plugin_id, + {'last_updated': datetime.now()} + ) + if api_v3.operation_history: + version = _get_plugin_version(plugin_id) + api_v3.operation_history.record_operation( + "update", + plugin_id=plugin_id, + status="success", + details={ + "version": version, + "previous_commit": current_commit[:7] if current_commit else None, + "commit": updated_commit[:7] if updated_commit else None, + "branch": updated_branch + } + ) + + return success_response( + data={ + 'last_updated': updated_last_updated, + 'commit': updated_commit + }, + message=message + ) + else: + plugin_path_dir = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id + if not plugin_path_dir.exists(): + client_msg = 'Plugin update failed: plugin not found' + else: + git_info = api_v3.plugin_store_manager._get_local_git_info(plugin_path_dir) + if not git_info: + plugin_info = api_v3.plugin_store_manager.get_plugin_info(plugin_id) + if not plugin_info: + client_msg = 'Plugin update failed: not found in registry' + else: + client_msg = 'Plugin update failed; check logs for details' + else: + client_msg = 'Plugin update failed; check logs for details' + logger.error("update_plugin failed for plugin_id=%s: %s", plugin_id, client_msg) + + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "update", + plugin_id=plugin_id, + status="failed", + error=client_msg, + details={ + "previous_commit": current_commit[:7] if current_commit else None, + "branch": current_branch + } + ) + + return error_response( + ErrorCode.PLUGIN_UPDATE_FAILED, + client_msg, + status_code=500 + ) + + except Exception as e: + logger.error("Unhandled exception in update endpoint", exc_info=True) + + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.PLUGIN_UPDATE_FAILED) + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "update", + plugin_id=data.get('plugin_id') if 'data' in locals() else None, + status="failed", + error=str(e) + ) + return error_response( + error.error_code, + error.message, + details=error.details, + context=error.context, + status_code=500 + ) +@api_v3.route('/plugins/uninstall', methods=['POST']) +def uninstall_plugin(): + """Uninstall plugin""" + try: + # Validate request + data, error = validate_request_json(['plugin_id']) + if error: + return error + + if not api_v3.plugin_store_manager: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'Plugin store manager not initialized', + status_code=500 + ) + + plugin_id = data['plugin_id'] + preserve_config = data.get('preserve_config', False) + + # Both queued and direct paths use the same transactional helper so + # snapshot/rollback behaviour is consistent regardless of deployment. + if api_v3.operation_queue: + def uninstall_callback(operation): + """Callback to execute plugin uninstallation via transactional helper.""" + success, error_msg = _do_transactional_uninstall(plugin_id, preserve_config) + if not success: + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "uninstall", + plugin_id=plugin_id, + status="failed", + error=error_msg + ) + raise Exception(error_msg or f'Failed to uninstall plugin {plugin_id}') + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "uninstall", + plugin_id=plugin_id, + status="success", + details={"preserve_config": preserve_config} + ) + return {'success': True, 'message': 'Plugin uninstalled successfully'} + + # Enqueue operation + operation_id = api_v3.operation_queue.enqueue_operation( + OperationType.UNINSTALL, + plugin_id, + operation_callback=uninstall_callback + ) + + return success_response( + data={'operation_id': operation_id}, + message='Plugin uninstallation queued' + ) + else: + # Direct (non-queued) transactional uninstall + success, error_msg = _do_transactional_uninstall(plugin_id, preserve_config) + + if success: + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "uninstall", + plugin_id=plugin_id, + status="success", + details={"preserve_config": preserve_config} + ) + return success_response(message='Plugin uninstalled successfully') + else: + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "uninstall", + plugin_id=plugin_id, + status="failed", + error=error_msg + ) + return error_response( + ErrorCode.PLUGIN_UNINSTALL_FAILED, + error_msg or 'Plugin uninstall failed', + status_code=500 + ) + + except Exception as e: + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.PLUGIN_UNINSTALL_FAILED) + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "uninstall", + plugin_id=data.get('plugin_id') if 'data' in locals() else None, + status="failed", + error=str(e) + ) + return error_response( + error.error_code, + error.message, + details=error.details, + context=error.context, + status_code=500 + ) +@api_v3.route('/plugins/install', methods=['POST']) +def install_plugin(): + """Install plugin from store""" + try: + if not api_v3.plugin_store_manager: + return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 + + data = request.get_json(silent=True) + if not data or 'plugin_id' not in data: + return jsonify({'status': 'error', 'message': 'plugin_id required'}), 400 + + plugin_id = data['plugin_id'] + branch = data.get('branch') # Optional branch parameter + + # Install the plugin + # Log the plugins directory being used for debugging + plugins_dir = api_v3.plugin_store_manager.plugins_dir + branch_info = f" (branch: {branch})" if branch else "" + logger.info("Installing plugin to directory: %s", plugins_dir) + + # Use operation queue if available + if api_v3.operation_queue: + def install_callback(operation): + """Callback to execute plugin installation.""" + success = api_v3.plugin_store_manager.install_plugin(plugin_id, branch=branch) + + if success: + # Invalidate schema cache + if api_v3.schema_manager: + api_v3.schema_manager.invalidate_cache(plugin_id) + + # Discover and load the new plugin + if api_v3.plugin_manager: + api_v3.plugin_manager.discover_plugins() + api_v3.plugin_manager.load_plugin(plugin_id) + + # Update state manager + if api_v3.plugin_state_manager: + api_v3.plugin_state_manager.set_plugin_installed(plugin_id) + + # Record in history + if api_v3.operation_history: + version = _get_plugin_version(plugin_id) + api_v3.operation_history.record_operation( + "install", + plugin_id=plugin_id, + status="success", + details={"version": version, "branch": branch} + ) + + branch_msg = f" (branch: {branch})" if branch else "" + return {'success': True, 'message': f'Plugin {plugin_id} installed successfully{branch_msg}'} + else: + error_msg = f'Failed to install plugin {plugin_id}' + if branch: + error_msg += f' (branch: {branch})' + plugin_info = api_v3.plugin_store_manager.get_plugin_info(plugin_id) + if not plugin_info: + error_msg += ' (plugin not found in registry)' + + # Record failure in history + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "install", + plugin_id=plugin_id, + status="failed", + error=error_msg, + details={"branch": branch} + ) + + raise Exception(error_msg) + + # Enqueue operation + operation_id = api_v3.operation_queue.enqueue_operation( + OperationType.INSTALL, + plugin_id, + operation_callback=install_callback + ) + + branch_msg = f" (branch: {branch})" if branch else "" + return success_response( + data={'operation_id': operation_id}, + message=f'Plugin {plugin_id} installation queued{branch_msg}' + ) + else: + # Fallback to direct installation + success = api_v3.plugin_store_manager.install_plugin(plugin_id, branch=branch) + + if success: + if api_v3.schema_manager: + api_v3.schema_manager.invalidate_cache(plugin_id) + if api_v3.plugin_manager: + api_v3.plugin_manager.discover_plugins() + api_v3.plugin_manager.load_plugin(plugin_id) + if api_v3.plugin_state_manager: + api_v3.plugin_state_manager.set_plugin_installed(plugin_id) + if api_v3.operation_history: + version = _get_plugin_version(plugin_id) + api_v3.operation_history.record_operation( + "install", + plugin_id=plugin_id, + status="success", + details={"version": version, "branch": branch} + ) + + branch_msg = f" (branch: {branch})" if branch else "" + return success_response(message=f'Plugin installed successfully{branch_msg}') + else: + error_msg = f'Failed to install plugin {plugin_id}' + if branch: + error_msg += f' (branch: {branch})' + plugin_info = api_v3.plugin_store_manager.get_plugin_info(plugin_id) + if not plugin_info: + error_msg += ' (plugin not found in registry)' + + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "install", + plugin_id=plugin_id, + status="failed", + error=error_msg, + details={"branch": branch} + ) + + return error_response( + ErrorCode.PLUGIN_INSTALL_FAILED, + error_msg, + status_code=500 + ) + + except Exception as e: + logger.error('Error in install_plugin', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/install-from-url', methods=['POST']) +def install_plugin_from_url(): + """Install plugin from custom GitHub URL""" + try: + if not api_v3.plugin_store_manager: + return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 + + data = request.get_json(silent=True) + if not data or 'repo_url' not in data: + return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 + + # A non-string repo_url is a client mistake, not a server fault: + # .strip() would raise and the catch-all would report it as a 500. + if not isinstance(data['repo_url'], str) or not data['repo_url'].strip(): + return jsonify({'status': 'error', 'message': 'repo_url must be a non-empty string'}), 400 + + repo_url = data['repo_url'].strip() + plugin_id = data.get('plugin_id') # Optional, for monorepo installations + plugin_path = data.get('plugin_path') # Optional, for monorepo subdirectory + branch = data.get('branch') # Optional branch parameter + + # Install the plugin + result = api_v3.plugin_store_manager.install_from_url( + repo_url=repo_url, + plugin_id=plugin_id, + plugin_path=plugin_path, + branch=branch + ) + + if result.get('success'): + # Invalidate schema cache for the installed plugin + installed_plugin_id = result.get('plugin_id') + if api_v3.schema_manager and installed_plugin_id: + api_v3.schema_manager.invalidate_cache(installed_plugin_id) + + # Discover and load the new plugin + if api_v3.plugin_manager and installed_plugin_id: + api_v3.plugin_manager.discover_plugins() + api_v3.plugin_manager.load_plugin(installed_plugin_id) + + branch_msg = f" (branch: {result.get('branch', branch)})" if (result.get('branch') or branch) else "" + response_data = { + 'status': 'success', + 'message': f"Plugin {installed_plugin_id} installed successfully{branch_msg}", + 'plugin_id': installed_plugin_id, + 'name': result.get('name') + } + if result.get('branch'): + response_data['branch'] = result.get('branch') + return jsonify(response_data) + else: + return jsonify({ + 'status': 'error', + 'message': result.get('error', 'Failed to install plugin from URL') + }), 500 + + except Exception as e: + logger.error('Error in install_plugin_from_url', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/registry-from-url', methods=['POST']) +def get_registry_from_url(): + """Get plugin list from a registry-style monorepo URL""" + try: + if not api_v3.plugin_store_manager: + return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 + + data = request.get_json(silent=True) + if not data or 'repo_url' not in data: + return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 + + # A non-string repo_url is a client mistake, not a server fault: + # .strip() would raise and the catch-all would report it as a 500. + if not isinstance(data['repo_url'], str) or not data['repo_url'].strip(): + return jsonify({'status': 'error', 'message': 'repo_url must be a non-empty string'}), 400 + + repo_url = data['repo_url'].strip() + + # Get registry from the URL + registry = api_v3.plugin_store_manager.fetch_registry_from_url(repo_url) + + if registry: + return jsonify({ + 'status': 'success', + 'plugins': registry.get('plugins', []), + 'registry_url': repo_url + }) + else: + return jsonify({ + 'status': 'error', + 'message': 'Failed to fetch registry from URL or URL does not contain a valid registry' + }), 400 + + except Exception as e: + logger.error('Error in get_registry_from_url', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/saved-repositories', methods=['GET']) +def get_saved_repositories(): + """Get all saved repositories""" + try: + if not api_v3.saved_repositories_manager: + return jsonify({'status': 'error', 'message': 'Saved repositories manager not initialized'}), 500 + + repositories = api_v3.saved_repositories_manager.get_all() + return jsonify({'status': 'success', 'data': {'repositories': repositories}}) + except Exception as e: + logger.error('Error in get_saved_repositories', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/saved-repositories', methods=['POST']) +def add_saved_repository(): + """Add a repository to saved list""" + try: + if not api_v3.saved_repositories_manager: + return jsonify({'status': 'error', 'message': 'Saved repositories manager not initialized'}), 500 + + data = request.get_json(silent=True) + if not data or 'repo_url' not in data: + return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 + + # A non-string repo_url is a client mistake, not a server fault: + # .strip() would raise and the catch-all would report it as a 500. + if not isinstance(data['repo_url'], str) or not data['repo_url'].strip(): + return jsonify({'status': 'error', 'message': 'repo_url must be a non-empty string'}), 400 + + repo_url = data['repo_url'].strip() + name = data.get('name') + + success = api_v3.saved_repositories_manager.add(repo_url, name) + + if success: + return jsonify({ + 'status': 'success', + 'message': 'Repository saved successfully', + 'data': {'repositories': api_v3.saved_repositories_manager.get_all()} + }) + else: + return jsonify({ + 'status': 'error', + 'message': 'Repository already exists or failed to save' + }), 400 + except Exception as e: + logger.error('Error in add_saved_repository', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/saved-repositories', methods=['DELETE']) +def remove_saved_repository(): + """Remove a repository from saved list""" + try: + if not api_v3.saved_repositories_manager: + return jsonify({'status': 'error', 'message': 'Saved repositories manager not initialized'}), 500 + + data = request.get_json(silent=True) + if not data or 'repo_url' not in data: + return jsonify({'status': 'error', 'message': 'repo_url required'}), 400 + + repo_url = data['repo_url'] + + success = api_v3.saved_repositories_manager.remove(repo_url) + + if success: + return jsonify({ + 'status': 'success', + 'message': 'Repository removed successfully', + 'data': {'repositories': api_v3.saved_repositories_manager.get_all()} + }) + else: + return jsonify({ + 'status': 'error', + 'message': 'Repository not found' + }), 404 + except Exception as e: + logger.error('Error in remove_saved_repository', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/store/list', methods=['GET']) +def list_plugin_store(): + """Search plugin store""" + try: + if not api_v3.plugin_store_manager: + return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 + + query = request.args.get('query', '') + category = request.args.get('category', '') + tags = request.args.getlist('tags') + # Default to fetching commit metadata to ensure accurate commit timestamps + fetch_commit_param = request.args.get('fetch_commit_info', request.args.get('fetch_latest_versions', '')).lower() + fetch_commit = fetch_commit_param != 'false' + + # Search plugins from the registry (including saved repositories) + plugins = api_v3.plugin_store_manager.search_plugins( + query=query, + category=category, + tags=tags, + fetch_commit_info=fetch_commit, + include_saved_repos=True, + saved_repositories_manager=api_v3.saved_repositories_manager + ) + + # Format plugins for the web interface + formatted_plugins = [] + for plugin in plugins: + formatted_plugins.append({ + 'id': plugin.get('id'), + 'name': plugin.get('name'), + 'author': plugin.get('author'), + 'category': plugin.get('category'), + 'description': plugin.get('description'), + 'tags': plugin.get('tags', []), + 'stars': plugin.get('stars', 0), + 'verified': plugin.get('verified', False), + 'repo': plugin.get('repo', ''), + 'last_updated': plugin.get('last_updated') or plugin.get('last_updated_iso', ''), + 'last_updated_iso': plugin.get('last_updated_iso', ''), + 'last_commit': plugin.get('last_commit') or plugin.get('last_commit_sha'), + 'last_commit_message': plugin.get('last_commit_message'), + 'last_commit_author': plugin.get('last_commit_author'), + 'version': plugin.get('latest_version') or plugin.get('version', ''), + 'branch': plugin.get('branch') or plugin.get('default_branch'), + 'default_branch': plugin.get('default_branch'), + 'plugin_path': plugin.get('plugin_path', '') + }) + + return jsonify({'status': 'success', 'data': {'plugins': formatted_plugins}}) + except Exception as e: + logger.error('Error in list_plugin_store', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/store/github-status', methods=['GET']) +def get_github_auth_status(): + """Check if GitHub authentication is configured and validate token""" + try: + if not api_v3.plugin_store_manager: + return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 + + token = api_v3.plugin_store_manager.github_token + + # Check if GitHub token is configured + if not token or len(token) == 0: + return jsonify({ + 'status': 'success', + 'data': { + 'token_status': 'none', + 'authenticated': False, + 'rate_limit': 60, + 'message': 'No GitHub token configured', + 'error': None + } + }) + + # Validate the token + is_valid, error_message = api_v3.plugin_store_manager._validate_github_token(token) + + if is_valid: + return jsonify({ + 'status': 'success', + 'data': { + 'token_status': 'valid', + 'authenticated': True, + 'rate_limit': 5000, + 'message': 'GitHub API authenticated', + 'error': None + } + }) + else: + return jsonify({ + 'status': 'success', + 'data': { + 'token_status': 'invalid', + 'authenticated': False, + 'rate_limit': 60, + 'message': f'GitHub token is invalid: {error_message}' if error_message else 'GitHub token is invalid', + 'error': error_message + } + }) + except Exception as e: + logger.error('Error in get_github_auth_status', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/store/refresh', methods=['POST']) +def refresh_plugin_store(): + """Refresh plugin store repository""" + try: + if not api_v3.plugin_store_manager: + return jsonify({'status': 'error', 'message': 'Plugin store manager not initialized'}), 500 + + data = request.get_json(silent=True) or {} + fetch_commit_info = data.get('fetch_commit_info', data.get('fetch_latest_versions', False)) + + # Force refresh the registry + registry = api_v3.plugin_store_manager.fetch_registry(force_refresh=True) + plugin_count = len(registry.get('plugins', [])) + + message = 'Plugin store refreshed' + if fetch_commit_info: + message += ' (with refreshed commit metadata from GitHub)' + + return jsonify({ + 'status': 'success', + 'message': message, + 'plugin_count': plugin_count + }) + except Exception as e: + logger.error('Error in refresh_plugin_store', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/config', methods=['POST']) +def save_plugin_config(): + """Save plugin configuration, separating secrets from regular config""" + try: + if not api_v3.config_manager: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'Config manager not initialized', + status_code=500 + ) + + # Support both JSON and form data (for HTMX submissions) + content_type = request.content_type or '' + + if 'application/json' in content_type: + # JSON request + data, error = validate_request_json(['plugin_id']) + if error: + return error + plugin_id = data['plugin_id'] + plugin_config = data.get('config', {}) + else: + # Form data (HTMX submission) + # plugin_id comes from query string, config from form fields + plugin_id = request.args.get('plugin_id') + if not plugin_id: + return error_response( + ErrorCode.INVALID_INPUT, + 'plugin_id required in query string', + status_code=400 + ) + + # Load existing config as base (partial form updates should merge, not replace) + existing_config = {} + if api_v3.config_manager: + full_config = api_v3.config_manager.load_config() + existing_config = full_config.get(plugin_id, {}).copy() + + # Get schema manager instance (needed for type conversion) + schema_mgr = api_v3.schema_manager + if not schema_mgr: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'Schema manager not initialized', + status_code=500 + ) + + # Load plugin schema BEFORE processing form data (needed for type conversion) + schema = schema_mgr.load_schema(plugin_id, use_cache=False) + + # Start with existing config and apply form updates + plugin_config = existing_config + + # Convert form data to config dict + # Form fields can use dot notation for nested values (e.g., "transition.type") + form_data = request.form.to_dict() + + # First pass: handle bracket notation array fields (e.g., "field_name[]" from checkbox-group) + # These fields use getlist() to preserve all values, then replace in form_data + # Sentinel empty value ("") allows clearing array to [] when all checkboxes unchecked + bracket_array_fields = {} # Maps base field path to list of values + for key in request.form.keys(): + # Check if key ends with "[]" (bracket notation for array fields) + if key.endswith('[]'): + base_path = key[:-2] # Remove "[]" suffix + values = request.form.getlist(key) + # Filter out sentinel empty string - if only sentinel present, array should be [] + # If sentinel + values present, use the actual values + filtered_values = [v for v in values if v and v.strip()] + # If no non-empty values but key exists, it means all checkboxes unchecked (empty array) + bracket_array_fields[base_path] = filtered_values + # Remove the bracket notation key from form_data if present + if key in form_data: + del form_data[key] + + # Process bracket notation fields and set directly in plugin_config + # Use JSON encoding instead of comma-join to handle values containing commas + import json + for base_path, values in bracket_array_fields.items(): + # Get schema property to verify it's an array + base_prop = _get_schema_property(schema, base_path) + if base_prop and base_prop.get('type') == 'array': + # Filter out empty values and sentinel empty strings + filtered_values = [v for v in values if v and v.strip()] + # Set directly in plugin_config (values are already strings, no need to parse) + # Empty array (all unchecked) is represented as [] + _set_nested_value(plugin_config, base_path, filtered_values) + logger.debug(f"Processed bracket notation array field {base_path}: {values} -> {filtered_values}") + # Remove from form_data to avoid double processing + if base_path in form_data: + del form_data[base_path] + + # Second pass: detect and combine array index fields (e.g., "text_color.0", "text_color.1" -> "text_color" as array) + # This handles cases where forms send array fields as indexed inputs + array_fields = {} # Maps base field path to list of (index, value) tuples + processed_keys = set() + indexed_base_paths = set() # Track which base paths have indexed fields + + for key, value in form_data.items(): + # Check if this looks like an array index field (ends with .0, .1, .2, etc.) + if '.' in key: + parts = key.rsplit('.', 1) # Split on last dot + if len(parts) == 2: + base_path, last_part = parts + # Check if last part is a numeric string (array index) + if last_part.isdigit(): + # Get schema property for the base path to verify it's an array + base_prop = _get_schema_property(schema, base_path) + if base_prop and base_prop.get('type') == 'array': + # This is an array index field + index = int(last_part) + if base_path not in array_fields: + array_fields[base_path] = [] + array_fields[base_path].append((index, value)) + processed_keys.add(key) + indexed_base_paths.add(base_path) + continue + + # Process combined array fields + for base_path, index_values in array_fields.items(): + # Sort by index and extract values + index_values.sort(key=lambda x: x[0]) + values = [v for _, v in index_values] + # Combine values into comma-separated string for parsing + combined_value = ', '.join(str(v) for v in values) + # Parse as array using schema + parsed_value = _parse_form_value_with_schema(combined_value, base_path, schema) + # Debug logging + logger.debug(f"Combined indexed array field {base_path}: {values} -> {combined_value} -> {parsed_value}") + # Only set if not skipped + if parsed_value is not _SKIP_FIELD: + _set_nested_value(plugin_config, base_path, parsed_value) + + # Process remaining (non-indexed) fields + # Skip any base paths that were processed as indexed arrays + for key, value in form_data.items(): + if key not in processed_keys: + # Skip if this key is a base path that was processed as indexed array + # (to avoid overwriting the combined array with a single value) + if key not in indexed_base_paths: + # Parse value using schema to determine correct type + parsed_value = _parse_form_value_with_schema(value, key, schema) + # Debug logging for array fields + if schema: + prop = _get_schema_property(schema, key) + if prop and prop.get('type') == 'array': + logger.debug(f"Array field {key}: form value='{value}' -> parsed={parsed_value}") + # Use helper to set nested values correctly (skips if _SKIP_FIELD) + if parsed_value is not _SKIP_FIELD: + _set_nested_value(plugin_config, key, parsed_value) + + # Post-process: Fix array fields that might have been incorrectly structured + # This handles cases where array fields are stored as dicts (e.g., from indexed form fields) + def fix_array_structures(config_dict, schema_props, prefix=''): + """Recursively fix array structures (convert dicts with numeric keys to arrays, fix length issues)""" + for prop_key, prop_schema in schema_props.items(): + prop_type = prop_schema.get('type') + + if prop_type == 'array': + # Navigate to the field location + if prefix: + parent_parts = prefix.split('.') + parent = config_dict + for part in parent_parts: + if isinstance(parent, dict) and part in parent: + parent = parent[part] + else: + parent = None + break + + if parent is not None and isinstance(parent, dict) and prop_key in parent: + current_value = parent[prop_key] + # If it's a dict with numeric string keys, convert to array + if isinstance(current_value, dict) and not isinstance(current_value, list): + try: + # Check if all keys are numeric strings (array indices) + keys = [k for k in current_value.keys()] + if all(k.isdigit() for k in keys): + # Convert to sorted array by index + sorted_keys = sorted(keys, key=int) + array_value = [current_value[k] for k in sorted_keys] + # Convert array elements to correct types based on schema + items_schema = prop_schema.get('items', {}) + item_type = items_schema.get('type') + if item_type in ('number', 'integer'): + converted_array = [] + for v in array_value: + if isinstance(v, str): + try: + if item_type == 'integer': + converted_array.append(int(v)) + else: + converted_array.append(float(v)) + except (ValueError, TypeError, OverflowError): + converted_array.append(v) + else: + converted_array.append(v) + array_value = converted_array + parent[prop_key] = array_value + current_value = array_value # Update for length check below + except (ValueError, KeyError, TypeError): + # Conversion failed, check if we should use default + pass + + # If it's an array, ensure correct types and check minItems + if isinstance(current_value, list): + # First, ensure array elements are correct types + items_schema = prop_schema.get('items', {}) + item_type = items_schema.get('type') + if item_type in ('number', 'integer'): + converted_array = [] + for v in current_value: + if isinstance(v, str): + try: + if item_type == 'integer': + converted_array.append(int(v)) + else: + converted_array.append(float(v)) + except (ValueError, TypeError, OverflowError): + converted_array.append(v) + else: + converted_array.append(v) + parent[prop_key] = converted_array + current_value = converted_array + + # Then check minItems + min_items = prop_schema.get('minItems') + if min_items is not None and len(current_value) < min_items: + # Use default if available, otherwise keep as-is (validation will catch it) + default = prop_schema.get('default') + if default and isinstance(default, list) and len(default) >= min_items: + parent[prop_key] = default + else: + # Top-level field + if prop_key in config_dict: + current_value = config_dict[prop_key] + # If it's a dict with numeric string keys, convert to array + if isinstance(current_value, dict) and not isinstance(current_value, list): + try: + keys = list(current_value.keys()) + if keys and all(str(k).isdigit() for k in keys): + sorted_keys = sorted(keys, key=lambda x: int(str(x))) + array_value = [current_value[k] for k in sorted_keys] + # Convert array elements to correct types based on schema + items_schema = prop_schema.get('items', {}) + item_type = items_schema.get('type') + if item_type in ('number', 'integer'): + converted_array = [] + for v in array_value: + if isinstance(v, str): + try: + if item_type == 'integer': + converted_array.append(int(v)) + else: + converted_array.append(float(v)) + except (ValueError, TypeError, OverflowError): + converted_array.append(v) + else: + converted_array.append(v) + array_value = converted_array + config_dict[prop_key] = array_value + current_value = array_value # Update for length check below + except (ValueError, KeyError, TypeError) as e: + logger.debug(f"Failed to convert {prop_key} to array: {e}") + + # If it's an array, ensure correct types and check minItems + if isinstance(current_value, list): + # First, ensure array elements are correct types + items_schema = prop_schema.get('items', {}) + item_type = items_schema.get('type') + if item_type in ('number', 'integer'): + converted_array = [] + for v in current_value: + if isinstance(v, str): + try: + if item_type == 'integer': + converted_array.append(int(v)) + else: + converted_array.append(float(v)) + except (ValueError, TypeError, OverflowError): + converted_array.append(v) + else: + converted_array.append(v) + config_dict[prop_key] = converted_array + current_value = converted_array + + # Then check minItems + min_items = prop_schema.get('minItems') + if min_items is not None and len(current_value) < min_items: + default = prop_schema.get('default') + if default and isinstance(default, list) and len(default) >= min_items: + config_dict[prop_key] = default + + # Recurse into nested objects + elif prop_type == 'object' and 'properties' in prop_schema: + nested_prefix = f"{prefix}.{prop_key}" if prefix else prop_key + if prefix: + parent_parts = prefix.split('.') + parent = config_dict + for part in parent_parts: + if isinstance(parent, dict) and part in parent: + parent = parent[part] + else: + parent = None + break + nested_dict = parent.get(prop_key) if parent is not None and isinstance(parent, dict) else None + else: + nested_dict = config_dict.get(prop_key) + + if isinstance(nested_dict, dict): + # Pass no prefix: config_dict is already the navigated sub-dict, + # so path segments from the parent would mis-navigate it. + fix_array_structures(nested_dict, prop_schema['properties']) + + # Also ensure array fields that are None get converted to empty arrays + def ensure_array_defaults(config_dict, schema_props, prefix=''): + """Recursively ensure array fields have defaults if None""" + for prop_key, prop_schema in schema_props.items(): + prop_type = prop_schema.get('type') + + if prop_type == 'array': + if prefix: + parent_parts = prefix.split('.') + parent = config_dict + for part in parent_parts: + if isinstance(parent, dict) and part in parent: + parent = parent[part] + else: + parent = None + break + + if parent is not None and isinstance(parent, dict): + if prop_key not in parent or parent[prop_key] is None: + default = prop_schema.get('default', []) + parent[prop_key] = default if default else [] + else: + if prop_key not in config_dict or config_dict[prop_key] is None: + default = prop_schema.get('default', []) + config_dict[prop_key] = default if default else [] + + elif prop_type == 'object' and 'properties' in prop_schema: + nested_prefix = f"{prefix}.{prop_key}" if prefix else prop_key + if prefix: + parent_parts = prefix.split('.') + parent = config_dict + for part in parent_parts: + if isinstance(parent, dict) and part in parent: + parent = parent[part] + else: + parent = None + break + nested_dict = parent.get(prop_key) if parent is not None and isinstance(parent, dict) else None + else: + nested_dict = config_dict.get(prop_key) + + if nested_dict is None: + if prefix: + parent_parts = prefix.split('.') + parent = config_dict + for part in parent_parts: + if part not in parent: + parent[part] = {} + parent = parent[part] + if prop_key not in parent: + parent[prop_key] = {} + nested_dict = parent[prop_key] + else: + if prop_key not in config_dict: + config_dict[prop_key] = {} + nested_dict = config_dict[prop_key] + + if isinstance(nested_dict, dict): + # Pass no prefix: config_dict is already navigated. + ensure_array_defaults(nested_dict, prop_schema['properties']) + + if schema and 'properties' in schema: + # First, fix any dict structures that should be arrays + # This must be called BEFORE validation to convert dicts with numeric keys to arrays + fix_array_structures(plugin_config, schema['properties']) + # Then, ensure None arrays get defaults + ensure_array_defaults(plugin_config, schema['properties']) + + # Debug: Log the structure after fixing + if 'feeds' in plugin_config and 'custom_feeds' in plugin_config.get('feeds', {}): + custom_feeds = plugin_config['feeds']['custom_feeds'] + logger.debug(f"After fix_array_structures: custom_feeds type={type(custom_feeds)}, value={custom_feeds}") + + # Force fix for feeds.custom_feeds if it's still a dict (fallback) + if 'feeds' in plugin_config: + feeds_config = plugin_config.get('feeds') or {} + if feeds_config and 'custom_feeds' in feeds_config and isinstance(feeds_config['custom_feeds'], dict): + custom_feeds_dict = feeds_config['custom_feeds'] + # Check if all keys are numeric + keys = list(custom_feeds_dict.keys()) + if keys and all(str(k).isdigit() for k in keys): + # Convert to array + sorted_keys = sorted(keys, key=lambda x: int(str(x))) + feeds_config['custom_feeds'] = [custom_feeds_dict[k] for k in sorted_keys] + logger.info(f"Force-converted feeds.custom_feeds from dict to array: {len(feeds_config['custom_feeds'])} items") + + # Fix unchecked boolean checkboxes: HTML checkboxes don't submit values + # when unchecked, so the existing config value (potentially True) persists. + # Walk the schema and set any boolean fields missing from form data to False. + if schema and 'properties' in schema: + form_keys = set(request.form.keys()) + _set_missing_booleans_to_false(plugin_config, schema['properties'], form_keys) + + # Get schema manager instance (for JSON requests) + schema_mgr = api_v3.schema_manager + if not schema_mgr: + return error_response( + ErrorCode.SYSTEM_ERROR, + 'Schema manager not initialized', + status_code=500 + ) + + # Load plugin schema using SchemaManager (force refresh to get latest schema) + # For JSON requests, schema wasn't loaded yet + if 'application/json' in content_type: + schema = schema_mgr.load_schema(plugin_id, use_cache=False) + + # JSON path: fix numeric-keyed dicts that should be arrays. + # JS dotToNested() converts feeds.custom_feeds.0.name → {'0': {name:...}} + # instead of [{name:...}]. The form-data path has fix_array_structures for this; + # mirror that logic here for JSON submissions. + if 'application/json' in content_type and schema and 'properties' in schema: + def _fix_json_arrays(cfg, props): + for k, ps in props.items(): + if not isinstance(cfg, dict) or k not in cfg: + continue + pt = ps.get('type') + val = cfg[k] + if pt == 'array': + items_schema = ps.get('items', {}) + item_type = items_schema.get('type') + if isinstance(val, dict): + keys = list(val.keys()) + if keys and all(str(x).isdigit() for x in keys): + sorted_keys = sorted(keys, key=lambda x: int(str(x))) + arr = [val[sk] for sk in sorted_keys] + if item_type in ('integer', 'number'): + converted = [] + for v in arr: + if isinstance(v, str): + try: + converted.append(int(v) if item_type == 'integer' else float(v)) + except (ValueError, TypeError, OverflowError): + converted.append(v) + else: + converted.append(v) + arr = converted + cfg[k] = arr + elif not keys: + cfg[k] = [] + # Recurse into each element when items are objects with properties, + # covering both freshly-converted and already-list values. + if item_type == 'object' and 'properties' in items_schema: + for elem in (cfg[k] if isinstance(cfg[k], list) else []): + if isinstance(elem, dict): + _fix_json_arrays(elem, items_schema['properties']) + elif pt == 'object' and 'properties' in ps and isinstance(val, dict): + _fix_json_arrays(val, ps['properties']) + _fix_json_arrays(plugin_config, schema['properties']) + + # PRE-PROCESSING: Preserve 'enabled' state if not in request + # This prevents overwriting the enabled state when saving config from a form that doesn't include the toggle + if 'enabled' not in plugin_config: + try: + current_config = api_v3.config_manager.load_config() + if plugin_id in current_config and 'enabled' in current_config[plugin_id]: + plugin_config['enabled'] = current_config[plugin_id]['enabled'] + # logger.debug(f"Preserving enabled state for {plugin_id}: {plugin_config['enabled']}") + elif api_v3.plugin_manager: + # Fallback to plugin instance if config doesn't have it + plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) + if plugin_instance: + plugin_config['enabled'] = plugin_instance.enabled + # Final fallback: default to True if plugin is loaded (matches BasePlugin default) + if 'enabled' not in plugin_config: + plugin_config['enabled'] = True + except Exception as e: + logger.debug("Error preserving enabled state: %s", e) + # Default to True on error to avoid disabling plugins + plugin_config['enabled'] = True + + # Find secret fields (supports nested schemas and array-item secrets) + secret_fields = set() + + if schema and 'properties' in schema: + secret_fields = find_secret_fields(schema['properties']) + + # Apply defaults from schema to config BEFORE validation + # This ensures required fields with defaults are present before validation + # Store preserved enabled value before merge to protect it from defaults + preserved_enabled = None + if 'enabled' in plugin_config: + preserved_enabled = plugin_config['enabled'] + + if schema: + defaults = schema_mgr.generate_default_config(plugin_id, use_cache=True) + plugin_config = schema_mgr.merge_with_defaults(plugin_config, defaults) + + # After merging defaults, replace any None array values with their schema defaults. + # merge_with_defaults gives user config higher priority, so a None submitted by + # the client can survive the merge — this pass cleans those up. + def _fix_none_arrays(cfg, props): + for k, pschema in props.items(): + if pschema.get('type') == 'array': + if isinstance(cfg, dict) and (k not in cfg or cfg[k] is None): + cfg[k] = pschema.get('default', []) + elif pschema.get('type') == 'object' and 'properties' in pschema: + if isinstance(cfg, dict) and isinstance(cfg.get(k), dict): + _fix_none_arrays(cfg[k], pschema['properties']) + + if schema and 'properties' in schema and isinstance(plugin_config, dict): + _fix_none_arrays(plugin_config, schema['properties']) + + # Ensure enabled state is preserved after defaults merge + # Defaults should not overwrite an explicitly preserved enabled value + if preserved_enabled is not None: + # Restore preserved value if it was changed by defaults merge + if plugin_config.get('enabled') != preserved_enabled: + plugin_config['enabled'] = preserved_enabled + + # Normalize config data: convert string numbers to integers/floats where schema expects numbers + # This handles form data which sends everything as strings + def normalize_config_values(config, schema_props, prefix=''): + """Recursively normalize config values based on schema types""" + if not isinstance(config, dict) or not isinstance(schema_props, dict): + return config + + normalized = {} + for key, value in config.items(): + field_path = f"{prefix}.{key}" if prefix else key + + if key not in schema_props: + # Field not in schema, keep as-is (will be caught by additionalProperties check if needed) + normalized[key] = value + continue + + prop_schema = schema_props[key] + prop_type = prop_schema.get('type') + + # Handle union types (e.g., ["integer", "null"]) + if isinstance(prop_type, list): + # Check if null is allowed and value is empty/null + if 'null' in prop_type: + # Handle various representations of null/empty + if value is None: + normalized[key] = None + continue + elif isinstance(value, str): + # Strip whitespace and check for null representations + value_stripped = value.strip() + if value_stripped == '' or value_stripped.lower() in ('null', 'none', 'undefined'): + normalized[key] = None + continue + + # Try to normalize based on non-null types in the union + # Check integer first (more specific than number) + if 'integer' in prop_type: + if isinstance(value, str): + value_stripped = value.strip() + if value_stripped == '': + # Empty string with null allowed - already handled above, but double-check + if 'null' in prop_type: + normalized[key] = None + continue + try: + normalized[key] = int(value_stripped) + continue + except (ValueError, TypeError, OverflowError): + pass + elif isinstance(value, (int, float)): + normalized[key] = int(value) + continue + + # Check number (less specific, but handles floats) + if 'number' in prop_type: + if isinstance(value, str): + value_stripped = value.strip() + if value_stripped == '': + # Empty string with null allowed - already handled above, but double-check + if 'null' in prop_type: + normalized[key] = None + continue + try: + normalized[key] = float(value_stripped) + continue + except (ValueError, TypeError, OverflowError): + pass + elif isinstance(value, (int, float)): + normalized[key] = float(value) + continue + + # Check boolean + if 'boolean' in prop_type: + if isinstance(value, str): + normalized[key] = value.strip().lower() in ('true', '1', 'on', 'yes') + continue + + # If no conversion worked and null is allowed, try to set to None + # This handles cases where the value is an empty string or can't be converted + if 'null' in prop_type: + if isinstance(value, str): + value_stripped = value.strip() + if value_stripped == '' or value_stripped.lower() in ('null', 'none', 'undefined'): + normalized[key] = None + continue + # If it's already None, keep it + if value is None: + normalized[key] = None + continue + + # If no conversion worked, keep original value (will fail validation, but that's expected) + # Log a warning for debugging + logger.warning(f"Could not normalize field {field_path}: value={repr(value)}, type={type(value)}, schema_type={prop_type}") + normalized[key] = value + continue + + if isinstance(value, dict) and prop_type == 'object' and 'properties' in prop_schema: + # Recursively normalize nested objects + normalized[key] = normalize_config_values(value, prop_schema['properties'], field_path) + elif isinstance(value, list) and prop_type == 'array' and 'items' in prop_schema: + # Normalize array items + items_schema = prop_schema['items'] + item_type = items_schema.get('type') + + # Handle union types in array items + if isinstance(item_type, list): + normalized_array = [] + for v in value: + # Check if null is allowed + if 'null' in item_type: + if v is None or v == '' or (isinstance(v, str) and v.lower() in ('null', 'none')): + normalized_array.append(None) + continue + + # Try to normalize based on non-null types + if 'integer' in item_type: + if isinstance(v, str): + try: + normalized_array.append(int(v)) + continue + except (ValueError, TypeError, OverflowError): + pass + elif isinstance(v, (int, float)): + normalized_array.append(int(v)) + continue + elif 'number' in item_type: + if isinstance(v, str): + try: + normalized_array.append(float(v)) + continue + except (ValueError, TypeError, OverflowError): + pass + elif isinstance(v, (int, float)): + normalized_array.append(float(v)) + continue + + # If no conversion worked, keep original value + normalized_array.append(v) + normalized[key] = normalized_array + elif item_type == 'integer': + # Convert string numbers to integers + normalized_array = [] + for v in value: + if isinstance(v, str): + try: + normalized_array.append(int(v)) + except (ValueError, TypeError, OverflowError): + normalized_array.append(v) + elif isinstance(v, (int, float)): + normalized_array.append(int(v)) + else: + normalized_array.append(v) + normalized[key] = normalized_array + elif item_type == 'number': + # Convert string numbers to floats + normalized_array = [] + for v in value: + if isinstance(v, str): + try: + normalized_array.append(float(v)) + except (ValueError, TypeError, OverflowError): + normalized_array.append(v) + else: + normalized_array.append(v) + normalized[key] = normalized_array + elif item_type == 'object' and 'properties' in items_schema: + # Recursively normalize array of objects + normalized_array = [] + for v in value: + if isinstance(v, dict): + normalized_array.append( + normalize_config_values(v, items_schema['properties'], f"{field_path}[]") + ) + else: + normalized_array.append(v) + normalized[key] = normalized_array + else: + normalized[key] = value + elif prop_type == 'integer': + # Convert string to integer + if isinstance(value, str): + try: + normalized[key] = int(value) + except (ValueError, TypeError, OverflowError): + normalized[key] = value + else: + normalized[key] = value + elif prop_type == 'number': + # Convert string to float + if isinstance(value, str): + try: + normalized[key] = float(value) + except (ValueError, TypeError, OverflowError): + normalized[key] = value + else: + normalized[key] = value + elif prop_type == 'boolean': + # Convert string booleans + if isinstance(value, str): + normalized[key] = value.lower() in ('true', '1', 'on', 'yes') + else: + normalized[key] = value + else: + normalized[key] = value + + return normalized + + # Normalize config before validation + if schema and 'properties' in schema: + plugin_config = normalize_config_values(plugin_config, schema['properties']) + + # Filter config to only include schema-defined fields (important when additionalProperties is false) + # Use enhanced schema with core properties to ensure core properties are preserved during filtering + if schema and 'properties' in schema: + enhanced_schema_for_filtering = _enhance_schema_with_core_properties(schema) + plugin_config = _filter_config_by_schema(plugin_config, enhanced_schema_for_filtering) + + # Debug logging for union type fields (temporary) + if 'rotation_settings' in plugin_config and 'random_seed' in plugin_config.get('rotation_settings', {}): + seed_value = plugin_config['rotation_settings']['random_seed'] + logger.debug(f"After normalization, random_seed value: {repr(seed_value)}, type: {type(seed_value)}") + + # Validate configuration against schema before saving + if schema: + # Log what we're validating for debugging + logger.info(f"Validating config for {plugin_id}") + # Only the shape. plugin_config still holds the submitted secret + # values at this point -- separate_secrets does not run until + # below -- so logging it wrote live credentials to the journal. + logger.info(f"Config keys being validated: {list(plugin_config.keys())}") + + # Get enhanced schema keys (including injected core properties) + # We need to create an enhanced schema to get the actual allowed keys + import copy + enhanced_schema = copy.deepcopy(schema) + if "properties" not in enhanced_schema: + enhanced_schema["properties"] = {} + + # Core properties that are always injected during validation + core_properties = ["enabled", "display_duration", "live_priority"] + for prop_name in core_properties: + if prop_name not in enhanced_schema["properties"]: + # Add placeholder to get the full list of allowed keys + enhanced_schema["properties"][prop_name] = {"type": "any"} + + is_valid, validation_errors = schema_mgr.validate_config_against_schema( + plugin_config, schema, plugin_id + ) + if not is_valid: + # Log validation errors for debugging + logger.error(f"Config validation failed for {plugin_id}") + logger.error(f"Validation errors: {validation_errors}") + # Keys only, for the same reason as above. + logger.error(f"Config keys that failed: {list(plugin_config.keys())}") + logger.error(f"Schema properties: {list(enhanced_schema.get('properties', {}).keys())}") + + # Also print to console for immediate visibility + import json + logger.warning("Config validation failed for plugin (see debug logs)") + + # Log raw form data if this was a form submission + if 'application/json' not in (request.content_type or ''): + form_data = request.form.to_dict() + return error_response( + ErrorCode.CONFIG_VALIDATION_FAILED, + 'Configuration validation failed', + details='; '.join(validation_errors) if validation_errors else 'Unknown validation error', + context={ + 'plugin_id': plugin_id, + 'validation_errors': validation_errors, + 'config_keys': list(plugin_config.keys()), + 'schema_keys': list(enhanced_schema.get('properties', {}).keys()) + }, + suggested_fixes=[ + 'Review validation errors above', + 'Check config against schema', + 'Verify all required fields are present' + ], + status_code=400 + ) + + # Separate secrets from regular config (handles nested configs and + # array-item secrets — see src/web_interface/secret_helpers.py) + regular_config, secrets_config = separate_secrets(plugin_config, secret_fields) + # The config form renders secrets masked, so every save posts + # them back blank. Without this the blank is merged over the + # stored value and the credential is destroyed by the act of + # changing an unrelated setting. A blank means "unchanged". + secrets_config = remove_empty_secrets(secrets_config) + + # Get current configs + current_config = api_v3.config_manager.load_config() + current_secrets = api_v3.config_manager.get_raw_file_content('secrets') + + # Deep merge plugin configuration in main config (preserves nested structures) + if plugin_id not in current_config: + current_config[plugin_id] = {} + + current_config[plugin_id] = deep_merge(current_config[plugin_id], regular_config) + + # Deep merge plugin secrets in secrets config + if secrets_config: + if plugin_id not in current_secrets: + current_secrets[plugin_id] = {} + # See above -- secrets lists must merge element-wise. + current_secrets[plugin_id] = merge_secrets( + current_secrets[plugin_id], secrets_config) + # Save secrets file + try: + api_v3.config_manager.save_raw_file_content('secrets', current_secrets) + except PermissionError as e: + # Log the error with more details + import os + secrets_path = api_v3.config_manager.secrets_path + secrets_dir = os.path.dirname(secrets_path) if secrets_path else None + + # Check permissions + dir_readable = os.access(secrets_dir, os.R_OK) if secrets_dir and os.path.exists(secrets_dir) else False + dir_writable = os.access(secrets_dir, os.W_OK) if secrets_dir and os.path.exists(secrets_dir) else False + file_writable = os.access(secrets_path, os.W_OK) if secrets_path and os.path.exists(secrets_path) else False + + logger.error( + f"Permission error saving secrets config for {plugin_id}: {e}\n" + f"Secrets path: {secrets_path}\n" + f"Directory readable: {dir_readable}, writable: {dir_writable}\n" + f"File writable: {file_writable}", + exc_info=True + ) + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + f"Failed to save secrets configuration: Permission denied. Check file permissions on {secrets_path}", + status_code=500 + ) + except Exception as e: + # Log the error but don't fail the entire config save + import os + secrets_path = api_v3.config_manager.secrets_path + logger.error("Error saving secrets config for %s (path=%s)", plugin_id, secrets_path, exc_info=True) + # Return error response with more context + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + "Failed to save secrets configuration; see logs for details", + status_code=500 + ) + + # Save the updated main config using atomic save + success, error_msg = _pkg._save_config_atomic(api_v3.config_manager, current_config, create_backup=True) + if not success: + return error_response( + ErrorCode.CONFIG_SAVE_FAILED, + f"Failed to save configuration: {error_msg}", + status_code=500 + ) + + # If the plugin is loaded, notify it of the config change with merged config + try: + if api_v3.plugin_manager: + plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) + if plugin_instance: + # Reload merged config (includes secrets) and pass the plugin-specific section + merged_config = api_v3.config_manager.load_config() + plugin_full_config = merged_config.get(plugin_id, {}) + if hasattr(plugin_instance, 'on_config_change'): + plugin_instance.on_config_change(plugin_full_config) + + # Update plugin state manager and call lifecycle methods based on enabled state + # This ensures the plugin state is synchronized with the config + enabled = plugin_full_config.get('enabled', plugin_instance.enabled) + + # Update state manager if available + if api_v3.plugin_state_manager: + api_v3.plugin_state_manager.set_plugin_enabled(plugin_id, enabled) + + # Call lifecycle methods to ensure plugin state matches config + try: + if enabled: + if hasattr(plugin_instance, 'on_enable'): + plugin_instance.on_enable() + else: + if hasattr(plugin_instance, 'on_disable'): + plugin_instance.on_disable() + except Exception as lifecycle_error: + # Log the error but don't fail the save - config is already saved + import logging + logging.warning(f"Lifecycle method error for {plugin_id}: {lifecycle_error}", exc_info=True) + except Exception as hook_err: + # Do not fail the save if hook fails; just log + logger.warning("on_config_change failed: %s", hook_err) + + secret_count = len(secrets_config) + message = f'Plugin {plugin_id} configuration saved successfully' + if secret_count > 0: + message += f' ({secret_count} secret field(s) saved to config_secrets.json)' + + return success_response(message=message) + except Exception as e: + from src.web_interface.errors import WebInterfaceError + error = WebInterfaceError.from_exception(e, ErrorCode.CONFIG_SAVE_FAILED) + if api_v3.operation_history: + api_v3.operation_history.record_operation( + "configure", + plugin_id=data.get('plugin_id') if 'data' in locals() else None, + status="failed", + error=str(e) + ) + return error_response( + error.error_code, + error.message, + details=error.details, + context=error.context, + status_code=500 + ) +@api_v3.route('/plugins/schema', methods=['GET']) +def get_plugin_schema(): + """Get plugin configuration schema""" + try: + plugin_id = request.args.get('plugin_id') + if not plugin_id: + return jsonify({'status': 'error', 'message': 'plugin_id required'}), 400 + + # Get schema manager instance + schema_mgr = api_v3.schema_manager + if not schema_mgr: + return jsonify({'status': 'error', 'message': 'Schema manager not initialized'}), 500 + + # Load schema using SchemaManager (uses caching) + schema = schema_mgr.load_schema(plugin_id, use_cache=True) + + if schema: + # Offer installed visual skins as a dropdown (returns a copy; + # the cached schema and validation are never enum-restricted) + try: + current_skin = None + if api_v3.config_manager: + config = api_v3.config_manager.load_config() + current_skin = config.get(plugin_id, {}).get('skin') + injected = schema_mgr.inject_skin_selector(schema, plugin_id, current_skin) + if isinstance(injected, dict): + schema = injected + except Exception: + logger.debug('Skin selector injection failed for %s', plugin_id, exc_info=True) + return jsonify({'status': 'success', 'data': {'schema': schema}}) + + # Return a simple default schema if file not found + default_schema = { + 'type': 'object', + 'properties': { + 'enabled': { + 'type': 'boolean', + 'title': 'Enable Plugin', + 'description': 'Enable or disable this plugin', + 'default': True + }, + 'display_duration': { + 'type': 'integer', + 'title': 'Display Duration', + 'description': 'How long to show content (seconds)', + 'minimum': 5, + 'maximum': 300, + 'default': 30 + } + } + } + + return jsonify({'status': 'success', 'data': {'schema': default_schema}}) + except Exception as e: + logger.error('Error in get_plugin_schema', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/config/reset', methods=['POST']) +def reset_plugin_config(): + """Reset plugin configuration to schema defaults""" + try: + if not api_v3.config_manager: + return jsonify({'status': 'error', 'message': 'Config manager not initialized'}), 500 + + data = request.get_json(silent=True) or {} + plugin_id = data.get('plugin_id') + preserve_secrets = data.get('preserve_secrets', True) + + if not plugin_id: + return jsonify({'status': 'error', 'message': 'plugin_id required'}), 400 + + # Get schema manager instance + schema_mgr = api_v3.schema_manager + if not schema_mgr: + return jsonify({'status': 'error', 'message': 'Schema manager not initialized'}), 500 + + # Generate defaults from schema + defaults = schema_mgr.generate_default_config(plugin_id, use_cache=True) + + # Get current configs + current_config = api_v3.config_manager.load_config() + current_secrets = api_v3.config_manager.get_raw_file_content('secrets') + + # Load schema to identify secret fields + schema = schema_mgr.load_schema(plugin_id, use_cache=True) + secret_fields = set() + + if schema and 'properties' in schema: + secret_fields = find_secret_fields(schema['properties']) + + # Separate defaults into regular and secret configs + default_regular, default_secrets = separate_secrets(defaults, secret_fields) + + # Update main config with defaults + current_config[plugin_id] = default_regular + + # Update secrets config (preserve existing secrets if preserve_secrets=True) + if preserve_secrets: + # Keep existing secrets for this plugin + if plugin_id in current_secrets: + # Merge defaults with existing secrets + existing_secrets = current_secrets[plugin_id] + for key, value in default_secrets.items(): + if key not in existing_secrets or not existing_secrets[key]: + existing_secrets[key] = value + else: + current_secrets[plugin_id] = default_secrets + else: + # Replace all secrets with defaults + current_secrets[plugin_id] = default_secrets + + # Save updated configs + api_v3.config_manager.save_config(current_config) + if default_secrets or not preserve_secrets: + api_v3.config_manager.save_raw_file_content('secrets', current_secrets) + + # Notify plugin of config change if loaded + try: + if api_v3.plugin_manager: + plugin_instance = api_v3.plugin_manager.get_plugin(plugin_id) + if plugin_instance: + merged_config = api_v3.config_manager.load_config() + plugin_full_config = merged_config.get(plugin_id, {}) + if hasattr(plugin_instance, 'on_config_change'): + plugin_instance.on_config_change(plugin_full_config) + except Exception as hook_err: + logger.warning("on_config_change failed: %s", hook_err) + + return jsonify({ + 'status': 'success', + 'message': f'Plugin {plugin_id} configuration reset to defaults', + 'data': {'config': defaults} + }) + except Exception as e: + logger.error('Error in reset_plugin_config', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/action', methods=['POST']) +def execute_plugin_action(): + """Execute a plugin-defined action (e.g., authentication)""" + try: + # Try to get JSON data, with better error handling + try: + data = request.get_json(force=True) or {} + except Exception as e: + import logging + logger = logging.getLogger(__name__) + logger.error(f"Error parsing JSON in execute_plugin_action: {e}") + return jsonify({ + 'status': 'error', + 'message': 'Invalid JSON in request body', + 'content_type': request.content_type }), 400 + + plugin_id = data.get('plugin_id') + action_id = data.get('action_id') + action_params = data.get('params', {}) + + if not plugin_id or not action_id: + return jsonify({ + 'status': 'error', + 'message': 'plugin_id and action_id required', + 'received': {'plugin_id': plugin_id, 'action_id': action_id, 'has_params': bool(action_params)} + }), 400 + + # Get plugin directory + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + else: + plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id + + if not plugin_dir or not Path(plugin_dir).exists(): + return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 + + # Load manifest to get action definition + manifest_path = Path(plugin_dir) / 'manifest.json' + if not manifest_path.exists(): + return jsonify({'status': 'error', 'message': 'Plugin manifest not found'}), 404 + + with open(manifest_path, 'r', encoding='utf-8') as f: + manifest = json.load(f) + + web_ui_actions = manifest.get('web_ui_actions', []) + action_def = None + for action in web_ui_actions: + if action.get('id') == action_id: + action_def = action + break + + if not action_def: + return jsonify({'status': 'error', 'message': f'Action {action_id} not found in plugin manifest'}), 404 + + # Set LEDMATRIX_ROOT environment variable + env = os.environ.copy() + env['LEDMATRIX_ROOT'] = str(PROJECT_ROOT) + + # Execute action based on type + action_type = action_def.get('type', 'script') + + if action_type == 'script': + # Execute a Python script + script_path = action_def.get('script') + if not script_path: + return jsonify({'status': 'error', 'message': 'Script path not defined for action'}), 400 + + script_file = Path(plugin_dir) / script_path + if not script_file.exists(): + return jsonify({'status': 'error', 'message': f'Script not found: {script_path}'}), 404 + + # Handle multi-step actions (like Spotify OAuth) + step = action_params.get('step') + + if step == '2' and action_params.get('redirect_url'): + # Step 2: Complete authentication with redirect URL + redirect_url = action_params.get('redirect_url') + import tempfile + import json as json_lib + + redirect_url_escaped = json_lib.dumps(redirect_url) + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as wrapper: + wrapper.write(f'''import sys +import subprocess +import os + +# Set LEDMATRIX_ROOT +os.environ['LEDMATRIX_ROOT'] = r"{PROJECT_ROOT}" + +# Run the script and provide redirect URL +proc = subprocess.Popen( + [sys.executable, r"{script_file}"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=os.environ +) + +# Send redirect URL to stdin +redirect_url = {redirect_url_escaped} +stdout, _ = proc.communicate(input=redirect_url + "\\n", timeout=120) +print(stdout) +sys.exit(proc.returncode) +''') + wrapper_path = wrapper.name + + try: + result = subprocess.run( + ['python3', wrapper_path], + capture_output=True, + text=True, + timeout=120, + env=env + ) + os.unlink(wrapper_path) + + if result.returncode == 0: + return jsonify({ + 'status': 'success', + 'message': action_def.get('success_message', 'Action completed successfully'), + 'output': result.stdout + }) + else: + return jsonify({ + 'status': 'error', + 'message': action_def.get('error_message', 'Action failed'), + 'output': result.stdout + result.stderr + }), 400 + except subprocess.TimeoutExpired: + if os.path.exists(wrapper_path): + os.unlink(wrapper_path) + return jsonify({'status': 'error', 'message': 'Action timed out'}), 408 + else: + # Regular script execution - pass params via stdin if provided + if action_params: + # Pass params as JSON via stdin + import tempfile + import json as json_lib + + params_json = json_lib.dumps(action_params) + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as wrapper: + wrapper.write(f'''import sys +import subprocess +import os +import json + +# Set LEDMATRIX_ROOT +os.environ['LEDMATRIX_ROOT'] = r"{PROJECT_ROOT}" + +# Run the script and provide params as JSON via stdin +proc = subprocess.Popen( + [sys.executable, r"{script_file}"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=os.environ +) + +# Send params as JSON to stdin +params = {params_json} +stdout, _ = proc.communicate(input=json.dumps(params), timeout=120) +print(stdout) +sys.exit(proc.returncode) +''') + wrapper_path = wrapper.name + + try: + result = subprocess.run( + ['python3', wrapper_path], + capture_output=True, + text=True, + timeout=120, + env=env + ) + os.unlink(wrapper_path) + + # Try to parse output as JSON + try: + output_data = json.loads(result.stdout) + if result.returncode == 0: + return jsonify(output_data) + else: + return jsonify({ + 'status': 'error', + 'message': output_data.get('message', action_def.get('error_message', 'Action failed')), + 'output': result.stdout + result.stderr + }), 400 + except json.JSONDecodeError: + # Output is not JSON, return as text + if result.returncode == 0: + return jsonify({ + 'status': 'success', + 'message': action_def.get('success_message', 'Action completed successfully'), + 'output': result.stdout + }) + else: + return jsonify({ + 'status': 'error', + 'message': action_def.get('error_message', 'Action failed'), + 'output': result.stdout + result.stderr + }), 400 + except subprocess.TimeoutExpired: + if os.path.exists(wrapper_path): + os.unlink(wrapper_path) + return jsonify({'status': 'error', 'message': 'Action timed out'}), 408 + else: + # No params - check for OAuth flow first, then run script normally + # Step 1: Get initial data (like auth URL) + # For OAuth flows, we might need to import the script as a module + if action_def.get('oauth_flow'): + # Import script as module to get auth URL + import sys + import importlib.util + + spec = importlib.util.spec_from_file_location("plugin_action", script_file) + action_module = importlib.util.module_from_spec(spec) + sys.modules["plugin_action"] = action_module + + try: + spec.loader.exec_module(action_module) + + # Try to get auth URL using common patterns + auth_url = None + if hasattr(action_module, 'get_auth_url'): + auth_url = action_module.get_auth_url() + elif hasattr(action_module, 'load_spotify_credentials'): + # Spotify-specific pattern + client_id, client_secret, redirect_uri = action_module.load_spotify_credentials() + if all([client_id, client_secret, redirect_uri]): + from spotipy.oauth2 import SpotifyOAuth + sp_oauth = SpotifyOAuth( + client_id=client_id, + client_secret=client_secret, + redirect_uri=redirect_uri, + scope=getattr(action_module, 'SCOPE', ''), + cache_path=getattr(action_module, 'SPOTIFY_AUTH_CACHE_PATH', None), + open_browser=False + ) + auth_url = sp_oauth.get_authorize_url() + + if auth_url: + return jsonify({ + 'status': 'success', + 'message': action_def.get('step1_message', 'Authorization URL generated'), + 'auth_url': auth_url, + 'requires_step2': True + }) + else: + return jsonify({ + 'status': 'error', + 'message': 'Could not generate authorization URL' + }), 400 + except Exception as e: + logger.error("Error executing action step 1", exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) + }), 500 + else: + # Simple script execution + result = subprocess.run( + ['python3', str(script_file)], + capture_output=True, + text=True, + timeout=60, + env=env + ) + + # Try to parse output as JSON + try: + import json as json_module + output_data = json_module.loads(result.stdout) + if result.returncode == 0: + return jsonify(output_data) + else: + return jsonify({ + 'status': 'error', + 'message': output_data.get('message', action_def.get('error_message', 'Action failed')), + 'output': result.stdout + result.stderr + }), 400 + except json.JSONDecodeError: + # Output is not JSON, return as text + if result.returncode == 0: + return jsonify({ + 'status': 'success', + 'message': action_def.get('success_message', 'Action completed successfully'), + 'output': result.stdout + }) + else: + return jsonify({ + 'status': 'error', + 'message': action_def.get('error_message', 'Action failed'), + 'output': result.stdout + result.stderr + }), 400 + + elif action_type == 'endpoint': + # Call a plugin-defined HTTP endpoint (future feature) + return jsonify({'status': 'error', 'message': 'Endpoint actions not yet implemented'}), 501 + + else: + return jsonify({'status': 'error', 'message': f'Unknown action type: {action_type}'}), 400 + + except subprocess.TimeoutExpired: + return jsonify({'status': 'error', 'message': 'Action timed out'}), 408 + except Exception as e: + logger.error('Error in execute_plugin_action', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/authenticate/spotify', methods=['POST']) +def authenticate_spotify(): + """Run Spotify authentication script""" + try: + data = request.get_json(silent=True) or {} + redirect_url = data.get('redirect_url', '').strip() + + # Get plugin directory + plugin_id = 'ledmatrix-music' + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + else: + plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id + + if not plugin_dir or not Path(plugin_dir).exists(): + return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 + + auth_script = Path(plugin_dir) / 'authenticate_spotify.py' + if not auth_script.exists(): + return jsonify({'status': 'error', 'message': 'Authentication script not found'}), 404 + + # Set LEDMATRIX_ROOT environment variable + env = os.environ.copy() + env['LEDMATRIX_ROOT'] = str(PROJECT_ROOT) + + if redirect_url: + # Step 2: Complete authentication with redirect URL + # Create a wrapper script that provides the redirect URL as input + import tempfile + + # Create a wrapper script that provides the redirect URL + import json + redirect_url_escaped = json.dumps(redirect_url) # Properly escape the URL + with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as wrapper: + wrapper.write(f'''import sys +import subprocess +import os + +# Set LEDMATRIX_ROOT +os.environ['LEDMATRIX_ROOT'] = r"{PROJECT_ROOT}" + +# Run the auth script and provide redirect URL +proc = subprocess.Popen( + [sys.executable, r"{auth_script}"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env=os.environ +) + +# Send redirect URL to stdin +redirect_url = {redirect_url_escaped} +stdout, _ = proc.communicate(input=redirect_url + "\\n", timeout=120) +print(stdout) +sys.exit(proc.returncode) +''') + wrapper_path = wrapper.name + + try: + result = subprocess.run( + ['python3', wrapper_path], + capture_output=True, + text=True, + timeout=120, + env=env + ) + + if result.returncode == 0: + return jsonify({ + 'status': 'success', + 'message': 'Spotify authentication completed successfully', + 'output': result.stdout + }) + else: + return jsonify({ + 'status': 'error', + 'message': 'Spotify authentication failed', + 'output': result.stdout + result.stderr + }), 400 + except subprocess.TimeoutExpired: + return jsonify({'status': 'error', 'message': 'Authentication timed out'}), 408 + finally: + # The wrapper carries the user's redirect URL, so it must not + # survive the request on any path — including a failure to + # launch, which the previous per-branch unlinks missed. + if os.path.exists(wrapper_path): + os.unlink(wrapper_path) + else: + # Step 1: Get authorization URL + # Import the script's functions directly to get the auth URL + import sys + import importlib.util + + # Load the authentication script as a module + spec = importlib.util.spec_from_file_location("auth_spotify", auth_script) + auth_module = importlib.util.module_from_spec(spec) + sys.modules["auth_spotify"] = auth_module + + # Set LEDMATRIX_ROOT before loading + os.environ['LEDMATRIX_ROOT'] = str(PROJECT_ROOT) + + try: + spec.loader.exec_module(auth_module) + + # Get credentials and create OAuth object + client_id, client_secret, redirect_uri = auth_module.load_spotify_credentials() + if not all([client_id, client_secret, redirect_uri]): + return jsonify({ + 'status': 'error', + 'message': 'Could not load Spotify credentials. Please check config/config_secrets.json.' + }), 400 + + from spotipy.oauth2 import SpotifyOAuth + sp_oauth = SpotifyOAuth( + client_id=client_id, + client_secret=client_secret, + redirect_uri=redirect_uri, + scope=auth_module.SCOPE, + cache_path=auth_module.SPOTIFY_AUTH_CACHE_PATH, + open_browser=False + ) + + auth_url = sp_oauth.get_authorize_url() + + return jsonify({ + 'status': 'success', + 'message': 'Authorization URL generated', + 'auth_url': auth_url + }) + except Exception as e: + logger.error("Error getting Spotify auth URL", exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) + }), 500 + + except Exception as e: + logger.error('Error in authenticate_spotify', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/authenticate/ytm', methods=['POST']) +def authenticate_ytm(): + """Run YouTube Music authentication script""" + try: + # Get plugin directory + plugin_id = 'ledmatrix-music' + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + else: + plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id + + if not plugin_dir or not Path(plugin_dir).exists(): + return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 + + auth_script = Path(plugin_dir) / 'authenticate_ytm.py' + if not auth_script.exists(): + return jsonify({'status': 'error', 'message': 'Authentication script not found'}), 404 + + # Set LEDMATRIX_ROOT environment variable + env = os.environ.copy() + env['LEDMATRIX_ROOT'] = str(PROJECT_ROOT) + + # Run the authentication script + result = subprocess.run( + ['python3', str(auth_script)], + capture_output=True, + text=True, + timeout=60, + env=env + ) + + if result.returncode == 0: + return jsonify({ + 'status': 'success', + 'message': 'YouTube Music authentication completed successfully', + 'output': result.stdout + }) + else: + return jsonify({ + 'status': 'error', + 'message': 'YouTube Music authentication failed', + 'output': result.stdout + result.stderr + }), 400 + + except subprocess.TimeoutExpired: + return jsonify({'status': 'error', 'message': 'Authentication timed out'}), 408 + except Exception as e: + logger.error('Error in authenticate_ytm', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/assets/upload', methods=['POST']) +def upload_plugin_asset(): + """Upload asset files for a plugin""" + try: + plugin_id = request.form.get('plugin_id') + if not plugin_id: + return jsonify({'status': 'error', 'message': 'plugin_id is required'}), 400 + + if 'files' not in request.files: + return jsonify({'status': 'error', 'message': 'No files provided'}), 400 + + files = request.files.getlist('files') + if not files or all(not f.filename for f in files): + return jsonify({'status': 'error', 'message': 'No files provided'}), 400 + + # Validate file count + if len(files) > 10: + return jsonify({'status': 'error', 'message': 'Maximum 10 files per upload'}), 400 + + # Setup plugin assets directory + assets_dir = PROJECT_ROOT / 'assets' / 'plugins' / plugin_id / 'uploads' + assets_dir.mkdir(parents=True, exist_ok=True) + + # Load metadata file + metadata_file = assets_dir / '.metadata.json' + if metadata_file.exists(): + with open(metadata_file, 'r') as f: + metadata = json.load(f) + else: + metadata = {} + + uploaded_files = [] + total_size = 0 + max_size_per_file = 5 * 1024 * 1024 # 5MB + max_total_size = 50 * 1024 * 1024 # 50MB + + # Calculate current total size + for entry in metadata.values(): + if 'size' in entry: + total_size += entry.get('size', 0) + + for file in files: + if not file.filename: + continue + + # Validate file type + allowed_extensions = ['.png', '.jpg', '.jpeg', '.bmp', '.gif'] + file_ext = '.' + file.filename.lower().split('.')[-1] + if file_ext not in allowed_extensions: + return jsonify({ + 'status': 'error', + 'message': f'Invalid file type: {file_ext}. Allowed: {allowed_extensions}' + }), 400 + + # Read file to check size and validate + file.seek(0, os.SEEK_END) + file_size = file.tell() + file.seek(0) + + if file_size > max_size_per_file: + return jsonify({ + 'status': 'error', + 'message': f'File {file.filename} exceeds 5MB limit' + }), 400 + + if total_size + file_size > max_total_size: + return jsonify({ + 'status': 'error', + 'message': f'Upload would exceed 50MB total storage limit' + }), 400 + + # Validate file is actually an image (check magic bytes) + file_content = file.read(8) + file.seek(0) + is_valid_image = False + if file_content.startswith(b'\x89PNG\r\n\x1a\n'): # PNG + is_valid_image = True + elif file_content[:2] == b'\xff\xd8': # JPEG + is_valid_image = True + elif file_content[:2] == b'BM': # BMP + is_valid_image = True + elif file_content[:6] in [b'GIF87a', b'GIF89a']: # GIF + is_valid_image = True + + if not is_valid_image: + return jsonify({ + 'status': 'error', + 'message': f'File {file.filename} is not a valid image file' + }), 400 + + # Generate unique filename + timestamp = int(_pkg.time.time()) + file_hash = hashlib.md5(file_content + file.filename.encode()).hexdigest()[:8] + safe_filename = f"image_{timestamp}_{file_hash}{file_ext}" + file_path = assets_dir / safe_filename + + # Ensure filename is unique + counter = 1 + while file_path.exists(): + safe_filename = f"image_{timestamp}_{file_hash}_{counter}{file_ext}" + file_path = assets_dir / safe_filename + counter += 1 + + # Save file + file.save(str(file_path)) + + # Make file readable + os.chmod(file_path, 0o644) + + # Generate unique ID + image_id = str(uuid.uuid4()) + + # Store metadata + relative_path = f"assets/plugins/{plugin_id}/uploads/{safe_filename}" + metadata[image_id] = { + 'id': image_id, + 'filename': safe_filename, + 'path': relative_path, + 'size': file_size, + 'uploaded_at': datetime.utcnow().isoformat() + 'Z', + 'original_filename': file.filename + } + + uploaded_files.append({ + 'id': image_id, + 'filename': safe_filename, + 'path': relative_path, + 'size': file_size, + 'uploaded_at': metadata[image_id]['uploaded_at'] + }) + + total_size += file_size + + # Save metadata + with open(metadata_file, 'w') as f: + json.dump(metadata, f, indent=2) + + return jsonify({ + 'status': 'success', + 'uploaded_files': uploaded_files, + 'total_files': len(metadata) + }) + + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/of-the-day/json/upload', methods=['POST']) +def upload_of_the_day_json(): + """Upload JSON files for of-the-day plugin""" + try: + if 'files' not in request.files: + return jsonify({'status': 'error', 'message': 'No files provided'}), 400 + + files = request.files.getlist('files') + if not files or all(not f.filename for f in files): + return jsonify({'status': 'error', 'message': 'No files provided'}), 400 + + # Get plugin directory + plugin_id = 'ledmatrix-of-the-day' + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + else: + plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id + + if not plugin_dir or not Path(plugin_dir).exists(): + return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 + + # Setup of_the_day directory + data_dir = Path(plugin_dir) / 'of_the_day' + data_dir.mkdir(parents=True, exist_ok=True) + + uploaded_files = [] + max_size_per_file = 5 * 1024 * 1024 # 5MB + + for file in files: + if not file.filename: + continue + + # Validate file extension + if not file.filename.lower().endswith('.json'): + return jsonify({ + 'status': 'error', + 'message': f'File {file.filename} must be a JSON file (.json)' + }), 400 + + # Read and validate file size + file.seek(0, os.SEEK_END) + file_size = file.tell() + file.seek(0) + + if file_size > max_size_per_file: + return jsonify({ + 'status': 'error', + 'message': f'File {file.filename} exceeds 5MB limit' + }), 400 + + # Read and validate JSON content + try: + file_content = file.read().decode('utf-8') + json_data = json.loads(file_content) + except json.JSONDecodeError as e: + return jsonify({ + 'status': 'error', + 'message': 'Invalid JSON in request body' + }), 400 + except UnicodeDecodeError: + return jsonify({ + 'status': 'error', + 'message': f'File {file.filename} is not valid UTF-8 text' + }), 400 + + # Validate JSON structure (must be object with day number keys) + if not isinstance(json_data, dict): + return jsonify({ + 'status': 'error', + 'message': f'JSON in {file.filename} must be an object with day numbers (1-365) as keys' + }), 400 + + # Check if keys are valid day numbers + for key in json_data.keys(): + try: + day_num = int(key) + if day_num < 1 or day_num > 365: + return jsonify({ + 'status': 'error', + 'message': f'Day number {day_num} in {file.filename} is out of range (must be 1-365)' + }), 400 + except ValueError: + return jsonify({ + 'status': 'error', + 'message': f'Invalid key "{key}" in {file.filename}: must be a day number (1-365)' + }), 400 + + # Generate safe filename from original (preserve user's filename) + original_filename = file.filename + safe_filename = original_filename.lower().replace(' ', '_') + # Ensure it's a valid filename + safe_filename = ''.join(c for c in safe_filename if c.isalnum() or c in '._-') + if not safe_filename.endswith('.json'): + safe_filename += '.json' + + file_path = data_dir / safe_filename + + # If file exists, add counter + counter = 1 + base_name = safe_filename.replace('.json', '') + while file_path.exists(): + safe_filename = f"{base_name}_{counter}.json" + file_path = data_dir / safe_filename + counter += 1 + + # Save file + with open(file_path, 'w', encoding='utf-8') as f: + json.dump(json_data, f, indent=2, ensure_ascii=False) + + # Make file readable + os.chmod(file_path, 0o644) + + # Extract category name from filename (remove .json extension) + category_name = safe_filename.replace('.json', '') + display_name = category_name.replace('_', ' ').title() + + # Update plugin config to add category + try: + sys.path.insert(0, str(plugin_dir)) + from scripts.update_config import add_category_to_config + add_category_to_config(category_name, f'of_the_day/{safe_filename}', display_name) + except Exception as e: + logger.warning("Could not update config: %s", e) + # Continue anyway - file is uploaded + + # Generate file ID (use category name as ID for simplicity) + file_id = category_name + + uploaded_files.append({ + 'id': file_id, + 'filename': safe_filename, + 'original_filename': original_filename, + 'path': f'of_the_day/{safe_filename}', + 'size': file_size, + 'uploaded_at': datetime.utcnow().isoformat() + 'Z', + 'category_name': category_name, + 'display_name': display_name, + 'entry_count': len(json_data) + }) + + return jsonify({ + 'status': 'success', + 'uploaded_files': uploaded_files, + 'total_files': len(uploaded_files) + }) + + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/of-the-day/json/delete', methods=['POST']) +def delete_of_the_day_json(): + """Delete a JSON file from of-the-day plugin""" + try: + data = request.get_json(silent=True) or {} + file_id = data.get('file_id') # This is the category_name + + if not file_id: + return jsonify({'status': 'error', 'message': 'file_id is required'}), 400 + + # Get plugin directory + plugin_id = 'ledmatrix-of-the-day' + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + else: + plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id + + if not plugin_dir or not Path(plugin_dir).exists(): + return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 + + data_dir = Path(plugin_dir) / 'of_the_day' + filename = f"{file_id}.json" + file_path = data_dir / filename + + if not file_path.exists(): + return jsonify({'status': 'error', 'message': f'File {filename} not found'}), 404 + + # Delete file + file_path.unlink() + + # Update config to remove category + try: + sys.path.insert(0, str(plugin_dir)) + from scripts.update_config import remove_category_from_config + remove_category_from_config(file_id) + except Exception as e: + logger.warning("Could not update config: %s", e) + + return jsonify({ + 'status': 'success', + 'message': f'File {filename} deleted successfully' + }) + + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins//static/', methods=['GET']) +def serve_plugin_static(plugin_id, file_path): + """Serve static files from plugin directory""" + try: + # Get plugin directory + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + else: + plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id + + if not plugin_dir or not Path(plugin_dir).exists(): + return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 + + # Resolve file path (prevent directory traversal) + plugin_dir = Path(plugin_dir).resolve() + requested_file = (plugin_dir / file_path).resolve() + + # Security check: ensure file is within plugin directory + if not str(requested_file).startswith(str(plugin_dir)): + return jsonify({'status': 'error', 'message': 'Invalid file path'}), 403 + + # Check if file exists + if not requested_file.exists() or not requested_file.is_file(): + return jsonify({'status': 'error', 'message': 'File not found'}), 404 + + # Determine content type + content_type = 'text/plain' + if file_path.endswith('.html'): + content_type = 'text/html' + elif file_path.endswith('.js'): + content_type = 'application/javascript' + elif file_path.endswith('.css'): + content_type = 'text/css' + elif file_path.endswith('.json'): + content_type = 'application/json' + + # Read and return file + with open(requested_file, 'r', encoding='utf-8') as f: + content = f.read() + + return Response(content, mimetype=content_type) + + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/calendar/upload-credentials', methods=['POST']) +def upload_calendar_credentials(): + """Upload credentials.json file for calendar plugin""" + try: + if 'file' not in request.files: + return jsonify({'status': 'error', 'message': 'No file provided'}), 400 + + file = request.files['file'] + if not file or not file.filename: + return jsonify({'status': 'error', 'message': 'No file provided'}), 400 + + # Validate file extension + if not file.filename.lower().endswith('.json'): + return jsonify({'status': 'error', 'message': 'File must be a JSON file (.json)'}), 400 + + # Validate file size (max 1MB for credentials) + file.seek(0, os.SEEK_END) + file_size = file.tell() + file.seek(0) + + if file_size > 1024 * 1024: # 1MB + return jsonify({'status': 'error', 'message': 'File exceeds 1MB limit'}), 400 + + # Validate it's valid JSON + try: + file_content = file.read() + file.seek(0) + creds_data = json.loads(file_content) + except json.JSONDecodeError: + return jsonify({'status': 'error', 'message': 'File is not valid JSON'}), 400 + + # Validate it looks like Google OAuth credentials. A bare scalar, a + # list, true/null — all valid JSON, none of them credentials. Reject + # rather than save: a file written as credentials.json but unusable + # as credentials only fails later, somewhere less obvious. + if not isinstance(creds_data, dict) or not ( + 'installed' in creds_data or 'web' in creds_data): + return jsonify({ + 'status': 'error', + 'message': 'File does not appear to be a valid Google OAuth credentials file' + }), 400 + + # Get plugin directory + plugin_id = 'calendar' + if api_v3.plugin_manager: + plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + else: + plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id + + if not plugin_dir or not Path(plugin_dir).exists(): + return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 + + # Save file to plugin directory + credentials_path = Path(plugin_dir) / 'credentials.json' + + # Backup existing file if it exists + if credentials_path.exists(): + backup_path = Path(plugin_dir) / f'credentials.json.backup.{int(_pkg.time.time())}' + import shutil + shutil.copy2(credentials_path, backup_path) + _prune_credential_backups(Path(plugin_dir)) + + # Save new file + file.save(str(credentials_path)) + + # Set proper permissions + os.chmod(credentials_path, 0o600) # Read/write for owner only + + return jsonify({ + 'status': 'success', + 'message': 'Credentials file uploaded successfully', + 'path': str(credentials_path) + }) + + except Exception as e: + logger.error('Error in upload_calendar_credentials', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/calendar/authenticate', methods=['POST']) +def authenticate_calendar(): + """Google OAuth for the calendar plugin, in the two steps it requires. + + Step 1 (no body) returns the consent URL to open. Step 2 posts back the + URL Google redirected to -- it fails to load, because the redirect points + at a loopback address nothing is listening on, but the address bar carries + the authorization code -- and the script exchanges it for a token. + + Two calls rather than one because the user has to visit Google in between. + The script persists the PKCE verifier from step 1 for step 2 to reuse; the + exchange fails with "Missing code verifier" otherwise. + """ + try: + plugin_dir = _pkg._calendar_plugin_dir() + if plugin_dir is None: + return jsonify({ + 'status': 'error', + 'message': 'The calendar plugin is not installed' + }), 404 + + if not (plugin_dir / 'credentials.json').exists(): + return jsonify({ + 'status': 'error', + 'message': ('No credentials.json yet. Upload your Google OAuth ' + 'client file first (Step 1).') + }), 400 + + data = request.get_json(silent=True) or {} + redirect_url = (data.get('redirect_url') or data.get('code') or '').strip() + + payload, error = _run_calendar_registration(plugin_dir, redirect_url) + if error: + return jsonify({'status': 'error', 'message': error}), 500 + if payload.get('status') != 'success': + # The script's own diagnosis is more useful than anything that + # could be reconstructed here -- but it interpolates exceptions + # into its messages, so it reaches the client redacted and the + # original goes to the log. + logger.error('calendar authentication failed: %s', payload) + safe = dict(payload) + safe['message'] = redact_text(str(payload.get('message', '') + or 'Authentication failed')) + return jsonify(safe), 400 + return jsonify(payload) + + except Exception as e: + logger.error('Error in authenticate_calendar', exc_info=True) + return jsonify({'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/calendar/list-calendars', methods=['GET']) +def list_calendar_calendars(): + """The calendars this account can see, for the config picker. + + Reads the token the OAuth flow wrote rather than shelling out again: the + picker is used interactively and a subprocess per click is slower than the + API call it would be wrapping. + """ + try: + plugin_dir = _pkg._calendar_plugin_dir() + if plugin_dir is None: + return jsonify({ + 'status': 'error', + 'message': 'The calendar plugin is not installed' + }), 404 + + token_file = plugin_dir / 'token.pickle' + if not token_file.exists(): + return jsonify({ + 'status': 'error', + 'message': ('Not authenticated with Google yet. Complete Step 2 ' + 'first, then load your calendars.') + }), 400 + + try: + import pickle + from google.auth.transport.requests import Request as GoogleRequest + from googleapiclient.discovery import build as build_google_service + except ImportError as e: + return jsonify({ + 'status': 'error', + # The name of the missing module is the whole diagnosis, but it + # arrives as an exception, so it goes through the redactor like + # any other -- an ImportError can quote a path. + 'message': ('The Google API libraries are not installed. Install ' + "the calendar plugin's requirements.txt. (%s)" + % describe_exception(e)) + }), 500 + + with open(token_file, 'rb') as handle: + # Written only by this plugin's own OAuth flow, into its own + # directory, and read here exactly as the plugin itself reads it. + creds = pickle.load(handle) # nosec B301 - locally generated token + + if creds and creds.expired and creds.refresh_token: + creds.refresh(GoogleRequest()) + with open(token_file, 'wb') as handle: + pickle.dump(creds, handle) + os.chmod(token_file, 0o600) + + if not creds or not creds.valid: + return jsonify({ + 'status': 'error', + 'message': ('Stored Google credentials are no longer valid. ' + 'Run Step 2 again to re-authenticate.') + }), 400 + + service = build_google_service('calendar', 'v3', credentials=creds) + + # calendarList.list returns 100 entries per page by default and caps at + # 250, handing back a nextPageToken when there are more. Taking only + # the first page would silently hide calendars from the picker, and the + # user would have no way to tell the list was truncated. + entries = [] + page_token = None + for _ in range(_CALENDAR_LIST_MAX_PAGES): + response = service.calendarList().list( + maxResults=250, pageToken=page_token).execute() + entries.extend(response.get('items', [])) + page_token = response.get('nextPageToken') + if not page_token: + break + else: + # 2500 calendars in, something is wrong with the account or the + # token is looping; show what was collected rather than spin. + logger.warning( + 'calendarList paging stopped at %d pages with more remaining', + _CALENDAR_LIST_MAX_PAGES) + + calendars = [{ + 'id': entry.get('id'), + # The picker labels each row with summary and falls back to the id + # only in its own display, so send something either way. + 'summary': entry.get('summary') or entry.get('id'), + 'primary': bool(entry.get('primary', False)), + } for entry in entries if entry.get('id')] + + # Primary first, then alphabetically: the list is usually short but the + # one the user wants is almost always their own calendar. + calendars.sort(key=lambda c: (not c['primary'], c['summary'].lower())) + + return jsonify({'status': 'success', 'calendars': calendars}) + + except Exception as e: + logger.error('Error in list_calendar_calendars', exc_info=True) + return jsonify({'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/assets/delete', methods=['POST']) +def delete_plugin_asset(): + """Delete an asset file for a plugin""" + try: + data = request.get_json() + plugin_id = data.get('plugin_id') + image_id = data.get('image_id') + + if not plugin_id or not image_id: + return jsonify({'status': 'error', 'message': 'plugin_id and image_id are required'}), 400 + + # Get asset directory + assets_dir = PROJECT_ROOT / 'assets' / 'plugins' / plugin_id / 'uploads' + metadata_file = assets_dir / '.metadata.json' + + if not metadata_file.exists(): + return jsonify({'status': 'error', 'message': 'Metadata file not found'}), 404 + + # Load metadata + with open(metadata_file, 'r') as f: + metadata = json.load(f) + + if image_id not in metadata: + return jsonify({'status': 'error', 'message': 'Image not found'}), 404 + + # Delete file + file_path = PROJECT_ROOT / metadata[image_id]['path'] + if file_path.exists(): + file_path.unlink() + + # Remove from metadata + del metadata[image_id] + + # Save metadata + with open(metadata_file, 'w') as f: + json.dump(metadata, f, indent=2) + + return jsonify({'status': 'success', 'message': 'Image deleted successfully'}) + + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/plugins/assets/list', methods=['GET']) +def list_plugin_assets(): + """List asset files for a plugin""" + try: + plugin_id = request.args.get('plugin_id') + if not plugin_id: + return jsonify({'status': 'error', 'message': 'plugin_id is required'}), 400 + + # Get asset directory + assets_dir = PROJECT_ROOT / 'assets' / 'plugins' / plugin_id / 'uploads' + metadata_file = assets_dir / '.metadata.json' + + if not metadata_file.exists(): + return jsonify({'status': 'success', 'data': {'assets': []}}) + + # Load metadata + with open(metadata_file, 'r') as f: + metadata = json.load(f) + + # Convert to list + assets = list(metadata.values()) + + return jsonify({'status': 'success', 'data': {'assets': assets}}) + + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 diff --git a/web_interface/blueprints/api_v3/starlark.py b/web_interface/blueprints/api_v3/starlark.py new file mode 100644 index 00000000..39c40668 --- /dev/null +++ b/web_interface/blueprints/api_v3/starlark.py @@ -0,0 +1,712 @@ +"""Starlark / Tronbyte app management routes. + +Routes decorate the shared `api_v3` Blueprint from ._common, so their +endpoint names are unchanged by living here. +""" +from web_interface.blueprints.api_v3 import ( + PROJECT_ROOT, Path, _find_pixlet_binary, _install_star_file, _standalone_render_starlark_app, + _starlark_github_token, _starlark_manifest_lock, _validate_and_sanitize_app_id, + _validate_starlark_app_path, _validate_timing_value, api_v3, describe_exception, json, jsonify, + logger, os, request, shutil, subprocess, tempfile, +) +import web_interface.blueprints.api_v3 as _pkg +# Read through the module rather than bound by value: tests patch these +# as module attributes, and a value binding would not see the patch. +# Several are also called from helpers that live in __init__, so the +# package is the only patch point that covers every caller. + + +@api_v3.route('/starlark/status', methods=['GET']) +def get_starlark_status(): + """Get Starlark plugin status and Pixlet availability.""" + try: + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + info = starlark_plugin.get_info() + magnify_info = starlark_plugin.get_magnify_recommendation() + return jsonify({ + 'status': 'success', + 'pixlet_available': info.get('pixlet_available', False), + 'pixlet_version': info.get('pixlet_version'), + 'installed_apps': info.get('installed_apps', 0), + 'enabled_apps': info.get('enabled_apps', 0), + 'current_app': info.get('current_app'), + 'plugin_enabled': starlark_plugin.enabled, + 'display_info': magnify_info + }) + + # Plugin not loaded - check Pixlet availability via shared resolver + # (respects user-configured pixlet_path, bundled binary, and system PATH) + full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {} + pixlet_path = _find_pixlet_binary(full_config.get('starlark-apps', {}).get('pixlet_path')) + pixlet_available = pixlet_path is not None + + # Read app counts from manifest + manifest = _pkg._read_starlark_manifest() + apps = manifest.get('apps', {}) + installed_count = len(apps) + enabled_count = sum(1 for a in apps.values() if a.get('enabled', True)) + + return jsonify({ + 'status': 'success', + 'pixlet_available': pixlet_available, + 'pixlet_version': None, + 'installed_apps': installed_count, + 'enabled_apps': enabled_count, + 'plugin_enabled': True, + 'plugin_loaded': False, + 'display_info': {} + }) + + except Exception as e: + logger.exception("[Starlark] get_starlark_status failed") + return jsonify({'status': 'error', 'message': 'Failed to get Starlark status', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/install-pixlet', methods=['POST']) +def install_pixlet(): + """Download and install Pixlet binary.""" + try: + script_path = PROJECT_ROOT / 'scripts' / 'download_pixlet.sh' + if not script_path.exists(): + return jsonify({'status': 'error', 'message': 'Installation script not found'}), 404 + + os.chmod(script_path, 0o755) + + result = subprocess.run( + [str(script_path)], + cwd=str(PROJECT_ROOT), + capture_output=True, + text=True, + timeout=300 + ) + + if result.returncode == 0: + logger.info("Pixlet downloaded successfully") + return jsonify({'status': 'success', 'message': 'Pixlet installed successfully!', 'output': result.stdout}) + else: + return jsonify({'status': 'error', 'message': f'Failed to download Pixlet: {result.stderr}'}), 500 + + except subprocess.TimeoutExpired as err: + logger.exception("[Starlark] Pixlet download timed out") + return jsonify({'status': 'error', 'message': 'Download timed out', + 'details': describe_exception(err)}), 500 + except Exception as e: + logger.exception("[Starlark] install_pixlet failed") + return jsonify({'status': 'error', 'message': 'Failed to install Pixlet', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/apps', methods=['GET']) +def get_starlark_apps(): + """List all installed Starlark apps.""" + try: + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + apps_list = [] + for app_id, app_instance in starlark_plugin.apps.items(): + apps_list.append({ + 'id': app_id, + 'name': app_instance.manifest.get('name', app_id), + 'enabled': app_instance.is_enabled(), + 'has_frames': app_instance.frames is not None, + 'render_interval': app_instance.get_render_interval(), + 'display_duration': app_instance.get_display_duration(), + 'config': app_instance.config, + 'has_schema': app_instance.schema is not None, + 'last_render_time': app_instance.last_render_time + }) + return jsonify({'status': 'success', 'apps': apps_list, 'count': len(apps_list)}) + + # Standalone: read manifest from disk + manifest = _pkg._read_starlark_manifest() + apps_list = [] + for app_id, app_data in manifest.get('apps', {}).items(): + apps_list.append({ + 'id': app_id, + 'name': app_data.get('name', app_id), + 'enabled': app_data.get('enabled', True), + 'has_frames': False, + 'render_interval': app_data.get('render_interval', 300), + 'display_duration': app_data.get('display_duration', 15), + 'config': app_data.get('config', {}), + 'has_schema': False, + 'last_render_time': None + }) + return jsonify({'status': 'success', 'apps': apps_list, 'count': len(apps_list)}) + + except Exception as e: + logger.exception("[Starlark] get_starlark_apps failed") + return jsonify({'status': 'error', 'message': 'Failed to get Starlark apps', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/apps/', methods=['GET']) +def get_starlark_app(app_id): + """Get details for a specific Starlark app.""" + try: + # Validate app_id before any filesystem access + app_dir, error_msg = _validate_starlark_app_path(app_id) + if error_msg: + return jsonify({'status': 'error', 'message': error_msg}), 400 + + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + return jsonify({ + 'status': 'success', + 'app': { + 'id': app_id, + 'name': app.manifest.get('name', app_id), + 'enabled': app.is_enabled(), + 'config': app.config, + 'schema': app.schema, + 'render_interval': app.get_render_interval(), + 'display_duration': app.get_display_duration(), + 'has_frames': app.frames is not None, + 'frame_count': len(app.frames) if app.frames else 0, + 'last_render_time': app.last_render_time, + } + }) + + # Standalone: read from manifest + manifest = _pkg._read_starlark_manifest() + app_data = manifest.get('apps', {}).get(app_id) + if not app_data: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + # Load schema from schema.json if it exists (path already validated above) + schema = None + schema_file = app_dir / 'schema.json' + if schema_file.exists(): + try: + with open(schema_file, 'r') as f: + schema = json.load(f) + except (OSError, json.JSONDecodeError) as e: + logger.warning(f"Failed to load schema for {app_id}: {e}") + + return jsonify({ + 'status': 'success', + 'app': { + 'id': app_id, + 'name': app_data.get('name', app_id), + 'enabled': app_data.get('enabled', True), + 'config': app_data.get('config', {}), + 'schema': schema, + 'render_interval': app_data.get('render_interval', 300), + 'display_duration': app_data.get('display_duration', 15), + 'has_frames': False, + 'frame_count': 0, + 'last_render_time': None, + } + }) + + except Exception as e: + logger.exception("[Starlark] get_starlark_app failed") + return jsonify({'status': 'error', 'message': 'Failed to get Starlark app', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/upload', methods=['POST']) +def upload_starlark_app(): + """Upload and install a new Starlark app.""" + try: + if 'file' not in request.files: + return jsonify({'status': 'error', 'message': 'No file uploaded'}), 400 + + file = request.files['file'] + if not file.filename or not file.filename.endswith('.star'): + return jsonify({'status': 'error', 'message': 'File must have .star extension'}), 400 + + # Check file size (limit to 5MB for .star files) + file.seek(0, 2) # Seek to end + file_size = file.tell() + file.seek(0) # Reset to beginning + MAX_STAR_SIZE = 5 * 1024 * 1024 # 5MB + if file_size > MAX_STAR_SIZE: + return jsonify({'status': 'error', 'message': f'File too large (max 5MB, got {file_size/1024/1024:.1f}MB)'}), 400 + + app_name = request.form.get('name') + app_id_input = request.form.get('app_id') + filename_base = file.filename.replace('.star', '') if file.filename else None + app_id, app_id_error = _validate_and_sanitize_app_id(app_id_input, fallback_source=filename_base) + if app_id_error: + return jsonify({'status': 'error', 'message': f'Invalid app_id: {app_id_error}'}), 400 + + render_interval_input = request.form.get('render_interval') + render_interval = 300 + if render_interval_input is not None: + render_interval, err = _validate_timing_value(render_interval_input, 'render_interval') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + render_interval = render_interval or 300 + + display_duration_input = request.form.get('display_duration') + display_duration = 15 + if display_duration_input is not None: + display_duration, err = _validate_timing_value(display_duration_input, 'display_duration') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + display_duration = display_duration or 15 + + import tempfile + with tempfile.NamedTemporaryFile(delete=False, suffix='.star') as tmp: + file.save(tmp.name) + temp_path = tmp.name + + try: + metadata = {'name': app_name or app_id, 'render_interval': render_interval, 'display_duration': display_duration} + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + success = starlark_plugin.install_app(app_id, temp_path, metadata) + else: + success = _install_star_file(app_id, temp_path, metadata) + if success: + return jsonify({'status': 'success', 'message': f'App installed: {app_id}', 'app_id': app_id}) + else: + return jsonify({'status': 'error', 'message': 'Failed to install app'}), 500 + finally: + try: + os.unlink(temp_path) + except OSError: + pass + + except (OSError, IOError) as err: + # This used to withhold the detail because it names absolute paths on + # the device. A full disk and a bad permission are indistinguishable + # without it, though, and describe_exception redacts credentials and + # truncates -- the same trade-off every other handler here makes. + logger.exception("[Starlark] File error uploading starlark app: %s", err) + return jsonify({'status': 'error', 'message': 'File error during upload', + 'details': describe_exception(err)}), 500 + except ImportError as err: + logger.exception("[Starlark] Module load error uploading starlark app: %s", err) + return jsonify({'status': 'error', 'message': 'Failed to load app module', 'details': describe_exception(err)}), 500 + except Exception as err: + logger.exception("[Starlark] Unexpected error uploading starlark app: %s", err) + return jsonify({'status': 'error', 'message': 'Failed to upload app', 'details': describe_exception(err)}), 500 +@api_v3.route('/starlark/apps/', methods=['DELETE']) +def uninstall_starlark_app(app_id): + """Uninstall a Starlark app.""" + try: + # Validate app_id before any filesystem access + app_dir, error_msg = _validate_starlark_app_path(app_id) + if error_msg: + return jsonify({'status': 'error', 'message': error_msg}), 400 + + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + success = starlark_plugin.uninstall_app(app_id) + else: + # Standalone: remove app dir and manifest entry. app_dir is the + # path _validate_starlark_app_path checked, not a fresh join. + import shutil + if app_dir.exists(): + shutil.rmtree(app_dir) + with _starlark_manifest_lock(): + manifest = _pkg._read_starlark_manifest() + manifest.get('apps', {}).pop(app_id, None) + success = _pkg._write_starlark_manifest(manifest) + + if success: + return jsonify({'status': 'success', 'message': f'App uninstalled: {app_id}'}) + else: + return jsonify({'status': 'error', 'message': 'Failed to uninstall app'}), 500 + + except Exception as e: + logger.exception("[Starlark] uninstall_starlark_app failed") + return jsonify({'status': 'error', 'message': 'Failed to uninstall Starlark app', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/apps//config', methods=['GET']) +def get_starlark_app_config(app_id): + """Get configuration for a Starlark app.""" + try: + # Validate app_id before any filesystem access + app_dir, error_msg = _validate_starlark_app_path(app_id) + if error_msg: + return jsonify({'status': 'error', 'message': error_msg}), 400 + + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + return jsonify({'status': 'success', 'config': app.config, 'schema': app.schema}) + + # Standalone: read from config.json. app_dir is the path + # _validate_starlark_app_path checked, not a fresh join. + config_file = app_dir / "config.json" + + if not app_dir.exists(): + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + config = {} + if config_file.exists(): + try: + with open(config_file, 'r') as f: + config = json.load(f) + except (OSError, json.JSONDecodeError) as e: + logger.warning(f"Failed to load config for {app_id}: {e}") + + # Load schema from schema.json + schema = None + schema_file = app_dir / "schema.json" + if schema_file.exists(): + try: + with open(schema_file, 'r') as f: + schema = json.load(f) + except Exception as e: + logger.warning(f"Failed to load schema for {app_id}: {e}") + + return jsonify({'status': 'success', 'config': config, 'schema': schema}) + + except Exception as e: + logger.exception("[Starlark] get_starlark_app_config failed") + return jsonify({'status': 'error', 'message': 'Failed to get Starlark app config', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/apps//config', methods=['PUT']) +def update_starlark_app_config(app_id): + """Update configuration for a Starlark app.""" + try: + # Validate app_id before any filesystem access + app_dir, error_msg = _validate_starlark_app_path(app_id) + if error_msg: + return jsonify({'status': 'error', 'message': error_msg}), 400 + + data = request.get_json(silent=True) + if not data: + return jsonify({'status': 'error', 'message': 'No configuration provided'}), 400 + + if 'render_interval' in data: + val, err = _validate_timing_value(data['render_interval'], 'render_interval') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + data['render_interval'] = val + + if 'display_duration' in data: + val, err = _validate_timing_value(data['display_duration'], 'display_duration') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + data['display_duration'] = val + + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + # Extract timing keys from data before updating config (they belong in manifest, not config) + render_interval = data.pop('render_interval', None) + display_duration = data.pop('display_duration', None) + + # Snapshot before mutating. save_config() can fail, and the route + # answers 500 below when it does -- but the loaded app kept the new + # values anyway, so a later GET returned configuration that was + # never persisted and the plugin rendered with it. + prev_config = dict(app.config) + prev_manifest = dict(app.manifest) + + # Update config with non-timing fields only + app.config.update(data) + + # Update manifest with timing fields + timing_changed = False + if render_interval is not None: + app.manifest['render_interval'] = render_interval + timing_changed = True + if display_duration is not None: + app.manifest['display_duration'] = display_duration + timing_changed = True + saved = app.save_config() + if not saved: + app.config.clear(); app.config.update(prev_config) + app.manifest.clear(); app.manifest.update(prev_manifest) + if saved: + # Persist manifest if timing changed (same pattern as toggle endpoint) + if timing_changed: + try: + # Use safe manifest update to prevent race conditions + timing_updates = {} + if render_interval is not None: + timing_updates['render_interval'] = render_interval + if display_duration is not None: + timing_updates['display_duration'] = display_duration + + def update_fn(manifest): + manifest['apps'][app_id].update(timing_updates) + starlark_plugin._update_manifest_safe(update_fn) + except Exception as e: + logger.warning(f"Failed to persist timing to manifest for {app_id}: {e}") + starlark_plugin._render_app(app, force=True) + return jsonify({'status': 'success', 'message': 'Configuration updated', 'config': app.config}) + else: + return jsonify({'status': 'error', 'message': 'Failed to save configuration'}), 500 + + # Standalone: update both config.json and manifest. Both live under + # the manifest lock (serialized against every other standalone + # manifest read-modify-write -- see _starlark_manifest_lock), and if + # the manifest write fails after config.json was already written, + # config.json is rolled back so the two do not end up out of sync. + with _starlark_manifest_lock(): + manifest = _pkg._read_starlark_manifest() + app_data = manifest.get('apps', {}).get(app_id) + if not app_data: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + # Extract timing keys (they go in manifest, not config.json) + render_interval = data.pop('render_interval', None) + display_duration = data.pop('display_duration', None) + + # Update manifest with timing values + if render_interval is not None: + app_data['render_interval'] = render_interval + if display_duration is not None: + app_data['display_duration'] = display_duration + + # Load current config from config.json. app_dir is the path + # _validate_starlark_app_path checked, not a fresh join. + config_file = app_dir / "config.json" + existed_before = config_file.exists() + previous_config_bytes = None + current_config = {} + if existed_before: + try: + previous_config_bytes = config_file.read_bytes() + current_config = json.loads(previous_config_bytes) + except Exception as e: + logger.warning(f"Failed to load config for {app_id}: {e}") + + # Update config with new values (excluding timing keys) + current_config.update(data) + + # Write updated config to config.json + try: + with open(config_file, 'w') as f: + json.dump(current_config, f, indent=2) + except Exception as e: + logger.error(f"Failed to save config.json for {app_id}: {e}") + logger.exception("Failed to save Starlark configuration for %r", app_id) + return jsonify({'status': 'error', 'message': 'Failed to save configuration', + 'details': describe_exception(e)}), 500 + + # Also update manifest for backward compatibility + app_data.setdefault('config', {}).update(data) + + if _pkg._write_starlark_manifest(manifest): + return jsonify({'status': 'success', 'message': 'Configuration updated', 'config': current_config}) + + # The manifest write failed after config.json was already + # written -- roll config.json back rather than leave the two + # disagreeing about what was saved. + try: + if existed_before and previous_config_bytes is not None: + config_file.write_bytes(previous_config_bytes) + elif not existed_before: + config_file.unlink(missing_ok=True) + except OSError: + logger.exception( + "Failed to roll back config.json for %r after manifest write failure", app_id) + return jsonify({'status': 'error', 'message': 'Failed to save manifest'}), 500 + + except Exception as e: + logger.exception("[Starlark] update_starlark_app_config failed") + return jsonify({'status': 'error', 'message': 'Failed to update Starlark app config', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/apps//toggle', methods=['POST']) +def toggle_starlark_app(app_id): + """Enable or disable a Starlark app.""" + try: + data = request.get_json(silent=True) or {} + + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + enabled = data.get('enabled') + if enabled is None: + enabled = not app.is_enabled() + app.manifest['enabled'] = enabled + # Use safe manifest update to prevent race conditions + def update_fn(manifest): + manifest['apps'][app_id]['enabled'] = enabled + starlark_plugin._update_manifest_safe(update_fn) + return jsonify({'status': 'success', 'message': f"App {'enabled' if enabled else 'disabled'}", 'enabled': enabled}) + + # Standalone: update manifest directly + with _starlark_manifest_lock(): + manifest = _pkg._read_starlark_manifest() + app_data = manifest.get('apps', {}).get(app_id) + if not app_data: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + + enabled = data.get('enabled') + if enabled is None: + enabled = not app_data.get('enabled', True) + app_data['enabled'] = enabled + if _pkg._write_starlark_manifest(manifest): + return jsonify({'status': 'success', 'message': f"App {'enabled' if enabled else 'disabled'}", 'enabled': enabled}) + else: + return jsonify({'status': 'error', 'message': 'Failed to save'}), 500 + + except Exception as e: + logger.exception("[Starlark] toggle_starlark_app failed") + return jsonify({'status': 'error', 'message': 'Failed to toggle Starlark app', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/apps//render', methods=['POST']) +def render_starlark_app(app_id): + """Force render a Starlark app.""" + try: + app_dir, err = _validate_starlark_app_path(app_id) + if err: + return jsonify({'status': 'error', 'message': err}), 400 + + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + app = starlark_plugin.apps.get(app_id) + if not app: + return jsonify({'status': 'error', 'message': f'App not found: {app_id}'}), 404 + success = starlark_plugin._render_app(app, force=True) + if success: + return jsonify({'status': 'success', 'message': 'App rendered', + 'frame_count': len(app.frames) if app.frames else 0}) + return jsonify({'status': 'error', 'message': 'Failed to render app'}), 500 + + # Web-service context: plugin not loaded, call pixlet directly + success, status_code, error = _standalone_render_starlark_app(app_id) + if success: + return jsonify({'status': 'success', 'message': 'App rendered successfully', 'frame_count': 0}), status_code + return jsonify({'status': 'error', 'message': error or 'Render failed', 'frame_count': 0}), status_code + + except Exception as e: + logger.exception("[Starlark] render_starlark_app failed") + return jsonify({'status': 'error', 'message': 'Failed to render Starlark app', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/repository/browse', methods=['GET']) +def browse_tronbyte_repository(): + """Browse all apps in the Tronbyte repository (bulk cached fetch). + + Returns ALL apps with metadata, categories, and authors. + Filtering/sorting/pagination is handled client-side. + Results are cached server-side for 2 hours. + """ + try: + TronbyteRepository = _pkg._get_tronbyte_repository_class() + + repo = TronbyteRepository(github_token=_starlark_github_token()) + + result = repo.list_all_apps_cached() + + rate_limit = repo.get_rate_limit_info() + + # An upstream failure used to arrive here as an empty app list and go + # out as 'success', so the store drew an empty grid and said nothing. + # 502: the request was fine, GitHub was not. + if result.get('error'): + return jsonify({ + 'status': 'error', + 'message': result['error'], + 'rate_limit': rate_limit, + }), 502 + + return jsonify({ + 'status': 'success', + 'apps': result['apps'], + 'categories': result['categories'], + 'authors': result['authors'], + 'count': result['count'], + 'cached': result['cached'], + 'rate_limit': rate_limit, + }) + + except Exception as e: + logger.exception("[Starlark] browse_tronbyte_repository failed") + return jsonify({'status': 'error', 'message': 'Failed to browse repository', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/repository/install', methods=['POST']) +def install_from_tronbyte_repository(): + """Install an app from the Tronbyte repository.""" + try: + data = request.get_json(silent=True) + if not data or 'app_id' not in data: + return jsonify({'status': 'error', 'message': 'app_id is required'}), 400 + + app_id, app_id_error = _validate_and_sanitize_app_id(data['app_id']) + if app_id_error: + return jsonify({'status': 'error', 'message': f'Invalid app_id: {app_id_error}'}), 400 + + TronbyteRepository = _pkg._get_tronbyte_repository_class() + import tempfile + + repo = TronbyteRepository(github_token=_starlark_github_token()) + + success, metadata, error = repo.get_app_metadata(data['app_id']) + if not success: + return jsonify({'status': 'error', 'message': f'Failed to fetch app metadata: {error}'}), 404 + + with tempfile.NamedTemporaryFile(delete=False, suffix='.star') as tmp: + temp_path = tmp.name + + try: + # Pass filename from metadata (e.g., "analog_clock.star" for analogclock app) + # Note: manifest uses 'fileName' (camelCase), not 'filename' + filename = metadata.get('fileName') if metadata else None + success, error = repo.download_star_file(data['app_id'], Path(temp_path), filename=filename) + if not success: + return jsonify({'status': 'error', 'message': f'Failed to download app: {error}'}), 500 + + # Download assets (images, sources, etc.) to a temp directory + import tempfile + temp_assets_dir = tempfile.mkdtemp() + try: + success_assets, error_assets = repo.download_app_assets(data['app_id'], Path(temp_assets_dir)) + # Asset download is non-critical - log warning but continue if it fails + if not success_assets: + logger.warning(f"Failed to download assets for {data['app_id']}: {error_assets}") + + render_interval = data.get('render_interval', 300) + ri, err = _validate_timing_value(render_interval, 'render_interval') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + render_interval = ri or 300 + + display_duration = data.get('display_duration', 15) + dd, err = _validate_timing_value(display_duration, 'display_duration') + if err: + return jsonify({'status': 'error', 'message': err}), 400 + display_duration = dd or 15 + + install_metadata = { + 'name': metadata.get('name', app_id) if metadata else app_id, + 'render_interval': render_interval, + 'display_duration': display_duration + } + + starlark_plugin = _pkg._get_starlark_plugin() + if starlark_plugin: + success = starlark_plugin.install_app(app_id, temp_path, install_metadata, assets_dir=temp_assets_dir) + else: + success = _install_star_file(app_id, temp_path, install_metadata, assets_dir=temp_assets_dir) + finally: + # Clean up temp assets directory + import shutil + try: + shutil.rmtree(temp_assets_dir) + except OSError: + pass + + if success: + return jsonify({'status': 'success', 'message': f'App installed: {metadata.get("name", app_id) if metadata else app_id}', 'app_id': app_id}) + else: + return jsonify({'status': 'error', 'message': 'Failed to install app'}), 500 + finally: + try: + os.unlink(temp_path) + except OSError: + pass + + except Exception as e: + logger.exception("[Starlark] install_from_tronbyte_repository failed") + return jsonify({'status': 'error', 'message': 'Failed to install from repository', 'details': describe_exception(e)}), 500 +@api_v3.route('/starlark/repository/categories', methods=['GET']) +def get_tronbyte_categories(): + """Get list of available app categories (uses bulk cache).""" + try: + TronbyteRepository = _pkg._get_tronbyte_repository_class() + repo = TronbyteRepository(github_token=_starlark_github_token()) + + result = repo.list_all_apps_cached() + + if result.get('error'): + return jsonify({'status': 'error', 'message': result['error']}), 502 + + return jsonify({'status': 'success', 'categories': result['categories']}) + + except Exception as e: + logger.exception("[Starlark] get_tronbyte_categories failed") + return jsonify({'status': 'error', 'message': 'Failed to fetch categories', 'details': describe_exception(e)}), 500 diff --git a/web_interface/blueprints/api_v3/system.py b/web_interface/blueprints/api_v3/system.py new file mode 100644 index 00000000..c9061faf --- /dev/null +++ b/web_interface/blueprints/api_v3/system.py @@ -0,0 +1,662 @@ +"""Service control, updates, versions and system status. + +Routes decorate the shared `api_v3` Blueprint from ._common, so their +endpoint names are unchanged by living here. +""" +from web_interface.blueprints.api_v3 import ( + Any, Dict, PROJECT_ROOT, Path, _GIT, _UPDATE_CHECK_TTL, + _describe_git_failure, _get_display_service_status, _git_current_branch, + _git_remote_branch_exists, _git_upstream, _pip_install_requirements, + _scrub_git_remote_url, _truncate_output, _update_check_cache, + _update_check_failed, api_v3, checkout_branch, describe_exception, + get_git_version, jsonify, logger, os, request, resolve_pull_command, + shutil, subprocess, +) +import web_interface.blueprints.api_v3 as _pkg +# Read through the module rather than bound by value: tests patch these +# as module attributes, and a value binding would not see the patch. +# Several are also called from helpers that live in __init__, so the +# package is the only patch point that covers every caller. + + +@api_v3.route('/system/status', methods=['GET']) +def get_system_status(): + """Get system status""" + try: + # Check cache first (10 second TTL for system status) + try: + from web_interface.cache import get_cached, set_cached + cached_result = get_cached('system_status', ttl_seconds=10) + if cached_result is not None: + return jsonify({'status': 'success', 'data': cached_result}) + except ImportError: + # Cache not available, continue without caching + get_cached = None + set_cached = None + + # Import psutil for system monitoring + try: + import psutil + except ImportError: + # Fallback if psutil not available + return jsonify({ + 'status': 'error', + 'message': 'psutil not available for system monitoring' + }), 503 + + # Get system metrics using psutil + cpu_percent = psutil.cpu_percent(interval=0.1) # Short interval for responsiveness + memory = psutil.virtual_memory() + memory_percent = memory.percent + disk = psutil.disk_usage('/') + disk_percent = disk.percent + + # Calculate uptime + boot_time = psutil.boot_time() + uptime_seconds = _pkg.time.time() - boot_time + uptime_hours = uptime_seconds / 3600 + uptime_days = uptime_hours / 24 + + # Format uptime string + if uptime_days >= 1: + uptime_str = f"{int(uptime_days)}d {int(uptime_hours % 24)}h" + elif uptime_hours >= 1: + uptime_str = f"{int(uptime_hours)}h {int((uptime_seconds % 3600) / 60)}m" + else: + uptime_str = f"{int(uptime_seconds / 60)}m" + + # Get CPU temperature (Raspberry Pi) + cpu_temp = None + try: + temp_file = '/sys/class/thermal/thermal_zone0/temp' + if os.path.exists(temp_file): + with open(temp_file, 'r') as f: + temp_millidegrees = int(f.read().strip()) + cpu_temp = temp_millidegrees / 1000.0 # Convert to Celsius + except (IOError, ValueError, OSError): + # Temperature sensor not available or error reading + cpu_temp = None + + # Get display service status + service_status = _get_display_service_status() + + status = { + 'timestamp': _pkg.time.time(), + 'uptime': uptime_str, + 'uptime_seconds': int(uptime_seconds), + 'service_active': service_status.get('active', False), + 'cpu_percent': round(cpu_percent, 1), + 'memory_used_percent': round(memory_percent, 1), + 'memory_total_mb': round(memory.total / (1024 * 1024), 1), + 'memory_used_mb': round(memory.used / (1024 * 1024), 1), + # MemAvailable, not total-minus-used: it accounts for reclaimable + # page cache, so it is what actually predicts memory trouble. A + # board can read 70% "used" and be fine, or read the same and be + # about to fail fork(), and only this number tells them apart. + 'memory_available_mb': round(memory.available / (1024 * 1024), 1), + 'cpu_temp': round(cpu_temp, 1) if cpu_temp is not None else None, + 'disk_used_percent': round(disk_percent, 1), + 'disk_total_gb': round(disk.total / (1024 * 1024 * 1024), 1), + 'disk_used_gb': round(disk.used / (1024 * 1024 * 1024), 1) + } + + # Cache the result if available + if set_cached: + try: + set_cached('system_status', status, ttl_seconds=10) + except Exception: + pass # Cache write failed, but continue + + return jsonify({'status': 'success', 'data': status}) + except Exception as e: + logger.error('Unhandled exception', exc_info=True) + return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 +@api_v3.route('/system/version', methods=['GET']) +def get_system_version(): + """Get LEDMatrix repository version""" + try: + version = get_git_version() + return jsonify({'status': 'success', 'data': {'version': version}}) + except Exception as e: + logger.error("get_system_version failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'Unable to retrieve version'}), 500 +@api_v3.route('/system/check-update', methods=['GET']) +def check_for_update(): + """Check whether a newer LEDMatrix commit is available on origin/main.""" + now = _pkg.time.time() + if _update_check_cache['result'] and now - _update_check_cache['ts'] < _UPDATE_CHECK_TTL: + return jsonify(_update_check_cache['result']) + + _safe: Dict[str, Any] = {'update_available': False, 'remote_sha': 'unknown', 'commits_behind': 0} + try: + cwd = str(PROJECT_ROOT) + fetch_result = subprocess.run( + ['git', 'fetch', 'origin', 'main', '--quiet'], + capture_output=True, timeout=10, cwd=cwd, + ) + if fetch_result.returncode != 0: + stderr = fetch_result.stderr.decode(errors='replace').strip() + logger.warning("check-update: git fetch failed (rc=%d): %s", + fetch_result.returncode, stderr) + failed = _update_check_failed(_describe_git_failure(stderr)) + _update_check_cache['result'] = failed + _update_check_cache['ts'] = now + return jsonify(failed) + local = subprocess.run( + ['git', 'rev-parse', 'HEAD'], + capture_output=True, text=True, timeout=5, cwd=cwd, + ).stdout.strip() + remote = subprocess.run( + ['git', 'rev-parse', 'origin/main'], + capture_output=True, text=True, timeout=5, cwd=cwd, + ).stdout.strip() + + if not local or not remote: + return jsonify(_safe) + + if local == remote: + result: Dict[str, Any] = {'update_available': False, 'remote_sha': remote, 'commits_behind': 0} + else: + count_str = subprocess.run( + ['git', 'rev-list', 'HEAD..origin/main', '--count'], + capture_output=True, text=True, timeout=5, cwd=cwd, + ).stdout.strip() + count = int(count_str) if count_str.isdigit() else 0 + result = {'update_available': count > 0, 'remote_sha': remote, 'commits_behind': count} + + _update_check_cache['result'] = result + _update_check_cache['ts'] = now + return jsonify(result) + except Exception as e: + logger.warning("check-update failed: %s", e) + return jsonify(_update_check_failed( + "Could not check for updates; see logs for details.")) +#: sudo's own wording when it needs a password it cannot ask for. The web +#: interface runs unprivileged, so its systemctl/reboot/journalctl calls only +#: work once scripts/install/configure_web_sudo.sh has granted NOPASSWD -- +#: which first_time_install.sh does not do. That makes this the common case on +#: a fresh device, and "Action failed; see logs for details" named none of it, +#: while the log viewer was broken for the very same reason. +_SUDO_NEEDS_PASSWORD = ( + 'a password is required', + 'no tty present', + 'a terminal is required', +) + +_SUDO_HINT = ( + 'Passwordless sudo is not configured for the web interface user, so this ' + 'action cannot run. Run scripts/install/configure_web_sudo.sh as that user, ' + 'then retry.' +) + + +def _sudo_hint_for(text): + """An actionable hint when `text` is sudo refusing to prompt, else None.""" + lowered = (text or '').lower() + if any(marker in lowered for marker in _SUDO_NEEDS_PASSWORD): + return _SUDO_HINT + return None + + +@api_v3.route('/system/action', methods=['POST']) +def execute_system_action(): + """Execute system actions (start/stop/reboot/etc)""" + try: + # HTMX sends data as form data, not JSON + data = request.get_json(silent=True) or {} + if not data: + # Try to get from form data if JSON fails + data = { + 'action': request.form.get('action'), + 'mode': request.form.get('mode') + } + + if not data or 'action' not in data: + return jsonify({'status': 'error', 'message': 'Action required'}), 400 + + action = data['action'] + mode = data.get('mode') # For on-demand modes + + # Map actions to subprocess calls (similar to original implementation) + if action == 'start_display': + if mode: + # For on-demand modes, we would need to integrate with the display controller + # For now, just start the display service + try: + result = subprocess.run(['sudo', 'systemctl', 'start', 'ledmatrix.service'], + capture_output=True, text=True, timeout=10) + except subprocess.TimeoutExpired as e: + logger.error("start_display (%s) timed out: %s", mode, e) + return jsonify({'status': 'error', 'message': 'Command timed out', 'returncode': -1, 'stderr': 'timeout'}) + logger.info("start_display (%s) returned code %d", mode, result.returncode) + if result.returncode != 0 and result.stderr: + logger.error("start_display (%s) stderr: %s", mode, result.stderr.strip()) + resp = { + 'status': 'success' if result.returncode == 0 else 'error', + # This branch returns before the shared nonzero-result + # response below, so it needs the hint of its own or an + # on-demand start reports "Failed to start display" and + # says nothing about the sudo that actually refused it. + 'message': ( + 'Display started' if result.returncode == 0 + else _sudo_hint_for(result.stderr) or 'Failed to start display' + ), + } + if result.returncode != 0: + resp['returncode'] = result.returncode + resp['stderr'] = result.stderr.strip() + return jsonify(resp) + else: + result = subprocess.run(['sudo', 'systemctl', 'start', 'ledmatrix.service'], + capture_output=True, text=True, timeout=10) + elif action == 'stop_display': + result = subprocess.run(['sudo', 'systemctl', 'stop', 'ledmatrix.service'], + capture_output=True, text=True, timeout=10) + elif action == 'enable_autostart': + result = subprocess.run(['sudo', 'systemctl', 'enable', 'ledmatrix.service'], + capture_output=True, text=True, timeout=10) + elif action == 'disable_autostart': + result = subprocess.run(['sudo', 'systemctl', 'disable', 'ledmatrix.service'], + capture_output=True, text=True, timeout=10) + elif action == 'reboot_system': + result = subprocess.run(['sudo', 'reboot'], + capture_output=True, text=True, timeout=10) + elif action == 'shutdown_system': + result = subprocess.run(['sudo', 'poweroff'], + capture_output=True, text=True, timeout=10) + elif action == 'git_pull': + # Use PROJECT_ROOT instead of hardcoded path + project_dir = str(PROJECT_ROOT) + + # Decide how to pull BEFORE stashing. If this checkout cannot be + # updated at all, stashing first would put the user's local changes + # away for an update that was never going to run. + pull_args, upstream_note, pull_error = resolve_pull_command(project_dir) + if pull_error: + logger.warning("git pull not attempted: %s", pull_error) + return jsonify({'status': 'error', 'message': pull_error}) + + # Check if there are local changes that need to be stashed + # Exclude plugins directory - plugins are separate repos and shouldn't be stashed with base project + # Use --untracked-files=no to skip untracked files check (much faster with symlinked plugins) + try: + status_result = subprocess.run( + ['git', 'status', '--porcelain', '--untracked-files=no'], + capture_output=True, + text=True, + timeout=30, + cwd=project_dir + ) + # Filter out any changes in plugins directory - plugins are separate repositories + # Git status format: XY filename (where X is status of index, Y is status of work tree) + status_lines = [line for line in status_result.stdout.strip().split('\n') + if line.strip() and 'plugins/' not in line] + has_changes = bool('\n'.join(status_lines).strip()) + except subprocess.TimeoutExpired: + # If status check times out, assume there might be changes and proceed + # This is safer than failing the update + has_changes = True + status_result = type('obj', (object,), {'stdout': '', 'stderr': 'Status check timed out'})() + + stash_info = "" + + # Stash local changes if they exist (excluding plugins) + # Plugins are separate repositories and shouldn't be stashed with base project updates + if has_changes: + try: + # Use pathspec to exclude plugins directory from stash + stash_result = subprocess.run( + ['git', 'stash', 'push', '-m', 'LEDMatrix auto-stash before update', '--', ':!plugins'], + capture_output=True, + text=True, + timeout=30, + cwd=project_dir + ) + if stash_result.returncode == 0: + logger.debug("git stash: stashed local changes before pull") + stash_info = " Local changes were stashed." + else: + logger.warning("git stash failed before pull (returncode=%d)", stash_result.returncode) + except subprocess.TimeoutExpired: + logger.warning("git stash timed out, proceeding with pull") + + # Record HEAD before the pull so dependency changes can be detected + old_head = None + try: + _pre = subprocess.run(['git', 'rev-parse', 'HEAD'], + capture_output=True, text=True, timeout=10, cwd=project_dir) + if _pre.returncode == 0: + old_head = _pre.stdout.strip() + except subprocess.TimeoutExpired: + logger.warning("git rev-parse timed out before pull") + + # Whether the pull actually brought new code in. "Already up to + # date" is a success too, and prompting for a restart then would + # train users to ignore the prompt. + code_changed = False + + # Perform the git pull. Branches without an upstream were given + # an explicit "origin " above so the update still works. + result = subprocess.run( + pull_args, + capture_output=True, + text=True, + timeout=60, + cwd=project_dir + ) + + # Give the branch tracking information so the next pull is a plain + # `git pull` — otherwise every update repeats the fallback. + if result.returncode == 0 and upstream_note: + branch = _git_current_branch(project_dir) + if branch: + try: + subprocess.run( + ['git', 'branch', f'--set-upstream-to=origin/{branch}', branch], + capture_output=True, text=True, timeout=10, cwd=project_dir) + except (subprocess.TimeoutExpired, OSError) as exc: + logger.debug("could not set upstream for %s: %s", branch, exc) + + # Return custom response for git_pull + if result.returncode == 0: + pull_message = "Code updated successfully." + if has_changes: + pull_message = f"Code updated successfully. Local changes were automatically stashed.{stash_info}" + if result.stdout and "Already up to date" not in result.stdout: + pull_message = f"Code updated successfully.{stash_info}" + if upstream_note: + pull_message = f"{pull_message} {upstream_note}" + + # Keep Python dependencies in sync automatically: if the pull + # changed a requirements file, install it now — users updating + # from the web UI (most of them) never SSH in to pip install. + # Installs go through the same root-visible path as the + # Tools-tab buttons (_pip_install_requirements). + dep_notes = [] + try: + _post = subprocess.run(['git', 'rev-parse', 'HEAD'], + capture_output=True, text=True, timeout=10, cwd=project_dir) + new_head = _post.stdout.strip() if _post.returncode == 0 else None + if old_head and new_head and old_head != new_head: + code_changed = True + diff = subprocess.run( + ['git', 'diff', '--name-only', f'{old_head}..{new_head}'], + capture_output=True, text=True, timeout=15, cwd=project_dir) + changed = set(diff.stdout.split()) if diff.returncode == 0 else set() + for rel in ('requirements.txt', 'web_interface/requirements.txt'): + req_path = PROJECT_ROOT / rel + if rel not in changed or not req_path.exists(): + continue + # Each file's install is isolated: a timeout or + # OSError (e.g. the sudo wrapper/interpreter + # missing) on one file must not abort the other. + try: + r = _pip_install_requirements(req_path, timeout=180) + if r.returncode == 0: + dep_notes.append(f"Dependencies from {rel} updated.") + else: + dep_notes.append( + f"Dependency install from {rel} failed — " + "run Install Base Requirements from the Tools tab.") + logger.warning("post-update pip install failed for %s: %s", + rel, _truncate_output(r.stdout, r.stderr)) + except subprocess.TimeoutExpired: + dep_notes.append( + f"Dependency install from {rel} timed out — " + "run Install Base Requirements from the Tools tab.") + logger.warning("post-update pip install timed out for %s", rel) + except OSError as install_err: + dep_notes.append( + f"Dependency install from {rel} failed — " + "run Install Base Requirements from the Tools tab.") + logger.warning("post-update pip install errored for %s: %s", + rel, install_err) + except subprocess.TimeoutExpired: + logger.warning("post-update dependency sync timed out") + if dep_notes: + pull_message += " " + " ".join(dep_notes) + # A `git pull` restores built-in plugins (committed under + # plugin-repos/) even if the user uninstalled them. Re-remove + # any the user previously uninstalled so the update doesn't + # resurrect them. + if api_v3.plugin_store_manager: + try: + purged = api_v3.plugin_store_manager.purge_uninstalled_plugins() + if purged: + logger.info( + "Re-removed %d uninstalled plugin(s) restored by update: %s", + len(purged), ", ".join(purged), + ) + except (OSError, RuntimeError) as purge_err: + logger.warning("Post-update plugin purge failed: %s", purge_err) + else: + logger.warning("git pull failed (returncode=%d): %s", result.returncode, result.stderr) + # Show git's own first line: "check logs" leaves the user with + # nothing to act on, and these failures are usually actionable + # (conflicting local commits, no upstream, network). + detail = next((ln.strip() for ln in (result.stderr or '').splitlines() + if ln.strip()), '') + pull_message = f"Update failed: {detail}" if detail else "Update failed; check logs for details" + + # Nothing here restarts anything: the pull replaces files on + # disk while the display and web services keep running the code + # they loaded at boot. Without this the user is told the update + # succeeded and sees no change until they happen to reboot. + return jsonify({ + 'status': 'success' if result.returncode == 0 else 'error', + 'message': pull_message, + 'restart_required': bool(result.returncode == 0 and code_changed), + }) + elif action == 'checkout_branch': + # Switch branches from the Tools tab. Needed because a checkout + # that predates tracking (or a restored backup) can leave the pi + # on a branch the update button cannot pull. + result_payload, http_status = checkout_branch( + str(PROJECT_ROOT), data.get('branch') or '', stash=bool(data.get('stash'))) + return jsonify(result_payload), http_status + + elif action == 'restart_display_service': + result = subprocess.run(['sudo', 'systemctl', 'restart', 'ledmatrix.service'], + capture_output=True, text=True, timeout=10) + elif action == 'restart_web_service': + # Try to restart the web service (assuming it's ledmatrix-web.service) + result = subprocess.run(['sudo', 'systemctl', 'restart', 'ledmatrix-web.service'], + capture_output=True, text=True, timeout=10) + elif action == 'install_base_requirements': + # Base + web interface requirements: flask-compress and friends + # live in web_interface/requirements.txt, not the root file. + req_files = [f for f in (PROJECT_ROOT / 'requirements.txt', + PROJECT_ROOT / 'web_interface' / 'requirements.txt') + if f.exists()] + if not req_files: + return jsonify({'status': 'error', 'message': 'No requirements.txt found at project root'}) + outputs = [] + all_ok = True + for req_file in req_files: + label = req_file.relative_to(PROJECT_ROOT) + # Isolate each file's install: a timeout or OSError on one + # (e.g. requirements.txt) must not abort the rest of the + # loop (e.g. web_interface/requirements.txt never attempted). + try: + result = _pip_install_requirements(req_file, timeout=120) + all_ok = all_ok and result.returncode == 0 + outputs.append(f"== {label} ==\n" + _truncate_output(result.stdout, result.stderr)) + except subprocess.TimeoutExpired: + all_ok = False + outputs.append(f"== {label} ==\nTimed out after 120s") + logger.warning("install_base_requirements timed out for %s", label) + except OSError as install_err: + all_ok = False + outputs.append(f"== {label} ==\nFailed: {install_err}") + logger.warning("install_base_requirements errored for %s: %s", label, install_err) + return jsonify({ + 'status': 'success' if all_ok else 'error', + 'message': 'Base requirements installed successfully' if all_ok else 'pip install failed', + 'output': "\n".join(outputs) + }) + elif action == 'install_plugin_requirements': + active_pm = getattr(api_v3, 'plugin_manager', None) + if active_pm: + plugins_dir = Path(active_pm.plugins_dir) + else: + _cm = getattr(api_v3, 'config_manager', None) + _cfg = _cm.load_config() if _cm else {} + _dir_name = _cfg.get('plugin_system', {}).get('plugins_directory', 'plugin-repos') + plugins_dir = Path(_dir_name) if os.path.isabs(_dir_name) else PROJECT_ROOT / _dir_name + results = [] + if plugins_dir.exists(): + for p in sorted(plugins_dir.iterdir()): + req = p / 'requirements.txt' + if p.is_dir() and req.exists(): + try: + r = _pip_install_requirements(req, timeout=60) + results.append({ + 'plugin': p.name, + 'ok': r.returncode == 0, + 'output': _truncate_output(r.stdout, r.stderr) + }) + except subprocess.TimeoutExpired: + results.append({'plugin': p.name, 'ok': False, 'output': 'pip install timed out'}) + except OSError as exc: + results.append({'plugin': p.name, 'ok': False, 'output': exc.strerror or 'OS error'}) + ok_count = sum(1 for r in results if r['ok']) + all_ok = all(r['ok'] for r in results) if results else True + return jsonify({ + 'status': 'success' if all_ok else 'error', + 'message': f'Processed {len(results)} plugin(s) — {ok_count} succeeded' if results else 'No plugin requirements.txt files found', + 'details': results + }) + elif action == 'force_git_reset': + if not _GIT: + return jsonify({'status': 'error', 'message': 'git not found on this system'}), 503 + project_dir = str(PROJECT_ROOT) + fetch = subprocess.run( + [_GIT, 'fetch', 'origin'], + capture_output=True, text=True, timeout=30, cwd=project_dir + ) + if fetch.returncode != 0: + return jsonify({'status': 'error', 'message': 'git fetch failed', 'output': fetch.stderr.strip()}) + reset = subprocess.run( + [_GIT, 'reset', '--hard', 'origin/main'], + capture_output=True, text=True, timeout=30, cwd=project_dir + ) + return jsonify({ + 'status': 'success' if reset.returncode == 0 else 'error', + 'message': 'Reset to origin/main successfully' if reset.returncode == 0 else 'git reset failed', + 'output': (reset.stdout + reset.stderr).strip() + }) + elif action == 'clear_pycache': + cleared = 0 + failed = 0 + for d in PROJECT_ROOT.rglob('__pycache__'): + if d.is_dir(): + try: + shutil.rmtree(d) + cleared += 1 + except OSError: + failed += 1 + msg = f'Cleared {cleared} __pycache__ directories' + if failed: + msg += f' ({failed} could not be removed)' + return jsonify({'status': 'success', 'message': msg}) + else: + return jsonify({'status': 'error', 'message': 'Unknown action'}), 400 + + logger.info("system action '%s' returncode=%d", action, result.returncode) + if result.returncode != 0 and result.stderr: + logger.error("system action '%s' stderr: %s", action, result.stderr.strip()) + resp = { + 'status': 'success' if result.returncode == 0 else 'error', + 'message': 'Action completed' if result.returncode == 0 else 'Action failed; check logs for details', + } + if result.returncode != 0: + resp['returncode'] = result.returncode + resp['stderr'] = result.stderr.strip() + hint = _sudo_hint_for(result.stderr) + if hint: + resp['message'] = hint + return jsonify(resp) + + except subprocess.TimeoutExpired as e: + logger.error("system action '%s' timed out: %s", action, e) + return jsonify({'status': 'error', 'message': 'Command timed out', 'returncode': -1, 'stderr': 'timeout'}) + except Exception as e: + logger.error("execute_system_action failed: %s", e, exc_info=True) + detail = describe_exception(e) + resp = { + 'status': 'error', + 'message': _sudo_hint_for(detail) or 'Action failed; see logs for details', + 'details': detail, + } + return jsonify(resp), 500 +@api_v3.route('/system/git-info', methods=['GET']) +def get_git_info(): + """Return branch, dirty state, recent commits and remote URL for the Tools tab.""" + if not _GIT: + return jsonify({'status': 'error', 'message': 'git not found on this system'}), 503 + d = str(PROJECT_ROOT) + try: + branch = subprocess.run([_GIT, 'branch', '--show-current'], capture_output=True, text=True, timeout=10, cwd=d) + if branch.returncode != 0: + return jsonify({'status': 'error', 'message': f'git branch failed: {branch.stderr.strip()}'}), 500 + + status = subprocess.run([_GIT, 'status', '--short', '--untracked-files=no'], capture_output=True, text=True, timeout=15, cwd=d) + if status.returncode != 0: + return jsonify({'status': 'error', 'message': f'git status failed: {status.stderr.strip()}'}), 500 + + log = subprocess.run([_GIT, 'log', '--oneline', '-5'], capture_output=True, text=True, timeout=10, cwd=d) + remote = subprocess.run([_GIT, 'remote', 'get-url', 'origin'], capture_output=True, text=True, timeout=10, cwd=d) + branch_name = branch.stdout.strip() + upstream = _git_upstream(d) + return jsonify({ + 'branch': branch_name, + 'dirty': bool(status.stdout.strip()), + 'status': status.stdout.strip(), + 'recent_commits': log.stdout.strip() if log.returncode == 0 else '', + 'remote_url': _scrub_git_remote_url(remote.stdout.strip()) if remote.returncode == 0 else '', + # Surfaced so the Tools tab can warn before the user clicks Pull + # Latest, rather than after it fails. + 'upstream': upstream, + 'can_pull': bool(upstream) or _git_remote_branch_exists(d, branch_name), + }) + except Exception as e: + logger.error("get_git_info failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'Failed to get git info'}), 500 +@api_v3.route('/system/git-branches', methods=['GET']) +def get_git_branches(): + """List branches available to switch to, for the Tools tab picker.""" + if not _GIT: + return jsonify({'status': 'error', 'message': 'git not found on this system'}), 503 + d = str(PROJECT_ROOT) + try: + # Refresh remote refs so a branch created since the last fetch shows up. + subprocess.run([_GIT, 'fetch', 'origin', '--prune'], + capture_output=True, text=True, timeout=60, cwd=d) + + local = subprocess.run([_GIT, 'for-each-ref', '--format=%(refname:short)', 'refs/heads'], + capture_output=True, text=True, timeout=15, cwd=d) + remote = subprocess.run([_GIT, 'for-each-ref', '--format=%(refname:short)', 'refs/remotes/origin'], + capture_output=True, text=True, timeout=15, cwd=d) + if local.returncode != 0: + return jsonify({'status': 'error', 'message': 'Could not list branches'}), 500 + + local_names = [b for b in local.stdout.split() if b] + remote_names = [] + for ref in remote.stdout.split() if remote.returncode == 0 else []: + name = ref.split('origin/', 1)[-1] + # origin/HEAD is a symbolic alias, not a branch a user can pick. + if name and name != 'HEAD' and name not in local_names: + remote_names.append(name) + + return jsonify({ + 'status': 'success', + 'current': _git_current_branch(d), + 'upstream': _git_upstream(d), + 'local': sorted(local_names), + 'remote_only': sorted(remote_names), + }) + except subprocess.TimeoutExpired: + return jsonify({'status': 'error', 'message': 'Timed out talking to the remote'}), 504 + except OSError as e: + logger.error("get_git_branches failed: %s", e, exc_info=True) + return jsonify({'status': 'error', 'message': 'Failed to list branches'}), 500 diff --git a/web_interface/blueprints/api_v3/wifi.py b/web_interface/blueprints/api_v3/wifi.py new file mode 100644 index 00000000..065a4b63 --- /dev/null +++ b/web_interface/blueprints/api_v3/wifi.py @@ -0,0 +1,413 @@ +"""Wi-Fi scanning, connection and status routes. + +Routes decorate the shared `api_v3` Blueprint from ._common, so their +endpoint names are unchanged by living here. +""" +from web_interface.blueprints.api_v3 import ( + api_v3, describe_exception, jsonify, logger, request, +) + + +def _parse_bool_ish(value): + """Coerce a JSON value that is supposed to be a boolean. + + A JSON boolean arrives as a real Python bool, but these routes are a + public HTTP contract and not every caller sends one. `bool(value)` gets + two common cases wrong: `bool("false")` is True (a non-empty string is + always truthy), and a plain int does not match an `is True` check + (`1 is True` is False, since `True` is a distinct singleton from the int + `1`) -- so a caller sending `{"enabled": 1}` was silently treated as + False. Recognizes a real bool, "true"/"false"/"1"/"0"/"yes"/"no" + case-insensitively, and int 1/0. + + Returns None for anything else, rather than guessing. A supplied-but- + unrecognized value (e.g. a typo) used to silently become False here, + which for `enabled` on the radio route could disconnect Wi-Fi the caller + never asked to turn off -- callers must treat None as a validation + error, not a real False. + """ + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in ('true', '1', 'yes'): + return True + if lowered in ('false', '0', 'no'): + return False + return None + if isinstance(value, int): + if value == 1: + return True + if value == 0: + return False + return None + return None + + +# WiFi Management Endpoints +@api_v3.route('/wifi/status', methods=['GET']) +def get_wifi_status(): + """Get current WiFi connection status""" + try: + from src.wifi_manager import WiFiManager + + wifi_manager = WiFiManager() + status = wifi_manager.get_wifi_status() + + # Get auto-enable setting from config + auto_enable_ap = wifi_manager.config.get("auto_enable_ap_mode", True) # Default: True (safe due to grace period) + + return jsonify({ + 'status': 'success', + 'data': { + 'connected': status.connected, + 'ssid': status.ssid, + 'ip_address': status.ip_address, + 'signal': status.signal, + 'ap_mode_active': status.ap_mode_active, + 'auto_enable_ap_mode': auto_enable_ap + } + }) + except Exception as e: + logger.error("%s failed", request.path, exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) + }), 500 +@api_v3.route('/wifi/scan', methods=['GET']) +def scan_wifi_networks(): + """Scan for available WiFi networks + + If AP mode is active, it will be temporarily disabled during scanning + and automatically re-enabled afterward. Users connected to the AP will + be briefly disconnected during this process. + """ + try: + from src.wifi_manager import WiFiManager + + wifi_manager = WiFiManager() + + # Check if AP mode is active before scanning (for user notification) + ap_was_active = wifi_manager._is_ap_mode_active() + + # Perform the scan (this will handle AP mode disabling/enabling internally) + networks, _was_cached = wifi_manager.scan_networks() + + # Convert to dict format + networks_data = [ + { + 'ssid': net.ssid, + 'signal': net.signal, + 'security': net.security, + 'frequency': net.frequency + } + for net in networks + ] + + response_data = { + 'status': 'success', + 'data': networks_data + } + + # Inform user if AP mode was temporarily disabled + if ap_was_active: + response_data['message'] = ( + f'Found {len(networks_data)} networks. ' + 'Note: AP mode was temporarily disabled during scanning and has been re-enabled. ' + 'If you were connected to the setup network, you may need to reconnect.' + ) + + return jsonify(response_data) + except Exception as e: + logger.error("Error scanning WiFi networks", exc_info=True) + error_message = 'An error occurred while scanning WiFi networks; see logs for details' + + # Provide more specific error messages for common issues + error_str = str(e).lower() + if 'permission' in error_str or 'sudo' in error_str: + error_message = ( + 'Permission error while scanning. ' + 'The WiFi scan requires appropriate permissions. ' + 'Please ensure the application has necessary privileges.' + ) + elif 'timeout' in error_str: + error_message = ( + 'WiFi scan timed out. ' + 'The scan took too long to complete. ' + 'This may happen if the WiFi interface is busy or in use.' + ) + elif 'no wifi' in error_str or 'not available' in error_str: + error_message = ( + 'WiFi scanning tools are not available. ' + 'Please ensure NetworkManager (nmcli) or iwlist is installed.' + ) + + return jsonify({ + 'status': 'error', + 'message': error_message + }), 500 +@api_v3.route('/wifi/connect', methods=['POST']) +def connect_wifi(): + """Connect to a WiFi network""" + try: + from src.wifi_manager import WiFiManager + + data = request.get_json(silent=True) + if not data: + return jsonify({ + 'status': 'error', + 'message': 'Request body is required' + }), 400 + + if 'ssid' not in data: + return jsonify({ + 'status': 'error', + 'message': 'SSID is required' + }), 400 + + ssid = data['ssid'] + if not ssid or not ssid.strip(): + return jsonify({ + 'status': 'error', + 'message': 'SSID cannot be empty' + }), 400 + + ssid = ssid.strip() + password = data.get('password', '') or '' + + wifi_manager = WiFiManager() + success, message = wifi_manager.connect_to_network(ssid, password) + + if success: + return jsonify({ + 'status': 'success', + 'message': message + }) + else: + return jsonify({ + 'status': 'error', + 'message': message or 'Failed to connect to network' + }), 400 + except Exception as e: + logger.error("Error connecting to WiFi", exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) + }), 500 +@api_v3.route('/wifi/disconnect', methods=['POST']) +def disconnect_wifi(): + """Disconnect from the current WiFi network""" + try: + from src.wifi_manager import WiFiManager + + wifi_manager = WiFiManager() + success, message = wifi_manager.disconnect_from_network() + + if success: + return jsonify({ + 'status': 'success', + 'message': message + }) + else: + return jsonify({ + 'status': 'error', + 'message': message or 'Failed to disconnect from network' + }), 400 + except Exception as e: + logger.error("Error disconnecting from WiFi", exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) + }), 500 +@api_v3.route('/wifi/ap/enable', methods=['POST']) +def enable_ap_mode(): + """Enable access point mode""" + try: + from src.wifi_manager import WiFiManager + + wifi_manager = WiFiManager() + _force_raw = (request.get_json(silent=True) or {}).get('force', False) + force = _force_raw is True or (isinstance(_force_raw, str) and _force_raw.lower() in ('true', '1')) + success, message = wifi_manager.enable_ap_mode(force=force) + + if success: + return jsonify({ + 'status': 'success', + 'message': message + }) + else: + return jsonify({ + 'status': 'error', + 'message': message + }), 400 + except Exception as e: + logger.error("%s failed", request.path, exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) + }), 500 +@api_v3.route('/wifi/ap/disable', methods=['POST']) +def disable_ap_mode(): + """Disable access point mode""" + try: + from src.wifi_manager import WiFiManager + + wifi_manager = WiFiManager() + success, message = wifi_manager.disable_ap_mode() + + if success: + return jsonify({ + 'status': 'success', + 'message': message + }) + else: + return jsonify({ + 'status': 'error', + 'message': message + }), 400 + except Exception as e: + logger.error("%s failed", request.path, exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) + }), 500 +@api_v3.route('/wifi/ap/auto-enable', methods=['GET']) +def get_auto_enable_ap_mode(): + """Get auto-enable AP mode setting""" + try: + from src.wifi_manager import WiFiManager + + wifi_manager = WiFiManager() + auto_enable = wifi_manager.config.get("auto_enable_ap_mode", True) # Default: True (safe due to grace period) + + return jsonify({ + 'status': 'success', + 'data': { + 'auto_enable_ap_mode': auto_enable + } + }) + except Exception as e: + logger.error("%s failed", request.path, exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) + }), 500 +@api_v3.route('/wifi/ap/auto-enable', methods=['POST']) +def set_auto_enable_ap_mode(): + """Set auto-enable AP mode setting""" + try: + from src.wifi_manager import WiFiManager + + data = request.get_json(silent=True) + if data is None or 'auto_enable_ap_mode' not in data: + return jsonify({ + 'status': 'error', + 'message': 'auto_enable_ap_mode is required' + }), 400 + + auto_enable = _parse_bool_ish(data['auto_enable_ap_mode']) + if auto_enable is None: + return jsonify({ + 'status': 'error', + 'message': 'auto_enable_ap_mode must be a boolean' + }), 400 + + wifi_manager = WiFiManager() + wifi_manager.config["auto_enable_ap_mode"] = auto_enable + wifi_manager._save_config() + + return jsonify({ + 'status': 'success', + 'message': f'Auto-enable AP mode set to {auto_enable}', + 'data': { + 'auto_enable_ap_mode': auto_enable + } + }) + except Exception as e: + logger.error("%s failed", request.path, exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', + 'details': describe_exception(e) + }), 500 +@api_v3.route('/wifi/radio', methods=['GET']) +def get_wifi_radio(): + """Get current WiFi radio state (enabled/disabled) and wired-fallback status.""" + try: + from src.wifi_manager import WiFiManager + + wifi_manager = WiFiManager() + state = wifi_manager.get_wifi_radio_state() + + return jsonify({ + 'status': 'success', + 'data': state + }) + except Exception as e: + logger.error("Error getting WiFi radio state", exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) + }), 500 +@api_v3.route('/wifi/radio', methods=['POST']) +def set_wifi_radio(): + """Turn the WiFi radio on or off. + + Body: {"enabled": bool, "force": bool (optional)}. Disabling is refused + unless Ethernet is connected or force=True, to avoid locking the user out + of this web interface. + """ + try: + from src.wifi_manager import WiFiManager + + data = request.get_json(silent=True) or {} + if 'enabled' not in data: + return jsonify({ + 'status': 'error', + 'message': 'enabled is required' + }), 400 + + # Parse defensively: bool("false") is True and a plain int never + # matches `is True`, so `_parse_bool_ish` handles bool, string and + # int 1/0 — the endpoint is a public contract, not just the shipped + # UI (which always sends real JSON booleans). An unrecognized value + # must be rejected, not silently disable the radio: this is the + # route that can drop the caller's own connection to this interface. + enabled = _parse_bool_ish(data['enabled']) + if enabled is None: + return jsonify({ + 'status': 'error', + 'message': 'enabled must be a boolean' + }), 400 + force = _parse_bool_ish(data.get('force', False)) + if force is None: + return jsonify({ + 'status': 'error', + 'message': 'force must be a boolean' + }), 400 + + wifi_manager = WiFiManager() + success, message, reason = wifi_manager.set_wifi_radio(enabled, force=force) + + if success: + return jsonify({ + 'status': 'success', + 'message': message, + 'data': wifi_manager.get_wifi_radio_state() + }) + else: + return jsonify({ + 'status': 'error', + 'message': message, + 'reason': reason + }), 400 + except Exception as e: + logger.error("Error setting WiFi radio state", exc_info=True) + return jsonify({ + 'status': 'error', + 'message': 'An error occurred; see logs for details', 'details': describe_exception(e) + }), 500 From a3d505384ded33b760b2520cb4b4508a71705d36 Mon Sep 17 00:00:00 2001 From: ant456 Date: Fri, 11 Sep 2026 11:22:52 -0400 Subject: [PATCH 13/16] Add render_width/render_height support to Starlark Apps (#552) --- plugin-repos/starlark-apps/manager.py | 16 ++++++++--- plugin-repos/starlark-apps/pixlet_renderer.py | 27 ++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/plugin-repos/starlark-apps/manager.py b/plugin-repos/starlark-apps/manager.py index 999b98a7..b3f26118 100644 --- a/plugin-repos/starlark-apps/manager.py +++ b/plugin-repos/starlark-apps/manager.py @@ -805,15 +805,25 @@ class StarlarkAppsPlugin(BasePlugin): magnify = self._get_effective_magnify() self.logger.debug(f"Using magnify={magnify} for {app.app_id}") - # Filter out LEDMatrix-internal timing keys before passing to pixlet - INTERNAL_KEYS = {'render_interval', 'display_duration'} + # Optional native render size for an app whose own declared canvas + # differs from Pixlet's 64x32 default -- without this an app + # declaring a wider native canvas got half its own content + # clipped at render time, before magnify ever got a chance to + # scale anything. + render_width = app.config.get("render_width") + render_height = app.config.get("render_height") + + # Filter out LEDMatrix-internal timing/sizing keys before passing to pixlet + INTERNAL_KEYS = {'render_interval', 'display_duration', 'render_width', 'render_height'} pixlet_config = {k: v for k, v in app.config.items() if k not in INTERNAL_KEYS} success, error = self.pixlet.render( star_file=str(app.star_file), output_path=str(app.cache_file), config=pixlet_config, - magnify=magnify + magnify=magnify, + width=render_width, + height=render_height ) if not success: diff --git a/plugin-repos/starlark-apps/pixlet_renderer.py b/plugin-repos/starlark-apps/pixlet_renderer.py index 29d75611..7f7f05e6 100644 --- a/plugin-repos/starlark-apps/pixlet_renderer.py +++ b/plugin-repos/starlark-apps/pixlet_renderer.py @@ -218,7 +218,9 @@ class PixletRenderer: star_file: str, output_path: str, config: Optional[Dict[str, Any]] = None, - magnify: int = 1 + magnify: int = 1, + width: Optional[int] = None, + height: Optional[int] = None ) -> Tuple[bool, Optional[str]]: """ Render a .star file to WebP output. @@ -228,6 +230,21 @@ class PixletRenderer: output_path: Where to save WebP output config: Configuration dictionary to pass to app magnify: Magnification factor (default 1) + width: Optional native render width in pixels. Previously + there was no way to tell Pixlet to render at anything + other than its own default (64), relying entirely on + magnify to scale up afterward -- fine for apps designed + at that native size, but wrong for an app whose own + declared canvas size is genuinely different (confirmed + on real hardware, 2026-09-06, with an imported app + declaring width=128: rendering at the default 64 and + then magnifying silently clipped half the app's own + content before scaling ever happened, rather than + producing a correctly-sized image). Passed through as + Pixlet's own -w flag when provided; omitted (Pixlet's + default) otherwise, preserving existing behavior for + every other app. + height: Same as width, for Pixlet's -t flag. Returns: Tuple of (success: bool, error_message: Optional[str]) @@ -287,6 +304,10 @@ class PixletRenderer: "-o", output_path, "-m", str(magnify) ]) + if width is not None: + cmd.extend(["-w", str(width)]) + if height is not None: + cmd.extend(["-t", str(height)]) # Build sanitized command for logging (redact sensitive values) sanitized_cmd = [self.pixlet_binary, "render", star_file] @@ -294,6 +315,10 @@ class PixletRenderer: config_keys = list(config.keys()) sanitized_cmd.append(f"[{len(config_keys)} config entries: {', '.join(config_keys)}]") sanitized_cmd.extend(["-o", output_path, "-m", str(magnify)]) + if width is not None: + sanitized_cmd.extend(["-w", str(width)]) + if height is not None: + sanitized_cmd.extend(["-t", str(height)]) logger.debug(f"Executing Pixlet: {' '.join(sanitized_cmd)}") # Execute rendering From 5137e86d16c27d3fb73fe25be0cac6b9801448f4 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:32:12 -0400 Subject: [PATCH 14/16] feat(tools): MQTT bridge and Pixlet editor, ported onto the api_v3 split (#554) * feat(tools): manage the MQTT bridge and Pixlet editor from the Tools tab PR #544's change, ported onto the api_v3 package split (#553). Identical behaviour; only the placement of the new code differs. The original added 508 lines to web_interface/blueprints/api_v3.py, which #553 deletes, so every hunk of it would conflict irreconcilably. Ported by AST: 26 new top-level items sorted to where the split puts each kind -- __init__.py 2 imports, 11 constants, 7 helpers starlark.py 4 routes (/starlark/editor/{apps,status,start,stop}) misc.py 2 routes (/integrations/mqtt-bridge{,/config}) Everything outside api_v3.py -- the Tools partial, the installer scripts, the JS tests -- applied unchanged. Routes: 111 from the split plus these 6 = 117, and the url-map snapshot is regenerated to match, which is exactly what test_api_v3_url_map.py is designed to make you do when routes are added. Full Python suite: 4,278 passed, 68 skipped, 0 failed. The JS tests this PR ships could not be run here -- node is not installed on this machine -- so test/js/dom/test_tools_sections.js is unverified. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014RRtqXDCnvnY6EQwhT5CV9 * fix(starlark): don't crash the pixlet editor's start/stop routes, and honor an operator-set PIXLET_EDITOR_HOST The AST-based port of #544 onto the api_v3 package split dropped `time` from starlark.py's import list. start_pixlet_editor() and stop_pixlet_editor() both call time.time()/time.sleep() directly, so every start (NameError building `state['started_at']`) and every stop that has to wait out the EXIT trap crashed with a 500. No test caught it because the route's own tests mock subprocess.Popen but never actually invoked it before now. Also carries over #544's later fix that this port branched before: env['PIXLET_EDITOR_HOST'] = '0.0.0.0' unconditionally overrode an operator who had already pinned PIXLET_EDITOR_HOST to loopback, forcing the unauthenticated `pixlet serve` process onto the LAN regardless (CodeQL CWE-1188). Switched to env.setdefault(...), same as api_v3.starlark.py's siblings already do for _pkg-owned names. Both fixes route the shared _pkg.time reference the rest of the package's route modules already use for anything a test might need to patch, rather than a bare `import time` local to this file. Ported the existing regression test from #544 (TestPixletEditorHostDefaultsButDoesNotOverride) onto this branch's module layout (web_interface.blueprints.api_v3.starlark instead of the old monolithic api_v3 module), which is what caught the NameError. Full suite: 4330 passed, 62 skipped, 2 failed -- identical on this branch and on origin/main (missing tzdata package breaks two timezone-alias tests in test_onboarding_checklist.py, unrelated to this change). Co-Authored-By: Claude Opus 5 * fix(api-v3): clear the six lint errors this rebase introduced All six were introduced by rebasing this branch onto the merged blueprint split, not by the split itself. Confirmed by diffing pyflakes output against main with line numbers normalised -- everything else it reports is present on main too and is the package's deliberate re-export pattern. starlark.py used _STARLARK_APPS_DIR three times without importing it (F821). The rebase resolved an import-list conflict as a union of both sides, and that symbol was on neither side of the conflict hunk, so it was silently lost. It is defined in __init__.py and is now imported like its neighbours. This was the only one of the six that would fail at runtime rather than merely lint. __init__.py imported contextlib twice (F811): the cherry-pick added one next to the existing import. Removed the duplicate; the original at line 19 is used. __init__.py imported signal purely to re-export it to starlark.py, so pyflakes saw it as unused (F401). signal is stdlib and does not need routing through the blueprint package, so starlark.py imports it directly and __init__.py no longer does. contextlib stays re-exported because this module genuinely uses it. _read_mqtt_bridge_config()'s local `config` shadowed the `config` submodule this module imports at the bottom for its route side effects (F811). Renamed to `settings`, with a comment saying why, since the name is otherwise the obvious one to reach for. Verified: pyflakes now reports nothing on this branch that main does not, the package imports, all nine route modules load, and 117 routes register, matching the pinned URL-map snapshot. Co-Authored-By: Claude Opus 5 * fix(api-v3): reject MQTT bridge bodies the endpoint cannot apply Two CodeRabbit findings on the bridge settings endpoint, both of which returned 200 while doing something other than what the caller asked. `request.get_json(silent=True) or {}` turned a missing or unparseable body -- and the JSON literals null, [] and false -- into an empty dict, which then satisfied the isinstance(data, dict) guard on the very next line. The guard was there to reject exactly those bodies. Dropping the `or {}` lets None fail it. The same `or {}` on /errors/clear is left alone: its docstring documents the body as optional, so an absent body legitimately means "use the defaults". The difference is that saving settings has nothing sensible to do with no body. `if data.get('clear_password'):` accepted any truthy value, and the string "false" is truthy in Python -- so a client echoing the field back as a string wiped a password it meant to keep. Now coerced through the package's existing _coerce_to_bool, which already maps 'true'/'on'/'1'/'yes' and nothing else. test_mqtt_bridge_config_endpoint.py covers both: five unusable body shapes plus a missing body, and clear_password across truthy and falsy spellings. Verified against the unfixed code -- reverting the body guard fails 5, reverting the coercion fails 3. Not changed here: CodeRabbit also asks this endpoint to reject MQTT credentials when TLS is off (CWE-319). That is a policy decision about the feature rather than a defect -- unencrypted MQTT on a trusted LAN is common and often deliberate -- so it is raised on the PR for a maintainer call instead. Co-Authored-By: Claude Opus 5 * fix: work through the remaining review findings on the editor and bridge allow_insecure_mqtt (CWE-319, requested): a password with TLS disabled crosses the network in cleartext. Refused now rather than merely warned about -- but refused, not forbidden, because unencrypted MQTT on a trusted LAN is a normal deliberate setup. allow_insecure_mqtt is the explicit acknowledgement, defaults false, and is coerced like the other booleans so the string "false" cannot switch the guard off. starlark.py:796 -- the supported service runs Flask threaded, so two start requests could each see running=False, each launch an editor, and the second state write replace the first PID, orphaning a process that holds the display down with nothing recording it. The check-launch-write sequence now takes a module-level lock. starlark.py:848 -- if the state write failed the route returned success with an editor running and no PID recorded: status and stop both reported no session while the display stayed down until the timeout expired. It now terminates the process group and returns an error. starlark.py:890 -- SIGKILL gives the script's EXIT trap no chance to run, so nothing hands the display back, yet the response said "the display is restarting". After an escalation the display is now restarted explicitly, and a failure to do so returns an error naming the manual step instead of a success. pixlet_config_editor.sh:184 -- find_pixlet supports Darwin but macOS ships no timeout(1); GNU coreutils installs it as gtimeout. Resolved up front so the failure lands before the display is stopped rather than after. pixlet_config_editor.sh:154 -- wildcard, loopback and an explicit interface address are three cases, not two. Collapsing the last two printed a URL saying "localhost" whenever PIXLET_EDITOR_HOST named a LAN address. tools.html:1254 -- escHtml does not encode single quotes, and the app id was interpolated into an inline onclick="startPixletEditor('...')", so a directory containing an apostrophe could break out of the JS string and run script. The handler binds with addEventListener and reads the id from dataset, where it is only ever parsed as an HTML attribute. Tests: test_mqtt_bridge_config_endpoint.py grows to 23 cases covering the opt-in in both directions. The tools DOM suite gains three guards asserting the edit buttons carry no inline onclick and pass the id via dataset -- those need jsdom and did not run here, so CI verifies them. Co-Authored-By: Claude Opus 5 * fix(api-v3): log the traceback on the editor state-write failure The 848 fix answers 500 when the session state cannot be written, and logged that at error level -- but without exc_info, so the traceback never reached the log. test_web_error_detail.py guards exactly this: a handler returning 5xx must write an error-level record *with* the traceback and return the sanitized detail, because checking that merely something was logged is too weak. Caught by Core unit tests on the previous commit, not locally: the guard parses every module under web_interface/blueprints/api_v3 as one source, so it only fires once the whole package is read together. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 (1M context) --- scripts/install/install_mqtt_bridge.sh | 0 scripts/utils/pixlet_config_editor.sh | 104 ++++- test/fixtures/api_v3_url_map.json | 51 +++ test/js/README.md | 1 + test/js/dom/test_tools_sections.js | 127 ++++++ test/js/run_all.js | 3 +- test/test_mqtt_bridge_config_endpoint.py | 138 +++++++ .../test_starlark_pixlet_routes.py | 63 +++ web_interface/blueprints/api_v3/__init__.py | 208 ++++++++++ web_interface/blueprints/api_v3/config.py | 6 +- web_interface/blueprints/api_v3/misc.py | 124 +++++- web_interface/blueprints/api_v3/plugins.py | 23 +- web_interface/blueprints/api_v3/starlark.py | 254 +++++++++++- .../templates/v3/partials/tools.html | 383 ++++++++++++++++++ 14 files changed, 1451 insertions(+), 34 deletions(-) mode change 100644 => 100755 scripts/install/install_mqtt_bridge.sh mode change 100644 => 100755 scripts/utils/pixlet_config_editor.sh create mode 100644 test/js/dom/test_tools_sections.js create mode 100644 test/test_mqtt_bridge_config_endpoint.py diff --git a/scripts/install/install_mqtt_bridge.sh b/scripts/install/install_mqtt_bridge.sh old mode 100644 new mode 100755 diff --git a/scripts/utils/pixlet_config_editor.sh b/scripts/utils/pixlet_config_editor.sh old mode 100644 new mode 100755 index 8437623b..124a9c6f --- a/scripts/utils/pixlet_config_editor.sh +++ b/scripts/utils/pixlet_config_editor.sh @@ -18,20 +18,33 @@ # ./scripts/utils/pixlet_config_editor.sh # list installed apps # ./scripts/utils/pixlet_config_editor.sh # edit # -# Binds loopback only, and there is deliberately no flag to change that: -# `pixlet serve` has no authentication, and anything that can reach it can -# rewrite the app's config. To edit from another machine, forward the port -- -# which authenticates as SSH and leaves nothing listening on the LAN: +# Binds the LAN by default, matching the web interface, which already serves +# 0.0.0.0:5000 with no authentication -- anything that can reach this can +# already reconfigure the display there. `pixlet serve` has no authentication +# either, so treat both the same way: fine on a home network, not on an open +# one. Override the bind and the session length with: +# +# PIXLET_EDITOR_HOST=127.0.0.1 ./scripts/utils/pixlet_config_editor.sh +# PIXLET_EDITOR_TIMEOUT=600 ./scripts/utils/pixlet_config_editor.sh +# +# For loopback-only editing from another machine, forward the port instead: # # ssh -L 8080:localhost:8080 pi@ledpi.local +# +# The session always ends by itself after PIXLET_EDITOR_TIMEOUT seconds +# (default 30 minutes). The display is stopped while editing, so a session +# left open would otherwise leave the panel dark indefinitely -- the timeout +# is what makes it safe to start one from the web interface. set -eu PROJECT_ROOT_DIR=$(cd "$(dirname "$0")/../.." && pwd) APPS_DIR="$PROJECT_ROOT_DIR/starlark-apps" PORT="${PIXLET_EDITOR_PORT:-8080}" -# Loopback only. See the header: pixlet serve is unauthenticated. -BIND_HOST="127.0.0.1" +# LAN by default; see the header for why, and how to force loopback. +BIND_HOST="${PIXLET_EDITOR_HOST:-0.0.0.0}" +# Hard stop, so the display cannot be left off by a forgotten session. +EDITOR_TIMEOUT="${PIXLET_EDITOR_TIMEOUT:-1800}" APP_ID="${1:-}" @@ -91,6 +104,29 @@ PIXLET_BIN=$(find_pixlet) || { exit 1 } +# find_pixlet supports Darwin, so this script has to as well. macOS ships no +# timeout(1); GNU coreutils installs it as gtimeout. Resolve whichever exists +# and fail here with instructions rather than at the invocation far below, +# where the failure would land after the display has already been stopped. +find_timeout() { + local candidate + for candidate in timeout gtimeout; do + if command -v "$candidate" >/dev/null 2>&1; then + command -v "$candidate" + return 0 + fi + done + return 1 +} + +TIMEOUT_BIN=$(find_timeout) || { + echo "Neither 'timeout' nor 'gtimeout' was found on PATH." + echo "This script needs one to bound the editing session." + echo "On macOS, install GNU coreutils:" + echo " brew install coreutils" + exit 1 +} + CONFIG_FILE="$APP_DIR/config.json" if [ -f "$CONFIG_FILE" ]; then cp "$CONFIG_FILE" "$CONFIG_FILE.backup" @@ -109,6 +145,19 @@ fi # failure worth guarding against. cleanup() { echo "" + # Kill the serve child explicitly. `timeout` is started with --foreground so + # it shares this script's process group (without that it makes its own, and + # a group signal aimed at this script would orphan pixlet with the port + # still bound). Belt and braces: signal the recorded pid too, because a + # group signal only reaches it while the group is shared. + if [ -n "${SERVE_PID:-}" ] && kill -0 "$SERVE_PID" 2>/dev/null; then + kill -TERM "$SERVE_PID" 2>/dev/null || true + for _ in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "$SERVE_PID" 2>/dev/null || break + sleep 0.3 + done + kill -KILL "$SERVE_PID" 2>/dev/null || true + fi if [ "$DISPLAY_WAS_RUNNING" = true ]; then echo "Restarting the display service..." sudo systemctl restart ledmatrix || echo "⚠ Could not restart ledmatrix - do it by hand" @@ -122,22 +171,53 @@ if [ "$DISPLAY_WAS_RUNNING" = true ]; then sudo systemctl stop ledmatrix fi +# Wildcard, loopback and an explicit interface address are three different +# cases. Collapsing the last two into "localhost" printed a URL pointing at the +# user's own machine whenever PIXLET_EDITOR_HOST named a LAN address. +case "$BIND_HOST" in + 0.0.0.0|::|"") REACH_HOST="$(hostname).local" ;; + 127.0.0.1|::1|localhost) REACH_HOST="localhost" ;; + *) REACH_HOST="$BIND_HOST" ;; +esac + echo "" echo "Editing: $APP_ID" echo "App file: $STAR_FILE" -echo "URL: http://localhost:$PORT/" +echo "URL: http://$REACH_HOST:$PORT/" echo "" -echo "Listening on localhost only -- pixlet serve has no authentication." -echo "From another machine, forward the port:" -echo " ssh -L $PORT:localhost:$PORT $(whoami)@$(hostname)" +if [ "$BIND_HOST" = "0.0.0.0" ]; then + echo "Reachable on the LAN, and pixlet serve has no authentication -- the" + echo "same footing as the web interface on port 5000. Set" + echo "PIXLET_EDITOR_HOST=127.0.0.1 to keep it to this machine." +else + echo "Listening on $BIND_HOST only. From another machine, forward the port:" + echo " ssh -L $PORT:localhost:$PORT $(whoami)@$(hostname)" +fi echo "" echo "Changes save straight to the real config as you make them." echo "Press Ctrl+C when finished - the display restarts automatically." +echo "This session stops on its own after ${EDITOR_TIMEOUT}s regardless." echo "" cd "$APP_DIR" -"$PIXLET_BIN" serve "$(basename "$STAR_FILE")" \ +# `timeout` owns the hard stop rather than the caller: the trap above restarts +# the display however this exits, so a session that outlives the person who +# started it still gives the panel back. Exit 124 is timeout's own code for +# "expired", which is a normal end here, not a failure. +# --foreground: stay in this script's process group so one signal reaches the +# whole session. Backgrounded + `wait` so the EXIT trap can run while the child +# is still alive; a foreground child would leave bash waiting on it instead. +"$TIMEOUT_BIN" --foreground "$EDITOR_TIMEOUT" "$PIXLET_BIN" serve "$(basename "$STAR_FILE")" \ --host "$BIND_HOST" \ --port "$PORT" \ --no-browser \ - --saveconfig "$CONFIG_FILE" + --saveconfig "$CONFIG_FILE" & +SERVE_PID=$! + +status=0 +wait "$SERVE_PID" || status=$? +if [ "$status" -eq 124 ]; then + echo "Session reached its ${EDITOR_TIMEOUT}s limit." + status=0 +fi +exit "$status" diff --git a/test/fixtures/api_v3_url_map.json b/test/fixtures/api_v3_url_map.json index f2557fd9..f66784e7 100644 --- a/test/fixtures/api_v3_url_map.json +++ b/test/fixtures/api_v3_url_map.json @@ -315,6 +315,23 @@ "OPTIONS" ] ], + [ + "/api/v3/integrations/mqtt-bridge", + "api_v3.get_mqtt_bridge", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/integrations/mqtt-bridge/config", + "api_v3.update_mqtt_bridge_config", + [ + "OPTIONS", + "PUT" + ] + ], [ "/api/v3/logs", "api_v3.get_logs", @@ -747,6 +764,40 @@ "POST" ] ], + [ + "/api/v3/starlark/editor/apps", + "api_v3.list_pixlet_editor_apps", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/editor/start", + "api_v3.start_pixlet_editor", + [ + "OPTIONS", + "POST" + ] + ], + [ + "/api/v3/starlark/editor/status", + "api_v3.get_pixlet_editor_status", + [ + "GET", + "HEAD", + "OPTIONS" + ] + ], + [ + "/api/v3/starlark/editor/stop", + "api_v3.stop_pixlet_editor", + [ + "OPTIONS", + "POST" + ] + ], [ "/api/v3/starlark/install-pixlet", "api_v3.install_pixlet", diff --git a/test/js/README.md b/test/js/README.md index dd91132c..023e0d9a 100644 --- a/test/js/README.md +++ b/test/js/README.md @@ -39,6 +39,7 @@ nothing is listening, so it stays useful in a bare checkout. | `dom/test_installed_dom.js` | yes | The toolbar in a real DOM: pill/search/sort interaction, the HTMX partial re-swap, and a `getComputedStyle` check that `.filter-pill[data-active]` really matches the emitted markup | | `dom/test_store_dom.js` | yes | Store pagination, per-page, category, tri-state Installed button, and persistence across a re-boot, against the live registry | | `dom/test_no_double_fetch.js` | yes | Loads the **whole** `plugins_manager.js` and counts requests: typing in the store search must filter the cached list, not refetch `/api/v3/plugins/store/list` | +| `dom/test_tools_sections.js` | yes | The Tools tab's MQTT bridge and Pixlet editor sections: form prefill, the write-only password (blank means unchanged), the running-session banner and countdown, and that the editor link points at the host you loaded the page from | Point the DOM suites at a rig with a full plugin set when it matters — a dev box with two plugins installed will pass while exercising very little. diff --git a/test/js/dom/test_tools_sections.js b/test/js/dom/test_tools_sections.js new file mode 100644 index 00000000..4625f25d --- /dev/null +++ b/test/js/dom/test_tools_sections.js @@ -0,0 +1,127 @@ +// Real-DOM (jsdom) test of the two new Tools sections. The HTML is the actual +// server-rendered /partials/tools, and the payloads are the real API's, so a +// renamed field or a changed shape fails this rather than passing quietly. +const http = require('http'); +const { JSDOM, VirtualConsole } = require('jsdom'); + +const BASE = process.env.BASE || 'http://localhost:5000'; +const get = p => new Promise((res, rej) => + http.get(BASE + p, r => { let d = ''; r.on('data', c => d += c); r.on('end', () => res(d)); }).on('error', rej)); + +(async () => { + const partial = await get('/partials/tools'); + const bridge = JSON.parse(await get('/api/v3/integrations/mqtt-bridge')); + const apps = JSON.parse(await get('/api/v3/starlark/editor/apps')); + + const errs = []; + const vc = new VirtualConsole(); + vc.on('jsdomError', e => errs.push(String(e.message || e).split('\n')[0])); + + // Controllable fetch: serve the real payloads, and let tests swap in others. + let editorStatus = { status: 'success', data: { running: false } }; + let bridgePayload = bridge; + let onPut = null; + const stubFetch = (url, opts) => { + const u = String(url); + if (onPut && opts && opts.method === 'PUT') onPut(JSON.parse(opts.body)); + let body = { status: 'success', data: {} }; + if (u.includes('/integrations/mqtt-bridge')) body = bridgePayload; + else if (u.includes('/starlark/editor/status')) body = editorStatus; + else if (u.includes('/starlark/editor/apps')) body = apps; + return Promise.resolve({ ok: true, status: 200, json: () => Promise.resolve(body) }); + }; + + // runScripts:'dangerously' so the partial's own From f6367d63ae877b0109cb8f8f3b84f92af982d635 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:32:03 -0400 Subject: [PATCH 15/16] =?UTF-8?q?security:=20triage=20the=20CodeQL=20backl?= =?UTF-8?q?og=20=E2=80=94=20129=20alerts,=20three=20of=20them=20live=20(#5?= =?UTF-8?q?61)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(web): escape quotes in every HTML escaper, not just & < > The escapers are all `div.textContent = x; return div.innerHTML`. That round-trip escapes &, < and > -- the only characters the HTML serializer must escape in a text node -- and leaves quotes alone. Every widget then interpolates the result into a quoted attribute value: value="${escapeHtml(v)}" title="${escapeHtml(v)}" so a value of `x" onmouseover="alert(1)` closes the attribute and adds an event handler of its own. CodeQL reported this 83 times (js/incomplete-html-attribute-sanitization) across the widget files. It is one bug, not 83: the widgets each carry a standalone fallback that did escape quotes, but they all prefer BaseWidget.escapeHtml when window.BaseWidget exists -- which it always does in the shipped page -- so the correct fallbacks were dead code and the incomplete shared one ran. Fixed at each source instead of at the call sites. app-shell.js already documented this exact gap in a comment and worked around it by building DOM nodes by hand; that workaround stays (setting a property cannot be got wrong), the comment is now accurate. cache.html's delete button interpolated the cache key into `onclick="deleteCacheFile('...')"`. Escaping cannot help there -- the browser HTML-decodes the attribute before parsing it as JS, so `'` becomes a real `'` again -- so the key moves to a data-cache-key attribute that the handler reads back. url-input.js additionally wrote a value straight into an after validating it against a schema-supplied protocol list, and that list accepted any RFC 3986 scheme -- "javascript" included. Scriptable schemes (javascript, data, vbscript, blob, filesystem) are now refused both when the list is normalised and when a URL is checked against it, and the render path routes its href through the same check instead of emitting whatever was stored (js/xss-through-dom). test/js/unit/test_html_escaping.js reads each escaper out of the shipped file and runs it, so losing the quote handling again fails a test rather than a scan. Co-Authored-By: Claude Opus 5 * fix(security): stop request-supplied names from reaching paths outside their base Three of the py/path-injection alerts were live, not lint: * GET /api/v3/plugins//static/ read any file whose resolved path *string-prefixed* the plugin directory. Flask's default converter forbids a slash but not dots, and get_plugin_directory('..') returned the parent of the plugins directory because it exists -- so every file under the project root then prefixed that directory, config/config_secrets.json included. The prefix check was also wrong on its own terms: with plugin dir "plugin-repos/foo", "../foo-evil/x" resolves to "plugin-repos/foo-evil/x", whose string does start with "plugin-repos/foo". * POST /api/v3/plugins/of-the-day/json/delete interpolated the request body's file_id into f"{file_id}.json" and unlinked it, unvalidated. A file_id of "../../../../etc/something" deleted that file. This is the one finding in the batch that destroyed data rather than exposing it. * POST /api/v3/cache/delete passed the body's key through CacheManager.clear_cache to DiskCache, which joined it as a filename and called os.remove. Same shape, same result. The guard goes in DiskCache.get_cache_path, the single choke point get/set/clear share, so every caller is covered rather than just this route. Real keys are the stems of files already flat in the cache directory -- that is how list_cache_files derives them -- so nothing legitimate is turned away. The rest of the cluster (web_interface/app.py's asset route, the plugin update handler, _get_plugin_version, the plugin-schema read in config.py) was guarded in ways that held, but each had grown its own version of the check. They now go through one helper, src/common/path_safety.py, which returns the *sanitised value* rather than a verdict -- so a caller cannot validate one string and open another, which is how the two real bugs above were shaped. Also: WiFiManager.connect_to_network took the SSID and password straight from POST /api/v3/wifi/connect into nmcli's argv. There is no shell there, so CodeQL's py/command-line-injection alert overstates the risk -- but nmcli reads a leading "-" as an option, so an SSID of "--ask" asks nmcli to run differently rather than to join a network. Both values are now checked for shape (802.11's 32-octet SSID limit, WPA's 8-63 char passphrase or 64-char hex key, no control characters, no leading dash) before any subprocess runs. test/test_path_traversal_guards.py asserts on the filesystem, not just the status code: a handler that returns 403 and deletes the file anyway would pass the weaker check. Twelve of its cases fail against the unpatched code. Co-Authored-By: Claude Opus 5 * fix(web): refuse a plugin id that is not a plain name, don't truncate it pages_v3 and scripts/dev_server.py ran request ids through os.path.basename and carried on with what came out, so "../weather" rendered the config form for "weather". Nothing escaped the plugins directory -- the relative_to guards held -- but the handler answered a request nobody made, and validating one string while the filesystem sees another is the shape both live traversals earlier in this branch had. Same treatment as the rest: safe_path_component rejects rather than truncates, resolve_under returns the path it checked, and the call sites use what those return. The three handlers that had hand-rolled resolve-and-relative_to blocks lose about twenty lines to the shared one. Co-Authored-By: Claude Opus 5 * docs(web): say what the plugin web_ui iframe actually is The docstring claimed the fragment runs "in a sandboxed iframe". The iframe in plugin_config.html carries no sandbox attribute, so the fragment runs with the interface's own origin. That is fine -- the file belongs to an installed plugin, and an installed plugin already runs Python on the device, so the trust boundary is install rather than this route -- but a comment promising containment that is not there is worse than no comment. This is the context for the py/reflective-xss alert on this handler. Also drops the now-unused os/os.path imports. Co-Authored-By: Claude Opus 5 * fix(web): inline url-input's scheme guard at the previewLink.href sink CodeQL flagged this line as a new high-severity js/xss-through-dom alert on this PR even though it is already covered by SCRIPTABLE_SCHEMES: the guard reached the sink through safeHref -> isValidUrl, two function calls away, which its DOM-based-XSS sanitizer recognition does not trace. Behavior is unchanged -- same scheme check, same SCRIPTABLE_SCHEMES list, same allowedProtocols gate -- just inlined directly above the previewLink.href assignment it guards, so the barrier is visible in the same scope as the sink. Added a regression test that runs the shipped onInput handler (not just the extracted helpers) against a mocked DOM, so a future change that reintroduces an unguarded previewLink.href assignment fails here. Co-Authored-By: Claude Opus 5 * fix(security): address CodeRabbit findings on the CodeQL triage PR - src/wifi_manager.py: reject non-ASCII WPA-PSK passphrases before any credential-saving or connect flow runs. NetworkManager only accepts printable ASCII passphrases (or a 64-char hex key); a non-ASCII value was previously saved/attempted before nmcli itself rejected it. - web_interface/blueprints/api_v3/config.py: fail closed when the plugin config schema path can't be resolved under the plugins directory (e.g. a symlinked plugin dir). Previously this fell through with secret_fields left empty, so submitted credentials for that plugin were saved as ordinary, unencrypted configuration. - web_interface/static/v3/js/widgets/plugin-file-manager.js: stop splicing the JSON day/column key into an inline oninput="..." handler string. escHtml() escapes quotes for a normal HTML attribute, but the browser HTML-decodes the attribute before running it as script, which undoes that escaping and lets a crafted column name (e.g. from an uploaded JSON file) break out of the JS string and execute. Cell edits now travel through data-day/data-col attributes read by one delegated 'input' listener instead. While in this file: fixed 6 pre-existing missing-')' typos on multi-line safeSetHTML(...) calls (already flagged by Biome in this PR's own CodeRabbit run as syntax errors blocking its lint pass). These predate this PR (present on main too) but made the whole file fail to parse in any JS engine, which is a bigger problem than the XSS finding itself and directly touches the same lines. Added/extended regression tests for each fix; full suites pass (pytest: 4580 passed, 62 skipped; JS: 84 assertions). Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- scripts/dev_server.py | 13 +- src/cache/disk_cache.py | 24 +- src/common/path_safety.py | 125 +++++++ src/wifi_manager.py | 80 ++++- test/js/run_all.js | 3 +- test/js/unit/test_html_escaping.js | 320 ++++++++++++++++++ test/test_pages_v3_path_guards.py | 123 +++++++ test/test_path_traversal_guards.py | 317 +++++++++++++++++ test/test_web_api.py | 25 ++ test/test_wifi_connect_argument_safety.py | 154 +++++++++ web_interface/app.py | 91 ++--- web_interface/blueprints/api_v3/__init__.py | 13 +- web_interface/blueprints/api_v3/config.py | 15 +- web_interface/blueprints/api_v3/misc.py | 7 + web_interface/blueprints/api_v3/plugins.py | 97 ++++-- web_interface/blueprints/pages_v3.py | 96 +++--- web_interface/static/v3/js/app-shell.js | 13 +- .../static/v3/js/utils/error_handler.js | 7 +- .../static/v3/js/widgets/base-widget.js | 16 +- .../v3/js/widgets/example-color-picker.js | 8 +- .../static/v3/js/widgets/file-upload.js | 8 +- .../static/v3/js/widgets/json-file-manager.js | 7 +- .../v3/js/widgets/plugin-file-manager.js | 43 ++- .../static/v3/js/widgets/url-input.js | 47 ++- web_interface/static/v3/plugins_manager.js | 21 +- .../templates/v3/partials/cache.html | 9 +- web_interface/templates/v3/partials/logs.html | 8 +- 27 files changed, 1520 insertions(+), 170 deletions(-) create mode 100644 src/common/path_safety.py create mode 100644 test/js/unit/test_html_escaping.js create mode 100644 test/test_pages_v3_path_guards.py create mode 100644 test/test_path_traversal_guards.py create mode 100644 test/test_wifi_connect_argument_safety.py diff --git a/scripts/dev_server.py b/scripts/dev_server.py index 0f493f55..7a7dd945 100644 --- a/scripts/dev_server.py +++ b/scripts/dev_server.py @@ -32,6 +32,8 @@ os.environ['EMULATOR'] = 'true' from flask import Flask, render_template, request, jsonify +from src.common.path_safety import resolve_under, safe_path_component + app = Flask(__name__, template_folder=str(Path(__file__).parent / 'templates')) logger = logging.getLogger(__name__) @@ -118,7 +120,8 @@ def find_plugin_dir(plugin_id: str) -> Optional[Path]: one of the plugin search dirs, so a crafted id can never name a path outside them. """ - if not isinstance(plugin_id, str) or not _SAFE_PLUGIN_ID_RE.match(plugin_id): + plugin_id = safe_path_component(plugin_id) + if not plugin_id or not _SAFE_PLUGIN_ID_RE.match(plugin_id): return None from src.plugin_system.plugin_loader import PluginLoader loader = PluginLoader() @@ -140,8 +143,8 @@ def find_plugin_dir(plugin_id: str) -> Optional[Path]: def load_config_defaults(plugin_dir: 'str | Path') -> Dict[str, Any]: """Extract default values from config_schema.json.""" - schema_path = Path(plugin_dir) / 'config_schema.json' - if not schema_path.exists(): + schema_path = resolve_under(plugin_dir, 'config_schema.json') + if schema_path is None or not schema_path.exists(): return {} with open(schema_path, 'r') as f: schema = json.load(f) @@ -175,8 +178,8 @@ def api_plugin_schema(plugin_id): if not plugin_dir: return jsonify({'error': f'Plugin not found: {plugin_id}'}), 404 - schema_path = plugin_dir / 'config_schema.json' - if not schema_path.exists(): + schema_path = resolve_under(plugin_dir, 'config_schema.json') + if schema_path is None or not schema_path.exists(): return jsonify({'schema': {'type': 'object', 'properties': {}}}) with open(schema_path, 'r') as f: diff --git a/src/cache/disk_cache.py b/src/cache/disk_cache.py index 92c3f465..c64c8632 100644 --- a/src/cache/disk_cache.py +++ b/src/cache/disk_cache.py @@ -15,6 +15,8 @@ import zlib from typing import Dict, Any, Optional, Protocol from datetime import datetime +from src.common.path_safety import safe_path_component + try: # optional: large speedup on the cache write path, see _dumps below import orjson except ImportError: # pragma: no cover - exercised on hosts without the wheel @@ -160,16 +162,30 @@ class DiskCache: def get_cache_path(self, key: str) -> Optional[str]: """ Get the path for a cache file. - + + The key becomes a filename, so it has to be one. Keys reach this + method from the web API -- POST /api/v3/cache/delete passes the + request body's ``key`` straight through CacheManager.clear_cache to + os.remove -- and a key of ``../../../../etc/whatever`` named a file + well outside the cache directory. Every real key is the stem of a + file already sitting flat in cache_dir (that is how list_cache_files + derives them), so rejecting anything with a path component turns + away only inputs that could never have been written here. + Args: key: Cache key - + Returns: - Path to cache file or None if cache is disabled + Path to cache file, or None if cache is disabled or the key is + not a usable filename """ if not self.cache_dir: return None - return os.path.join(self.cache_dir, f"{key}.json") + safe_key = safe_path_component(key) + if safe_key is None: + self.logger.warning("Rejected unsafe cache key %r", key) + return None + return os.path.join(self.cache_dir, f"{safe_key}.json") def get(self, key: str, max_age: Optional[int] = 300) -> Optional[Dict[str, Any]]: """ diff --git a/src/common/path_safety.py b/src/common/path_safety.py new file mode 100644 index 00000000..16bad09a --- /dev/null +++ b/src/common/path_safety.py @@ -0,0 +1,125 @@ +"""One place to turn a request-supplied name into a path you can open. + +Every web handler that opens a file under a fixed directory had grown its own +version of this: a regex here, an ``os.path.basename`` there, a +``str(x).startswith(str(base))`` somewhere else. They were not equivalent. +``startswith`` says ``plugin-repos/foo-evil`` is inside ``plugin-repos/foo``; +validating a name in one place and rebuilding the path from the *raw* value in +another leaves the guard checking something the filesystem never sees. + +Two functions, used the same way everywhere: + +``safe_path_component(value)`` + ``value`` if it is one harmless path segment, otherwise ``None``. + +``resolve_under(base, *parts)`` + the resolved path, or ``None`` if any part is unsafe or the result would + land outside ``base``. + +Both *return the sanitised value* rather than a boolean, so a caller cannot +validate one string and then open another -- and so a scanner can follow what +actually reaches ``open()``. ``os.path.basename`` does the stripping because it +is the sanitiser CodeQL's path-injection query recognises; the equality check +after it means an input with a directory part is rejected outright instead of +being silently truncated to something the caller did not ask for. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, List, Optional, Union + +__all__ = [ + 'safe_path_component', + 'safe_relative_parts', + 'resolve_under', +] + +# Names that are a path component syntactically but never name a real entry a +# caller means to reach. +_RESERVED_COMPONENTS = frozenset({'', '.', '..'}) + + +def safe_path_component(value: Any) -> Optional[str]: + """Return ``value`` when it is a single, harmless path segment. + + Returns ``None`` for anything else: a non-string, an empty string, ``.`` or + ``..``, a value carrying a directory separator (either platform's), a drive + letter, or an embedded NUL. + + The return value is what callers must join -- not the argument. + """ + if not isinstance(value, str) or not value: + return None + if '\x00' in value: + return None + + # basename strips any directory component, so what a caller joins cannot + # carry one. Comparing the result against the input rejects rather than + # truncates: "../etc/passwd" is an error, not a request for "passwd". + name = os.path.basename(value) + if name != value or name in _RESERVED_COMPONENTS: + return None + + # basename only knows the host platform's separator. On POSIX a backslash + # is an ordinary character, and "C:" is a plausible-looking name that + # os.path.join would treat as a drive on Windows. Rule both out everywhere + # so behaviour does not depend on where the service happens to run. + if '/' in name or '\\' in name or os.sep in name or (os.altsep and os.altsep in name): + return None + if ':' in name and len(name) >= 2 and name[1] == ':': + return None + + return name + + +def safe_relative_parts(value: Any) -> Optional[List[str]]: + """Split a multi-segment relative path into safe components. + + For Flask's ```` converter, where ``a/b/c.json`` is legitimate but + ``../../config/config_secrets.json`` is not. Returns the component list, or + ``None`` if any component fails :func:`safe_path_component`. + """ + if not isinstance(value, str) or not value: + return None + if value.startswith('/') or value.startswith('\\'): + return None + + parts: List[str] = [] + for raw in value.replace('\\', '/').split('/'): + if raw == '': + # A trailing or doubled slash names nothing; skip it rather than + # rejecting a path a browser may well send. + continue + part = safe_path_component(raw) + if part is None: + return None + parts.append(part) + + return parts or None + + +def resolve_under(base: Union[str, Path], *parts: Any) -> Optional[Path]: + """Resolve ``base/parts...``, or ``None`` if that would escape ``base``. + + Each part is validated with :func:`safe_path_component` first, so the value + that reaches the filesystem is the sanitised one. The containment check is + kept as well: it is what catches a symlink inside ``base`` pointing out of + it, which no amount of name validation can see. + """ + safe_parts: List[str] = [] + for part in parts: + component = safe_path_component(part) + if component is None: + return None + safe_parts.append(component) + + try: + base_resolved = Path(base).resolve() + candidate = base_resolved.joinpath(*safe_parts).resolve() + candidate.relative_to(base_resolved) + except (OSError, ValueError, TypeError): + return None + + return candidate diff --git a/src/wifi_manager.py b/src/wifi_manager.py index 808767ff..008cbaab 100644 --- a/src/wifi_manager.py +++ b/src/wifi_manager.py @@ -36,7 +36,7 @@ import os import time import re from pathlib import Path -from typing import Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from dataclasses import dataclass logger = logging.getLogger(__name__) @@ -1248,14 +1248,28 @@ class WiFiManager: def connect_to_network(self, ssid: str, password: str) -> Tuple[bool, str]: """ Connect to a WiFi network with failsafe to restore original connection on failure. - + Args: ssid: Network SSID password: Network password (empty for open networks) - + Returns: Tuple of (success, message) """ + # Both values arrive verbatim from POST /api/v3/wifi/connect and end up + # as nmcli argv entries. There is no shell here, so no metacharacter + # can start a second command -- but nmcli reads a leading "-" as an + # option, so an SSID of "--ask" or "-t" is a request to run nmcli + # differently rather than to join a network. See _validate_ssid. + ssid, error = self._validate_ssid(ssid) + if error: + logger.warning("Rejected WiFi connect request: %s", error) + return False, error + password, error = self._validate_wifi_password(password) + if error: + logger.warning("Rejected WiFi connect request: %s", error) + return False, error + # Save current connection info for failsafe restoration original_connection = None original_ssid = None @@ -1635,6 +1649,66 @@ class WiFiManager: self._show_led_message("Connection error", duration=5) return False, str(e) + # 802.11 caps an SSID at 32 octets. Control characters cannot appear in a + # real one, and a leading "-" would be read by nmcli as an option rather + # than a network name. + _SSID_MAX_OCTETS = 32 + # WPA-PSK passphrases are 8-63 printable ASCII characters, or a 64-char hex + # key. Anything outside that cannot authenticate, so refusing it early + # costs nothing and keeps argv clean. + _PSK_MIN_LEN = 8 + _PSK_MAX_LEN = 63 + + @classmethod + def _validate_ssid(cls, ssid: Any) -> Tuple[str, Optional[str]]: + """Return (ssid, None) for a usable SSID, or ('', reason) to refuse it. + + Returns the value rather than a boolean so callers pass on what was + checked instead of re-reading the original. + """ + if not isinstance(ssid, str): + return '', "SSID must be text" + ssid = ssid.strip() + if not ssid: + return '', "SSID cannot be empty" + if len(ssid.encode('utf-8')) > cls._SSID_MAX_OCTETS: + return '', f"SSID is longer than {cls._SSID_MAX_OCTETS} bytes" + if any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in ssid): + return '', "SSID contains control characters" + if ssid.startswith('-'): + # nmcli would take this for an option, not a network name. + return '', "SSID cannot start with '-'" + return ssid, None + + @classmethod + def _validate_wifi_password(cls, password: Any) -> Tuple[str, Optional[str]]: + """Return (password, None) for a usable passphrase, or ('', reason). + + An empty password means an open network and is allowed through. + """ + if password is None: + return '', None + if not isinstance(password, str): + return '', "Password must be text" + if password == '': + return '', None + if any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in password): + return '', "Password contains control characters" + if not password.isascii(): + # WPA-PSK passphrases are printable ASCII only; NetworkManager + # rejects anything else. + return '', "Password must be ASCII" + if password.startswith('-'): + # Same reason as the SSID: nmcli would read it as an option. + return '', "Password cannot start with '-'" + is_hex_key = len(password) == 64 and all(c in '0123456789abcdefABCDEF' for c in password) + if not is_hex_key and not (cls._PSK_MIN_LEN <= len(password) <= cls._PSK_MAX_LEN): + return '', ( + f"Password must be {cls._PSK_MIN_LEN}-{cls._PSK_MAX_LEN} characters " + f"(or a 64-character hex key)" + ) + return password, None + @staticmethod def _is_wrong_password_error(error_msg: str) -> bool: """Return True when nmcli's error output indicates an authentication failure.""" diff --git a/test/js/run_all.js b/test/js/run_all.js index 05f96b93..b10a9827 100755 --- a/test/js/run_all.js +++ b/test/js/run_all.js @@ -14,7 +14,8 @@ const path = require('path'); const fs = require('fs'); const BASE = process.env.BASE || 'http://localhost:5000'; -const UNIT = ['unit/test_list_filter.js', 'unit/test_render_cards.js']; +const UNIT = ['unit/test_list_filter.js', 'unit/test_render_cards.js', + 'unit/test_html_escaping.js']; const DOM = ['dom/test_installed_dom.js', 'dom/test_store_dom.js', 'dom/test_no_double_fetch.js', 'dom/test_tools_sections.js']; diff --git a/test/js/unit/test_html_escaping.js b/test/js/unit/test_html_escaping.js new file mode 100644 index 00000000..7b18a8fc --- /dev/null +++ b/test/js/unit/test_html_escaping.js @@ -0,0 +1,320 @@ +// Every HTML escaper in the web interface must escape quotes, not just +// & < >. +// +// The escapers are all written as `div.textContent = x; return div.innerHTML`. +// That round-trip escapes &, < and > because those are the only characters the +// HTML serializer has to escape in a *text node* -- quotes are left alone. But +// the widgets interpolate the result into quoted attribute values +// (`value="${escapeHtml(v)}"`, `title="${escapeHtml(v)}"`, ...), and there a +// bare `"` closes the attribute and lets the value add attributes of its own: +// +// name" onfocus="alert(1) +// +// CodeQL reported 83 js/incomplete-html-attribute-sanitization alerts for +// exactly this. This suite pins the fix at the source: it reads each real +// implementation out of the shipped file and runs it, so an escaper that loses +// its quote handling again fails here rather than in a scanner run weeks later. + +const fs = require('fs'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '../../../web_interface'); + +let pass = 0, fail = 0; +const ok = (label, cond, extra) => cond + ? (pass++, console.log(' ok ' + label)) + : (fail++, console.log(' FAIL ' + label + (extra !== undefined ? ' -> ' + JSON.stringify(extra).slice(0, 300) : ''))); + +// ── DOM shim ─────────────────────────────────────────────────────────────── +// Mirrors what a browser does reading innerHTML back off textContent: & < > +// are escaped, quotes are not. Anything the escaper adds on top of that is +// the escaper's own doing, which is what we are testing. +class FakeEl { + set textContent(v) { this._t = String(v == null ? '' : v); } + get textContent() { return this._t || ''; } + get innerHTML() { + return (this._t || '') + .replace(/&/g, '&') + .replace(//g, '>'); + } +} +global.document = { createElement: () => new FakeEl() }; +global.window = global; + +// ── source extraction ────────────────────────────────────────────────────── +// Pull a function out of a real source file by its opening line and balanced +// braces, so the test runs the shipped code rather than a copy of it. +function extract(file, opener) { + const src = fs.readFileSync(path.join(ROOT, file), 'utf8'); + const start = src.indexOf(opener); + if (start < 0) { + console.error(`FAIL: cannot find ${JSON.stringify(opener)} in ${file}`); + process.exit(1); + } + let i = src.indexOf('{', start), depth = 0; + for (let j = i; j < src.length; j++) { + if (src[j] === '{') depth++; + else if (src[j] === '}') { + depth--; + if (depth === 0) return src.slice(start, j + 1); + } + } + console.error(`FAIL: unbalanced braces after ${JSON.stringify(opener)} in ${file}`); + process.exit(1); +} + +// Evaluate an extracted escaper and return it as a callable. +function loadFn(file, opener, name, { method = false } = {}) { + const body = extract(file, opener); + // Class/object methods (`escapeHtml(text) {...}`) are not valid statements on + // their own -- wrap them in an object literal so they can be evaluated. + const code = method + ? `(function(){ const o = { ${body} }; return o.${name}.bind(o); })()` + : `(function(){ ${body}; return ${name}; })()`; + // eslint-disable-next-line no-eval + return eval(code); +} + +// ── the escapers, as shipped ─────────────────────────────────────────────── +const ESCAPERS = [ + ['base-widget.js (BaseWidget.escapeHtml)', + 'static/v3/js/widgets/base-widget.js', 'escapeHtml(text) {', 'escapeHtml', true], + ['plugins_manager.js (top-level escapeHtml)', + 'static/v3/plugins_manager.js', 'function escapeHtml(text) {', 'escapeHtml', false], + ['plugins_manager.js (starlark escapeHtml)', + 'static/v3/plugins_manager.js', 'function escapeHtml(str) {', 'escapeHtml', false], + ['error_handler.js (escapeHtml)', + 'static/v3/js/utils/error_handler.js', 'function escapeHtml(text) {', 'escapeHtml', false], + ['json-file-manager.js (_esc)', + 'static/v3/js/widgets/json-file-manager.js', '_esc(str) {', '_esc', true], + ['plugin-file-manager.js (escHtml)', + 'static/v3/js/widgets/plugin-file-manager.js', 'function escHtml(s) {', 'escHtml', false], + ['app-shell.js (escapeHtml)', + 'static/v3/js/app-shell.js', 'escapeHtml(text) {', 'escapeHtml', true], + ['logs.html (escapeHtml)', + 'templates/v3/partials/logs.html', 'function escapeHtml(text) {', 'escapeHtml', false], + ['cache.html (escapeHtml)', + 'templates/v3/partials/cache.html', 'function escapeHtml(text) {', 'escapeHtml', false], +]; + +// The breakout payload: closes a double-quoted attribute and opens an event +// handler. If `"` survives escaping, this is live script in the rendered page. +const BREAKOUT = 'x" onmouseover="alert(1)'; + +console.log('\n1. every escaper neutralises a double-quote attribute breakout'); +for (const [label, file, opener, name, method] of ESCAPERS) { + const fn = loadFn(file, opener, name, { method }); + const out = String(fn(BREAKOUT)); + ok(`${label}: no raw "`, !out.includes('"'), out); + ok(`${label}: emits "`, out.includes('"'), out); +} + +console.log('\n2. every escaper also escapes single quotes'); +for (const [label, file, opener, name, method] of ESCAPERS) { + const fn = loadFn(file, opener, name, { method }); + const out = String(fn("x' onmouseover='alert(1)")); + ok(`${label}: no raw '`, !out.includes("'"), out); +} + +console.log('\n3. the & < > behaviour they already had is unchanged'); +for (const [label, file, opener, name, method] of ESCAPERS) { + const fn = loadFn(file, opener, name, { method }); + const out = String(fn(' & done')); + ok(`${label}: no raw <`, !out.includes('<'), out); + ok(`${label}: no raw >`, !out.includes('>'), out); + ok(`${label}: & becomes &`, /&/.test(out), out); +} + +console.log('\n4. ampersands are escaped before quotes, so " cannot be forged'); +// If `&` were escaped last, the input `"` would come out as a literal +// `"` after the browser decodes the attribute. Order matters; pin it. +for (const [label, file, opener, name, method] of ESCAPERS) { + const fn = loadFn(file, opener, name, { method }); + const out = String(fn('"')); + ok(`${label}: " input stays inert`, out === '&quot;', out); +} + +// ── url-input scheme handling (js/xss-through-dom) ───────────────────────── +console.log('\n5. url-input never treats a scriptable scheme as a valid URL'); +{ + const src = fs.readFileSync(path.join(ROOT, 'static/v3/js/widgets/url-input.js'), 'utf8'); + const a = src.indexOf('const RFC_SCHEME_PATTERN'); + const b = src.indexOf("window.LEDMatrixWidgets.register('url-input'"); + if (a < 0 || b < 0) { console.error('FAIL: cannot locate url-input helpers'); process.exit(1); } + // eslint-disable-next-line no-eval + const helpers = eval(`(function(){ ${src.slice(a, b)}; return { normalizeProtocols, isValidUrl, safeHref }; })()`); + const { normalizeProtocols, isValidUrl, safeHref } = helpers; + + ok('javascript: rejected under the default protocols', + !isValidUrl('javascript:alert(1)', ['http', 'https'])); + ok('javascript: still rejected when a schema asks for it', + !isValidUrl('javascript:alert(1)', normalizeProtocols(['javascript']))); + ok('a schema asking only for javascript falls back to http/https', + JSON.stringify(normalizeProtocols(['javascript'])) === JSON.stringify(['http', 'https']), + normalizeProtocols(['javascript'])); + ok('data: rejected', !isValidUrl('data:text/html,', ['http', 'https', 'data'])); + ok('vbscript: rejected', !isValidUrl('vbscript:msgbox(1)', ['http', 'https', 'vbscript'])); + ok('normalizeProtocols drops scriptable schemes but keeps the rest', + JSON.stringify(normalizeProtocols('https,javascript,ftp')) === JSON.stringify(['https', 'ftp']), + normalizeProtocols('https,javascript,ftp')); + + ok('ordinary https URL still valid', isValidUrl('https://example.com/x?y=1', ['http', 'https'])); + ok('ordinary http URL still valid', isValidUrl('http://example.com', ['http', 'https'])); + ok('a scheme outside the allow-list is still rejected', + !isValidUrl('ftp://example.com', ['http', 'https'])); + + ok('safeHref passes a good URL through', safeHref('https://example.com', ['http', 'https']) === 'https://example.com'); + ok('safeHref blanks a javascript: URL', safeHref('javascript:alert(1)', ['http', 'https']) === ''); + ok('safeHref blanks an unparseable value', safeHref('not a url', ['http', 'https']) === ''); +} + +// ── url-input onInput: the sink itself, not just the pulled-out helpers ──── +// The scheme check now lives inline in onInput, right where the value reaches +// `previewLink.href`, instead of behind safeHref/isValidUrl -- see the comment +// at that assignment in url-input.js for why. Run the handler as shipped so a +// regression that reintroduces an unguarded `previewLink.href = value` fails +// here, not just in a scanner run weeks later. +console.log('\n6. url-input onInput: previewLink.href is guarded at the sink'); +{ + class FakeClassList { + constructor() { this.classes = new Set(['hidden']); } + add(c) { this.classes.add(c); } + remove(c) { this.classes.delete(c); } + contains(c) { return this.classes.has(c); } + } + const mockDoc = (value, protocolsAttr) => { + const elements = { + _input: { value, checkValidity: () => true, validationMessage: '', classList: new FakeClassList() }, + _preview: { classList: new FakeClassList() }, + _preview_link: { + classList: new FakeClassList(), + _href: undefined, + set href(v) { this._href = v; }, + get href() { return this._href; }, + removeAttribute(name) { if (name === 'href') this._href = undefined; }, + }, + _widget: { dataset: { protocols: protocolsAttr } }, + _error: { classList: new FakeClassList(), textContent: '' }, + }; + return { + elements, + getElementById: (id) => elements[Object.keys(elements).find(k => id === `field${k}`)] || null, + }; + }; + + function runOnInput(value, protocolsAttr) { + const { elements, getElementById } = mockDoc(value, protocolsAttr); + const savedDocument = global.document; + const savedWindow = global.window; + let registered = null; + global.document = { createElement: () => new FakeEl(), getElementById }; + global.window = { + LEDMatrixWidgets: { + register: (name, obj) => { registered = obj; }, + get: () => registered, + getHandlers: () => registered.handlers, + }, + }; + try { + const src = fs.readFileSync(path.join(ROOT, 'static/v3/js/widgets/url-input.js'), 'utf8'); + // eslint-disable-next-line no-eval + eval(src); + registered.handlers.onInput('field'); + } finally { + global.document = savedDocument; + global.window = savedWindow; + } + return elements; + } + + let els = runOnInput('javascript:alert(1)', 'http,https'); + ok('javascript: never reaches previewLink.href', els._preview_link.href === undefined, els._preview_link.href); + ok('javascript: leaves the preview hidden', els._preview.classList.contains('hidden')); + + els = runOnInput('https://example.com', 'http,https'); + ok('an ordinary https URL reaches previewLink.href', els._preview_link.href === 'https://example.com', els._preview_link.href); + ok('an ordinary https URL unhides the preview', !els._preview.classList.contains('hidden')); + + els = runOnInput('data:text/html,', 'http,https,data'); + ok('data: never reaches previewLink.href even when the schema allows it', + els._preview_link.href === undefined, els._preview_link.href); + + els = runOnInput('not a url', 'http,https'); + ok('an unparseable value never reaches previewLink.href', els._preview_link.href === undefined, els._preview_link.href); +} + +// ── plugin-file-manager: cell edits travel via data-*, not inline handlers ── +// A JSON key/day from an uploaded file used to be spliced, HTML-escaped, +// into an oninput="...('${escHtml(col)}'...)" attribute. escHtml neutralises +// a quote for an ordinary attribute, but here the value also has to survive +// as a *JS string literal* -- the browser HTML-decodes the attribute before +// running it as script, which turns the escaped quote back into a real one +// and lets a column named `x');alert(1);//` break out of the string and run +// arbitrary JS. Cell edits now reach _pfmCellEdit only via data-day/data-col +// read by one delegated listener, so this pins that no inline handler string +// is built from the value at all. +console.log("\n7. plugin-file-manager: cell edits never go through an inline handler string"); +{ + const src = fs.readFileSync(path.join(ROOT, 'static/v3/js/widgets/plugin-file-manager.js'), 'utf8'); + + function extractFn(opener) { + const start = src.indexOf(opener); + if (start < 0) { console.error(`FAIL: cannot find ${JSON.stringify(opener)}`); process.exit(1); } + let i = src.indexOf('{', start), depth = 0; + for (let j = i; j < src.length; j++) { + if (src[j] === '{') depth++; + else if (src[j] === '}') { depth--; if (depth === 0) return src.slice(start, j + 1); } + } + console.error(`FAIL: unbalanced braces after ${JSON.stringify(opener)}`); + process.exit(1); + } + + const escHtmlFn = loadFn('static/v3/js/widgets/plugin-file-manager.js', 'function escHtml(s) {', 'escHtml', false); + const renderEntryTableSrc = extractFn('function renderEntryTable(fieldId, container, content) {'); + + const calls = []; + const fakeWindow = { _pfmCellEdit: (fieldId, day, col, value) => calls.push({ fieldId, day, col, value }) }; + + class FakeContainer { + constructor() { this._html = ''; this._listeners = {}; } + set innerHTML(v) { this._html = v; } + get innerHTML() { return this._html; } + set textContent(v) { this._html = v; } + addEventListener(type, fn) { this._listeners[type] = fn; } + dispatch(type, target) { this._listeners[type]({ target }); } + } + + function fakeCell(day, col, value) { + return { + closest: (sel) => (sel.includes('data-day') ? { dataset: { day: String(day), col: String(col) }, value } : null), + }; + } + + // eslint-disable-next-line no-eval + const renderEntryTable = eval(`(function(getState, escHtml, safeSetHTML, window){ + ${renderEntryTableSrc} + return renderEntryTable; + })`)( + () => ({ entriesPerPage: 20, _tablePage: 1 }), + escHtmlFn, + (target, html) => { target.innerHTML = html; }, + fakeWindow + ); + + const maliciousCol = "x');alert(1);//"; + const container = new FakeContainer(); + renderEntryTable('field1', container, { '1': { [maliciousCol]: 'hello' } }); + + ok('no inline oninput handler is emitted for a cell', !/oninput=/.test(container.innerHTML), container.innerHTML); + ok('the malicious column name never appears unescaped in the markup', + !container.innerHTML.includes(maliciousCol), container.innerHTML); + + container.dispatch('input', fakeCell('1', maliciousCol, 'typed value')); + ok('the delegated listener still reaches _pfmCellEdit with the real day/col', + calls.length === 1 && calls[0].day === '1' && calls[0].col === maliciousCol && calls[0].value === 'typed value', + calls); +} + +console.log(`\n${pass} passed, ${fail} failed\n`); +process.exit(fail ? 1 : 0); diff --git a/test/test_pages_v3_path_guards.py b/test/test_pages_v3_path_guards.py new file mode 100644 index 00000000..2ec1c3e2 --- /dev/null +++ b/test/test_pages_v3_path_guards.py @@ -0,0 +1,123 @@ +"""pages_v3 must refuse an id that is not a plain name, not truncate it. + +Both partial loaders used to run the id through ``os.path.basename`` and carry +on with what came out, so "../weather" rendered the config form for "weather". +Nothing escaped the plugins directory -- the containment guards held -- but the +handler answered a request nobody made, and validating one string while the +filesystem sees another is the shape both live traversals in this branch had. + +These tests are about that: a rejected id gets a 400, and a real one still +renders. +""" + +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +BACKSLASH = chr(92) + + +@pytest.fixture +def pages(tmp_path): + """pages_v3 with a plugins directory holding one real plugin.""" + from web_interface.blueprints import pages_v3 as module + + plugins_dir = tmp_path / "plugin-repos" + (plugins_dir / "weather" / "web_ui").mkdir(parents=True) + (plugins_dir / "weather" / "config_schema.json").write_text( + '{"type": "object", "properties": {"enabled": {"type": "boolean"}}}', + encoding="utf-8", + ) + (plugins_dir / "weather" / "manifest.json").write_text( + '{"name": "Weather", "version": "1.0.0"}', encoding="utf-8" + ) + (plugins_dir / "weather" / "web_ui" / "panel.html").write_text( + "

panel

", encoding="utf-8" + ) + + original_pm = getattr(module.pages_v3, "plugin_manager", None) + original_cm = getattr(module.pages_v3, "config_manager", None) + + plugin_manager = MagicMock() + plugin_manager.plugins_dir = plugins_dir + plugin_manager.get_plugin_info.return_value = {"name": "Weather", "version": "1.0.0"} + plugin_manager.get_plugin.return_value = None + module.pages_v3.plugin_manager = plugin_manager + module.pages_v3.config_manager = MagicMock(load_config=lambda: {}) + + yield module, plugins_dir + + module.pages_v3.plugin_manager = original_pm + module.pages_v3.config_manager = original_cm + + +@pytest.mark.parametrize("plugin_id", [ + "../weather", "..", ".", "", None, "a/b", "x" + BACKSLASH + "y", +]) +def test_a_plugin_id_that_is_not_a_plain_name_is_a_400(pages, plugin_id): + module, _ = pages + body, status = module._load_plugin_config_partial(plugin_id) + assert status == 400 + assert "Invalid plugin ID" in body + + +def test_a_traversing_id_no_longer_renders_the_truncated_one(pages): + """"../weather" used to render "weather"'s form. It must not.""" + module, _ = pages + body, status = module._load_plugin_config_partial("../weather") + assert status == 400 + assert "Weather" not in body + + +@pytest.mark.parametrize("app_id", ["../demo", "..", "a/b", "", None]) +def test_a_starlark_app_id_that_is_not_a_plain_name_is_a_400(pages, app_id): + module, _ = pages + body, status = module._load_starlark_config_partial(app_id) + assert status == 400 + assert "Invalid app ID" in body + + +class TestServePluginWebUi: + """GET /plugin-ui//web-ui/""" + + @pytest.fixture + def client(self, pages): + from flask import Flask + + module, _ = pages + base = Path(module.__file__).resolve().parent.parent + app = Flask( + __name__, + template_folder=str(base / "templates"), + static_folder=str(base / "static"), + ) + app.config["TESTING"] = True + app.register_blueprint(module.pages_v3, url_prefix="") + return app.test_client() + + def test_a_real_fragment_is_still_wrapped_and_served(self, client): + response = client.get("/plugin-ui/weather/web-ui/panel.html") + assert response.status_code == 200 + body = response.get_data(as_text=True) + assert "

panel

" in body + assert 'window.PLUGIN_ID = "weather"' in body + + @pytest.mark.parametrize("url", [ + "/plugin-ui/../web-ui/panel.html", + "/plugin-ui/%2e%2e/web-ui/panel.html", + "/plugin-ui/weather/web-ui/../../manifest.json", + "/plugin-ui/weather/web-ui/..%2f..%2fmanifest.json", + ]) + def test_traversal_attempts_are_refused(self, client, url): + response = client.get(url) + assert response.status_code in (400, 403, 404) + assert "version" not in response.get_data(as_text=True) + + def test_a_non_html_filename_is_still_refused(self, client): + # The allowlist predates this change and must survive it. + response = client.get("/plugin-ui/weather/web-ui/manifest.json") + assert response.status_code == 400 diff --git a/test/test_path_traversal_guards.py b/test/test_path_traversal_guards.py new file mode 100644 index 00000000..9aaa04e4 --- /dev/null +++ b/test/test_path_traversal_guards.py @@ -0,0 +1,317 @@ +"""Request-supplied names must not be able to name a file outside their base. + +Three of these were live. Each test that stands in for one says so, and +asserts on the *filesystem* -- that the file outside the base is still there, +or was never read -- rather than only on the status code, because a handler +that returns 403 and deletes the file anyway would pass the weaker check. + +The rest pin the shared helper in src/common/path_safety.py that the handlers +now go through, so a future handler gets the same behaviour by using it. +""" + +import json +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.common.path_safety import ( # noqa: E402 + resolve_under, + safe_path_component, + safe_relative_parts, +) +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402 + +BACKSLASH = chr(92) + + +class TestSafePathComponent: + """One segment, or nothing.""" + + @pytest.mark.parametrize("value", [ + "plugin", "ledmatrix-of-the-day", "weather_data", "manifest.json", + "a.b.c", "UPPER", "with-dash", "with_underscore", "9lives", + ]) + def test_ordinary_names_pass_through_unchanged(self, value): + assert safe_path_component(value) == value + + @pytest.mark.parametrize("value", [ + "..", ".", "", "../etc", "a/b", "/abs", "etc/passwd", + "a" + BACKSLASH + "b", "C:" + BACKSLASH + "x", "C:", + "a\x00b", None, 123, b"bytes", ["list"], + ]) + def test_anything_that_could_name_another_directory_is_refused(self, value): + assert safe_path_component(value) is None + + def test_it_returns_the_value_rather_than_a_verdict(self): + # A boolean lets a caller validate one string and then open a + # different one. Returning the checked value is what stops that. + assert safe_path_component("ok") == "ok" + assert safe_path_component("../ok") is None + + +class TestSafeRelativeParts: + def test_a_multi_segment_path_splits(self): + assert safe_relative_parts("web_ui/index.html") == ["web_ui", "index.html"] + + def test_empty_segments_are_dropped_not_rejected(self): + assert safe_relative_parts("a//b") == ["a", "b"] + assert safe_relative_parts("x/") == ["x"] + + @pytest.mark.parametrize("value", [ + "../x", "a/../../etc", "/etc/passwd", BACKSLASH + "etc", + "a" + BACKSLASH + ".." + BACKSLASH + "b", "", None, + ]) + def test_traversal_and_absolute_paths_are_refused(self, value): + assert safe_relative_parts(value) is None + + +class TestResolveUnder: + def test_a_path_inside_the_base_resolves(self, tmp_path): + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "f.txt").write_text("x", encoding="utf-8") + resolved = resolve_under(tmp_path, "sub", "f.txt") + assert resolved == (tmp_path / "sub" / "f.txt").resolve() + + def test_a_path_that_would_escape_is_refused(self, tmp_path): + assert resolve_under(tmp_path, "..") is None + assert resolve_under(tmp_path, "..", "etc") is None + assert resolve_under(tmp_path, "/etc/passwd") is None + + def test_a_sibling_sharing_a_prefix_is_refused(self, tmp_path): + # The bug the old `str(x).startswith(str(base))` checks had: + # "/foo-evil" starts with "/foo" as a string, but is not + # inside it as a directory. + (tmp_path / "foo").mkdir() + (tmp_path / "foo-evil").mkdir() + (tmp_path / "foo-evil" / "secret").write_text("s", encoding="utf-8") + assert resolve_under(tmp_path / "foo", "..", "foo-evil", "secret") is None + # And the string check it replaces would have said yes: + naive = (tmp_path / "foo" / ".." / "foo-evil" / "secret").resolve() + assert str(naive).startswith(str((tmp_path / "foo").resolve())) + + def test_it_returns_none_rather_than_raising_on_junk(self, tmp_path): + assert resolve_under(tmp_path, None) is None + assert resolve_under(tmp_path, 42) is None + + +@pytest.fixture +def plugins_tree(tmp_path): + """A plugins directory with one plugin, plus a secret outside it.""" + plugins = tmp_path / "plugin-repos" + (plugins / "demo" / "web_ui").mkdir(parents=True) + (plugins / "demo" / "web_ui" / "index.html").write_text("

hi

", encoding="utf-8") + (plugins / "demo-evil").mkdir() + (plugins / "demo-evil" / "stolen.json").write_text('{"a":1}', encoding="utf-8") + secrets = tmp_path / "config" + secrets.mkdir() + (secrets / "config_secrets.json").write_text('{"api_key":"hunter2"}', encoding="utf-8") + return tmp_path + + +class TestServePluginStatic: + """GET /api/v3/plugins//static/ + + This handler read any file whose resolved path *string-prefixed* the + plugin directory. Two ways through: + + * plugin_id of "..", because Flask's default converter forbids a slash + but not dots, and get_plugin_directory returned the parent of the + plugins directory since it exists. Every file under the project then + prefixed that directory, config/config_secrets.json included. + * a sibling directory sharing a prefix, per the helper test above. + """ + + def _wire(self, api_v3_module, plugins_tree): + plugins = plugins_tree / "plugin-repos" + + def get_plugin_directory(plugin_id): + candidate = plugins / plugin_id + return str(candidate) if candidate.exists() else None + + api_v3_module.api_v3.plugin_manager = MagicMock() + api_v3_module.api_v3.plugin_manager.get_plugin_directory = get_plugin_directory + return plugins + + def test_a_real_plugin_file_is_still_served( + self, api_v3_client, api_v3_module, plugins_tree + ): + self._wire(api_v3_module, plugins_tree) + response = api_v3_client.get("/api/v3/plugins/demo/static/web_ui/index.html") + assert response.status_code == 200 + assert b"

hi

" in response.data + + def test_a_dotdot_plugin_id_cannot_reach_the_secrets_file( + self, api_v3_client, api_v3_module, plugins_tree + ): + self._wire(api_v3_module, plugins_tree) + response = api_v3_client.get( + "/api/v3/plugins/../static/config/config_secrets.json" + ) + assert response.status_code in (400, 403, 404) + assert b"hunter2" not in response.data + + def test_a_percent_encoded_dotdot_plugin_id_is_refused_too( + self, api_v3_client, api_v3_module, plugins_tree + ): + self._wire(api_v3_module, plugins_tree) + response = api_v3_client.get( + "/api/v3/plugins/%2e%2e/static/config/config_secrets.json" + ) + assert response.status_code in (400, 403, 404) + assert b"hunter2" not in response.data + + def test_a_sibling_directory_sharing_a_prefix_is_refused( + self, api_v3_client, api_v3_module, plugins_tree + ): + self._wire(api_v3_module, plugins_tree) + response = api_v3_client.get( + "/api/v3/plugins/demo/static/../demo-evil/stolen.json" + ) + assert response.status_code in (400, 403, 404) + assert b'"a"' not in response.data + + def test_an_unknown_plugin_is_still_a_404( + self, api_v3_client, api_v3_module, plugins_tree + ): + self._wire(api_v3_module, plugins_tree) + response = api_v3_client.get("/api/v3/plugins/nope/static/x.json") + assert response.status_code == 404 + + +class TestDeleteOfTheDayJson: + """POST /api/v3/plugins/of-the-day/json/delete + + file_id came from the request body and was interpolated into + ``f"{file_id}.json"`` and then unlinked, with no validation at all. A + file_id of "../../../../etc/something" deleted that file. This is the one + finding in the batch that destroyed data rather than exposing it. + """ + + @pytest.fixture + def plugin_tree(self, tmp_path, api_v3_module): + plugin_dir = tmp_path / "plugin-repos" / "ledmatrix-of-the-day" + (plugin_dir / "of_the_day").mkdir(parents=True) + (plugin_dir / "of_the_day" / "quotes.json").write_text("{}", encoding="utf-8") + outside = tmp_path / "victim.json" + outside.write_text("important", encoding="utf-8") + + api_v3_module.api_v3.plugin_manager = MagicMock() + api_v3_module.api_v3.plugin_manager.get_plugin_directory.return_value = str(plugin_dir) + return plugin_dir, outside + + URL = "/api/v3/plugins/of-the-day/json/delete" + + def test_a_real_file_in_the_plugin_is_still_deleted( + self, api_v3_client, plugin_tree + ): + plugin_dir, _ = plugin_tree + target = plugin_dir / "of_the_day" / "quotes.json" + response = api_v3_client.post(self.URL, json={"file_id": "quotes"}) + assert response.status_code == 200 + assert not target.exists() + + def test_a_traversing_file_id_deletes_nothing(self, api_v3_client, plugin_tree): + _, outside = plugin_tree + response = api_v3_client.post( + self.URL, json={"file_id": "../../../victim"} + ) + assert response.status_code == 400 + assert outside.exists(), "file outside the plugin directory was deleted" + assert outside.read_text(encoding="utf-8") == "important" + + @pytest.mark.parametrize("file_id", ["..", "a/b", "/etc/x", "x" + BACKSLASH + "y"]) + def test_other_shapes_of_traversal_are_refused( + self, api_v3_client, plugin_tree, file_id + ): + _, outside = plugin_tree + response = api_v3_client.post(self.URL, json={"file_id": file_id}) + assert response.status_code == 400 + assert outside.exists() + + +class TestDiskCacheKeys: + """The cache key becomes a filename, and POST /api/v3/cache/delete passes + the request body's key straight through CacheManager.clear_cache to + os.remove. A key of "../../../../etc/whatever" named a file well outside + the cache directory. + """ + + @pytest.fixture + def cache(self, tmp_path): + from src.cache.disk_cache import DiskCache + + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + return DiskCache(str(cache_dir)), cache_dir + + def test_an_ordinary_key_still_round_trips(self, cache): + disk_cache, cache_dir = cache + disk_cache.set("espn_nfl_2024", {"ok": True}) + assert (cache_dir / "espn_nfl_2024.json").exists() + assert disk_cache.get("espn_nfl_2024", max_age=None)["ok"] is True + + def test_a_traversing_key_has_no_path(self, cache): + disk_cache, _ = cache + assert disk_cache.get_cache_path("../../victim") is None + assert disk_cache.get_cache_path("..") is None + assert disk_cache.get_cache_path("a/b") is None + + def test_clearing_a_traversing_key_deletes_nothing(self, cache, tmp_path): + disk_cache, _ = cache + victim = tmp_path / "victim.json" + victim.write_text("important", encoding="utf-8") + disk_cache.clear("../victim") + assert victim.exists(), "file outside the cache directory was deleted" + + def test_writing_a_traversing_key_creates_nothing(self, cache, tmp_path): + disk_cache, _ = cache + disk_cache.set("../escaped", {"x": 1}) + assert not (tmp_path / "escaped.json").exists() + + def test_the_delete_endpoint_says_no_rather_than_claiming_success( + self, api_v3_client, api_v3_module + ): + api_v3_module.api_v3.cache_manager = MagicMock() + response = api_v3_client.post( + "/api/v3/cache/delete", json={"key": "../../etc/passwd"} + ) + assert response.status_code == 400 + api_v3_module.api_v3.cache_manager.clear_cache.assert_not_called() + + def test_the_delete_endpoint_still_deletes_an_ordinary_key( + self, api_v3_client, api_v3_module + ): + api_v3_module.api_v3.cache_manager = MagicMock() + response = api_v3_client.post( + "/api/v3/cache/delete", json={"key": "espn_nfl_2024"} + ) + assert response.status_code == 200 + api_v3_module.api_v3.cache_manager.clear_cache.assert_called_once_with( + "espn_nfl_2024" + ) + + +class TestPluginVersionLookup: + """_get_plugin_version joins a request-supplied id onto the plugins + directory and opens manifest.json under it.""" + + def test_a_real_manifest_is_read(self, tmp_path): + from web_interface.blueprints import api_v3 as module + + plugins = tmp_path / "plugin-repos" + (plugins / "demo").mkdir(parents=True) + (plugins / "demo" / "manifest.json").write_text( + json.dumps({"version": "1.2.3"}), encoding="utf-8" + ) + original = getattr(module.api_v3, "plugin_store_manager", None) + module.api_v3.plugin_store_manager = MagicMock(plugins_dir=str(plugins)) + try: + assert module._get_plugin_version("demo") == "1.2.3" + assert module._get_plugin_version("../demo") == "" + assert module._get_plugin_version("..") == "" + finally: + module.api_v3.plugin_store_manager = original diff --git a/test/test_web_api.py b/test/test_web_api.py index 679bfd84..3ed16caf 100644 --- a/test/test_web_api.py +++ b/test/test_web_api.py @@ -127,6 +127,31 @@ class TestConfigAPI: assert response.status_code == 200 mock_config_manager.save_config_atomic.assert_called_once() + def test_save_main_config_fails_closed_when_schema_path_is_unresolvable( + self, client, mock_config_manager, mock_plugin_manager + ): + """A plugin id whose schema path fails safe-resolution must not have + its config saved with secret_fields left empty. + + Regression test for the CodeQL/CodeRabbit finding on + web_interface/blueprints/api_v3/config.py: previously, when + resolve_under() returned None (e.g. a plugin directory reached via a + symlink), the code fell through to `secret_fields = set()` and saved + the plugin's submitted config -- credentials included -- as + ordinary, unencrypted configuration instead of refusing the request. + """ + mock_plugin_manager.plugin_manifests = {'evil': {}} + + with patch('web_interface.blueprints.api_v3.config.resolve_under', return_value=None): + response = client.post( + '/api/v3/config/main', + data=json.dumps({'evil': {'api_key': 'super-secret'}}), + content_type='application/json' + ) + + assert response.status_code == 400 + mock_config_manager.save_config_atomic.assert_not_called() + def test_save_main_config_validation_error(self, client, mock_config_manager): """Test saving config with validation error.""" invalid_config = {'invalid': 'data'} diff --git a/test/test_wifi_connect_argument_safety.py b/test/test_wifi_connect_argument_safety.py new file mode 100644 index 00000000..bd17afff --- /dev/null +++ b/test/test_wifi_connect_argument_safety.py @@ -0,0 +1,154 @@ +"""The SSID and password reaching nmcli's argv come from an HTTP request body. + +POST /api/v3/wifi/connect takes both verbatim and WiFiManager.connect_to_network +hands them to:: + + subprocess.run(["nmcli", "device", "wifi", "connect", ssid, "password", password]) + +There is no shell in that, so no metacharacter can start a second command -- +CodeQL's py/command-line-injection alert overstates it on that point. What is +real is argument injection: nmcli reads a leading "-" as an option, so an SSID +of "--ask" or "-t" asks nmcli to *run differently* rather than to join a +network. Neither value was checked for shape at all before reaching argv. + +These tests pin the validation without touching real networking: the +validators are classmethods, so nothing here constructs a WiFiManager. +""" + +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.wifi_manager import WiFiManager # noqa: E402 +from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402 + + +class TestSsidValidation: + @pytest.mark.parametrize("ssid", [ + "HomeNet", "my wifi 5G", "Cafe-Guest", "café", "x" * 32, "-not-leading".lstrip("-"), + ]) + def test_ordinary_ssids_pass_through(self, ssid): + value, error = WiFiManager._validate_ssid(ssid) + assert error is None + assert value == ssid + + def test_surrounding_whitespace_is_trimmed_not_rejected(self): + value, error = WiFiManager._validate_ssid(" HomeNet ") + assert error is None + assert value == "HomeNet" + + @pytest.mark.parametrize("ssid", ["--ask", "-t", "-"]) + def test_an_ssid_nmcli_would_read_as_an_option_is_refused(self, ssid): + value, error = WiFiManager._validate_ssid(ssid) + assert error is not None + assert value == "" + + def test_an_ssid_over_32_octets_is_refused(self): + # 802.11 caps the SSID element at 32 octets, so a longer one could + # never name a real network. + _, error = WiFiManager._validate_ssid("x" * 33) + assert error is not None + # Multi-byte characters count as octets, not characters. + _, error = WiFiManager._validate_ssid("é" * 17) + assert error is not None + + @pytest.mark.parametrize("ssid", ["a\nb", "a\rb", "a\x00b", "a\x7fb", "a\tb"]) + def test_control_characters_are_refused(self, ssid): + _, error = WiFiManager._validate_ssid(ssid) + assert error is not None + + @pytest.mark.parametrize("ssid", ["", " ", None, 42, ["HomeNet"]]) + def test_empty_and_non_text_values_are_refused(self, ssid): + _, error = WiFiManager._validate_ssid(ssid) + assert error is not None + + +class TestPasswordValidation: + def test_an_empty_password_means_an_open_network(self): + value, error = WiFiManager._validate_wifi_password("") + assert error is None + assert value == "" + value, error = WiFiManager._validate_wifi_password(None) + assert error is None + assert value == "" + + @pytest.mark.parametrize("password", ["hunter22", "a" * 63, "0" * 64, "AbCdEf0123" * 6 + "abcd"]) + def test_valid_psk_lengths_pass_through(self, password): + value, error = WiFiManager._validate_wifi_password(password) + assert error is None, f"{password!r} rejected: {error}" + assert value == password + + @pytest.mark.parametrize("password", ["short", "z" * 64, "a" * 200]) + def test_lengths_that_could_never_authenticate_are_refused(self, password): + # 8-63 chars for a passphrase, or exactly 64 hex chars for a raw key. + # "z" * 64 is the right length for a key but is not hex, so it is + # neither -- note "a" * 64 *is* valid hex and must stay accepted. + _, error = WiFiManager._validate_wifi_password(password) + assert error is not None + + @pytest.mark.parametrize("password", ["-password", "--ask"]) + def test_a_password_nmcli_would_read_as_an_option_is_refused(self, password): + _, error = WiFiManager._validate_wifi_password(password) + assert error is not None + + def test_control_characters_are_refused(self): + _, error = WiFiManager._validate_wifi_password("pass\nword") + assert error is not None + + @pytest.mark.parametrize("password", ["pässword", "你好12345678"]) + def test_non_ascii_passphrases_are_refused(self, password): + # NetworkManager accepts only printable ASCII WPA-PSK passphrases (or + # a 64-char hex key); a non-ASCII value would otherwise reach + # _connect_nmcli and be saved/attempted before nmcli itself rejects it. + _, error = WiFiManager._validate_wifi_password(password) + assert error is not None + + +class TestConnectRefusesBeforeRunningNmcli: + """connect_to_network must not reach subprocess with a rejected value.""" + + @pytest.mark.parametrize("ssid,password", [ + ("--ask", "hunter22"), + ("x" * 40, "hunter22"), + ("Home\nNet", "hunter22"), + ("HomeNet", "-secret1"), + ("HomeNet", "short"), + ]) + def test_no_subprocess_runs_for_a_rejected_request(self, ssid, password): + manager = WiFiManager.__new__(WiFiManager) # no __init__: no real host access + with patch("src.wifi_manager.subprocess.run") as run: + ok, message = manager.connect_to_network(ssid, password) + assert ok is False + assert message + run.assert_not_called() + + +class TestConnectEndpointSurfacesTheRefusal: + URL = "/api/v3/wifi/connect" + + @pytest.fixture + def wifi_manager(self): + with patch("src.wifi_manager.WiFiManager") as cls: + instance = MagicMock() + cls.return_value = instance + yield instance + + def test_a_rejected_ssid_comes_back_as_a_client_error( + self, api_v3_client, wifi_manager + ): + # The route delegates the shape check to the manager, so mirror what + # the real one now returns rather than asserting on a mock's default. + wifi_manager.connect_to_network.return_value = (False, "SSID cannot start with '-'") + response = api_v3_client.post(self.URL, json={"ssid": "--ask", "password": "hunter22"}) + assert response.status_code == 400 + assert "-" in response.get_json()["message"] + + def test_an_ordinary_request_is_unaffected(self, api_v3_client, wifi_manager): + wifi_manager.connect_to_network.return_value = (True, "Connected to HomeNet") + response = api_v3_client.post(self.URL, json={"ssid": "HomeNet", "password": "hunter22"}) + assert response.status_code == 200 + wifi_manager.connect_to_network.assert_called_once_with("HomeNet", "hunter22") diff --git a/web_interface/app.py b/web_interface/app.py index 13529a1d..e2f3c71a 100644 --- a/web_interface/app.py +++ b/web_interface/app.py @@ -17,6 +17,9 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from src.config_manager import ConfigManager from src.web_interface.error_handler import describe_exception +from src.common.path_safety import ( + resolve_under, safe_path_component, safe_relative_parts, +) from werkzeug.exceptions import HTTPException from src.exceptions import ConfigError from src.plugin_system.plugin_manager import PluginManager @@ -206,66 +209,66 @@ app.register_blueprint(api_v3, url_prefix='/api/v3') # Route to serve plugin asset files (registered on main app, not blueprint, for /assets/... path) @app.route('/assets/plugins//uploads/', methods=['GET']) def serve_plugin_asset(plugin_id, filename): - """Serve uploaded asset files from assets/plugins/{plugin_id}/uploads/""" + """Serve uploaded asset files from assets/plugins/{plugin_id}/uploads/ + + Both URL parts are validated before any path is built. The containment + check here used to compare the *asset directory* against the project root + rather than against assets/plugins, so a plugin_id of ``..`` moved the + served directory a level up and still passed. + """ try: - # Build the asset directory path - assets_dir = project_root / 'assets' / 'plugins' / plugin_id / 'uploads' - assets_dir = assets_dir.resolve() - + uploads_base = (project_root / 'assets' / 'plugins').resolve() + + safe_plugin_id = safe_path_component(plugin_id) + if not safe_plugin_id: + return jsonify({'status': 'error', 'message': 'Invalid asset path'}), 403 + + safe_parts = safe_relative_parts(filename) + if not safe_parts: + return jsonify({'status': 'error', 'message': 'Invalid file path'}), 403 + + assets_dir = resolve_under(uploads_base, safe_plugin_id, 'uploads') + if assets_dir is None: + return jsonify({'status': 'error', 'message': 'Invalid asset path'}), 403 + # Security check: ensure the assets directory exists and is within project_root if not assets_dir.exists() or not assets_dir.is_dir(): return jsonify({'status': 'error', 'message': 'Asset directory not found'}), 404 - - # Ensure we're serving from within the assets directory (prevent directory traversal) - # Use proper path resolution instead of string prefix matching to prevent bypasses - assets_dir_resolved = assets_dir.resolve() - project_root_resolved = project_root.resolve() - - # Check that assets_dir is actually within project_root using commonpath - try: - common_path = os.path.commonpath([str(assets_dir_resolved), str(project_root_resolved)]) - if common_path != str(project_root_resolved): - return jsonify({'status': 'error', 'message': 'Invalid asset path'}), 403 - except ValueError: - # commonpath raises ValueError if paths are on different drives (Windows) - return jsonify({'status': 'error', 'message': 'Invalid asset path'}), 403 - - # Resolve the requested file path - requested_file = (assets_dir / filename).resolve() - - # Security check: ensure file is within the assets directory using proper path comparison - # Use commonpath to ensure assets_dir is a true parent of requested_file - try: - common_path = os.path.commonpath([str(requested_file), str(assets_dir_resolved)]) - if common_path != str(assets_dir_resolved): - return jsonify({'status': 'error', 'message': 'Invalid file path'}), 403 - except ValueError: - # commonpath raises ValueError if paths are on different drives (Windows) + + # Resolve the requested file path. resolve_under repeats the + # containment check after resolving, which is what catches a symlink + # inside the uploads directory pointing out of it. + requested_file = resolve_under(assets_dir, *safe_parts) + if requested_file is None: return jsonify({'status': 'error', 'message': 'Invalid file path'}), 403 - + # Check if file exists if not requested_file.exists() or not requested_file.is_file(): return jsonify({'status': 'error', 'message': 'File not found'}), 404 - + # Determine content type based on file extension + lowered = requested_file.name.lower() content_type = 'application/octet-stream' - if filename.lower().endswith(('.png', '.jpg', '.jpeg')): - content_type = 'image/jpeg' if filename.lower().endswith(('.jpg', '.jpeg')) else 'image/png' - elif filename.lower().endswith('.gif'): + if lowered.endswith(('.png', '.jpg', '.jpeg')): + content_type = 'image/jpeg' if lowered.endswith(('.jpg', '.jpeg')) else 'image/png' + elif lowered.endswith('.gif'): content_type = 'image/gif' - elif filename.lower().endswith('.bmp'): + elif lowered.endswith('.bmp'): content_type = 'image/bmp' - elif filename.lower().endswith('.webp'): + elif lowered.endswith('.webp'): content_type = 'image/webp' - elif filename.lower().endswith('.svg'): + elif lowered.endswith('.svg'): content_type = 'image/svg+xml' - elif filename.lower().endswith('.json'): + elif lowered.endswith('.json'): content_type = 'application/json' - elif filename.lower().endswith('.txt'): + elif lowered.endswith('.txt'): content_type = 'text/plain' - - # Use send_from_directory to serve the file - return send_from_directory(str(assets_dir), filename, mimetype=content_type) + + # Use send_from_directory to serve the file. The path handed over is + # the validated one, rebuilt from the components that were checked. + return send_from_directory( + str(assets_dir), '/'.join(safe_parts), mimetype=content_type + ) except Exception: app.logger.exception('Error serving plugin asset file') diff --git a/web_interface/blueprints/api_v3/__init__.py b/web_interface/blueprints/api_v3/__init__.py index bff6cf16..80f8d5a1 100644 --- a/web_interface/blueprints/api_v3/__init__.py +++ b/web_interface/blueprints/api_v3/__init__.py @@ -48,6 +48,7 @@ from src.web_interface.validators import ( ) from src.error_aggregator import get_error_aggregator from src.common.permission_utils import install_requirements_file +from src.common.path_safety import resolve_under _SUDO = shutil.which('sudo') _JOURNALCTL = shutil.which('journalctl') _GIT = shutil.which('git') @@ -124,9 +125,17 @@ def _get_plugin_version(plugin_id: str) -> str: """Read the installed version from a plugin's manifest.json. Returns the version string on success, or '' if the manifest - cannot be read (missing, corrupt, permission denied, etc.). + cannot be read (missing, corrupt, permission denied, etc.) or if + ``plugin_id`` is not a plain directory name. Several callers pass an id + that arrived in a request body, so the name is validated here rather + than relying on each of them to have done it. """ - manifest_path = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id / "manifest.json" + manifest_path = resolve_under( + api_v3.plugin_store_manager.plugins_dir, plugin_id, "manifest.json" + ) + if manifest_path is None: + logger.warning("[PluginVersion] Rejected unsafe plugin id %r", plugin_id) + return '' try: with open(manifest_path, 'r', encoding='utf-8') as f: manifest = json.load(f) diff --git a/web_interface/blueprints/api_v3/config.py b/web_interface/blueprints/api_v3/config.py index d846d4fc..44251607 100644 --- a/web_interface/blueprints/api_v3/config.py +++ b/web_interface/blueprints/api_v3/config.py @@ -11,6 +11,7 @@ from web_interface.blueprints.api_v3 import ( remove_empty_secrets, request, separate_secrets, strip_masked_values, success_response, ) +from src.common.path_safety import resolve_under import web_interface.blueprints.api_v3 as _pkg # Read through the module rather than bound by value: tests patch these # as module attributes, and a value binding would not see the patch. @@ -934,7 +935,19 @@ def save_main_config(): plugins_dir = Path(plugins_dir_name) else: plugins_dir = PROJECT_ROOT / plugins_dir_name - schema_path = plugins_dir / plugin_id / 'config_schema.json' + # plugin_id is already known to be a loaded plugin (the + # membership test above), so this cannot currently traverse -- + # but the path is built from a request key, and the guard and + # the join are far enough apart that a later edit could + # separate them. Build it through the shared helper instead. + schema_path = resolve_under(plugins_dir, plugin_id, 'config_schema.json') + + if schema_path is None: + return error_response( + ErrorCode.VALIDATION_ERROR, + f"Invalid plugin id '{plugin_id}'", + status_code=400 + ) if schema_path.exists(): try: diff --git a/web_interface/blueprints/api_v3/misc.py b/web_interface/blueprints/api_v3/misc.py index 5998d06d..aea249c3 100644 --- a/web_interface/blueprints/api_v3/misc.py +++ b/web_interface/blueprints/api_v3/misc.py @@ -13,6 +13,7 @@ from web_interface.blueprints.api_v3 import ( error_response, get_error_aggregator, json, jsonify, logger, os, request, subprocess, success_response, tempfile, ) +from src.common.path_safety import safe_path_component import web_interface.blueprints.api_v3 as _pkg # Read through the module rather than bound by value: tests patch these # as module attributes, and a value binding would not see the patch. @@ -300,6 +301,12 @@ def delete_cache_file(): cache_key = data['key'] + # The key names the file about to be removed. DiskCache refuses an + # unusable key on its own, but silently: say so here instead of + # reporting a deletion that never happened. + if safe_path_component(cache_key) is None: + return jsonify({'status': 'error', 'message': 'Invalid cache key'}), 400 + # Delete the cache file api_v3.cache_manager.clear_cache(cache_key) diff --git a/web_interface/blueprints/api_v3/plugins.py b/web_interface/blueprints/api_v3/plugins.py index 33bf7ae6..2e61ef41 100644 --- a/web_interface/blueprints/api_v3/plugins.py +++ b/web_interface/blueprints/api_v3/plugins.py @@ -18,6 +18,9 @@ from web_interface.blueprints.api_v3 import ( separate_secrets, shutil, stat, subprocess, success_response, sys, tempfile, uuid, validate_request_json, ) +from src.common.path_safety import ( + resolve_under, safe_path_component, safe_relative_parts, +) import web_interface.blueprints.api_v3 as _pkg # Read through the module rather than bound by value: tests patch these # as module attributes, and a value binding would not see the patch. @@ -946,12 +949,29 @@ def update_plugin(): status_code=500 ) - plugin_id = data['plugin_id'] + # The id names a directory this handler reads a manifest out of and + # hands to the store manager to run git in. It comes from the request + # body, so it is validated before anything is joined to a path, and + # the validated value -- not the raw one -- is what gets used below. + plugin_id = safe_path_component(data['plugin_id']) + if not plugin_id: + return error_response( + ErrorCode.INVALID_INPUT, + 'Invalid plugin_id', + status_code=400 + ) # Always do direct updates (they're fast git pull operations) # Operation queue is reserved for longer operations like install/uninstall - plugin_dir = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id - manifest_path = plugin_dir / "manifest.json" + plugins_base = Path(api_v3.plugin_store_manager.plugins_dir) + plugin_dir = plugins_base / plugin_id + manifest_path = resolve_under(plugin_dir, "manifest.json") + if manifest_path is None: + return error_response( + ErrorCode.INVALID_INPUT, + 'Invalid plugin_id', + status_code=400 + ) current_last_updated = None current_commit = None @@ -983,7 +1003,9 @@ def update_plugin(): current_branch = git_info_before.get('branch') # Check if plugin is a git repo first (for better error messages) - plugin_path_dir = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id + # plugin_id is validated above; reuse the same directory rather + # than rebuilding it from a value that might not match. + plugin_path_dir = plugin_dir is_git_repo = False if plugin_path_dir.exists(): git_info = api_v3.plugin_store_manager._get_local_git_info(plugin_path_dir) @@ -1069,7 +1091,7 @@ def update_plugin(): message=message ) else: - plugin_path_dir = Path(api_v3.plugin_store_manager.plugins_dir) / plugin_id + plugin_path_dir = plugin_dir if not plugin_path_dir.exists(): client_msg = 'Plugin update failed: plugin not found' else: @@ -3525,6 +3547,14 @@ def delete_of_the_day_json(): if not file_id: return jsonify({'status': 'error', 'message': 'file_id is required'}), 400 + # file_id names a file that is about to be unlinked, and it arrives + # straight from the request body. Unvalidated, a file_id of + # "../../../../etc/cron" made this endpoint delete any .json file on + # the device the service could write to. + safe_file_id = safe_path_component(file_id) + if not safe_file_id: + return jsonify({'status': 'error', 'message': 'Invalid file_id'}), 400 + # Get plugin directory plugin_id = 'ledmatrix-of-the-day' if api_v3.plugin_manager: @@ -3535,9 +3565,10 @@ def delete_of_the_day_json(): if not plugin_dir or not Path(plugin_dir).exists(): return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - data_dir = Path(plugin_dir) / 'of_the_day' - filename = f"{file_id}.json" - file_path = data_dir / filename + filename = f"{safe_file_id}.json" + file_path = resolve_under(Path(plugin_dir) / 'of_the_day', filename) + if file_path is None: + return jsonify({'status': 'error', 'message': 'Invalid file_id'}), 400 if not file_path.exists(): return jsonify({'status': 'error', 'message': f'File {filename} not found'}), 404 @@ -3549,7 +3580,7 @@ def delete_of_the_day_json(): try: sys.path.insert(0, str(plugin_dir)) from scripts.update_config import remove_category_from_config - remove_category_from_config(file_id) + remove_category_from_config(safe_file_id) except Exception as e: logger.warning("Could not update config: %s", e) @@ -3563,23 +3594,44 @@ def delete_of_the_day_json(): return jsonify({'status': 'error', 'message': 'An error occurred; see logs for details', 'details': describe_exception(e)}), 500 @api_v3.route('/plugins//static/', methods=['GET']) def serve_plugin_static(plugin_id, file_path): - """Serve static files from plugin directory""" + """Serve static files from plugin directory. + + Both URL parts are validated before anything is opened. This handler used + to read whatever the path resolved to as long as ``str(file).startswith`` + the plugin directory, which let two different things through: + + * ``plugin_id`` of ``..`` -- Flask's default converter forbids a slash but + not dots, and ``get_plugin_directory('..')`` happily returned the parent + of the plugins directory because it exists. Every file under the project + root then "started with" that directory, ``config/config_secrets.json`` + included. + * a sibling directory sharing a prefix: with the plugin directory + ``plugin-repos/foo``, ``../foo-evil/x`` resolves to + ``plugin-repos/foo-evil/x``, whose string does start with + ``plugin-repos/foo``. + """ try: + safe_plugin_id = safe_path_component(plugin_id) + if not safe_plugin_id: + return jsonify({'status': 'error', 'message': 'Invalid plugin ID'}), 400 + + safe_parts = safe_relative_parts(file_path) + if not safe_parts: + return jsonify({'status': 'error', 'message': 'Invalid file path'}), 400 + # Get plugin directory if api_v3.plugin_manager: - plugin_dir = api_v3.plugin_manager.get_plugin_directory(plugin_id) + plugin_dir = api_v3.plugin_manager.get_plugin_directory(safe_plugin_id) else: - plugin_dir = PROJECT_ROOT / 'plugins' / plugin_id + plugin_dir = PROJECT_ROOT / 'plugins' / safe_plugin_id if not plugin_dir or not Path(plugin_dir).exists(): return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404 - # Resolve file path (prevent directory traversal) - plugin_dir = Path(plugin_dir).resolve() - requested_file = (plugin_dir / file_path).resolve() - - # Security check: ensure file is within plugin directory - if not str(requested_file).startswith(str(plugin_dir)): + # Containment is still checked after resolving: name validation cannot + # see a symlink inside the plugin directory that points out of it. + requested_file = resolve_under(plugin_dir, *safe_parts) + if requested_file is None: return jsonify({'status': 'error', 'message': 'Invalid file path'}), 403 # Check if file exists @@ -3588,13 +3640,14 @@ def serve_plugin_static(plugin_id, file_path): # Determine content type content_type = 'text/plain' - if file_path.endswith('.html'): + name = requested_file.name + if name.endswith('.html'): content_type = 'text/html' - elif file_path.endswith('.js'): + elif name.endswith('.js'): content_type = 'application/javascript' - elif file_path.endswith('.css'): + elif name.endswith('.css'): content_type = 'text/css' - elif file_path.endswith('.json'): + elif name.endswith('.json'): content_type = 'application/json' # Read and return file diff --git a/web_interface/blueprints/pages_v3.py b/web_interface/blueprints/pages_v3.py index 958683c8..a6e6cf4c 100644 --- a/web_interface/blueprints/pages_v3.py +++ b/web_interface/blueprints/pages_v3.py @@ -4,8 +4,6 @@ from markupsafe import escape from html.parser import HTMLParser import json import logging -import os -import os.path import re from pathlib import Path @@ -13,6 +11,7 @@ from pathlib import Path _SAFE_PLUGIN_ID_RE = re.compile(r'^[a-zA-Z0-9_-]{1,64}$') _SAFE_WEB_UI_FILE_RE = re.compile(r'^[a-zA-Z0-9_-]{1,64}\.html$') from src.web_interface.secret_helpers import mask_secret_fields +from src.common.path_safety import resolve_under, safe_path_component logger = logging.getLogger(__name__) @@ -271,7 +270,15 @@ def serve_plugin_web_ui(plugin_id, filename): """Serve a plugin's web_ui/ HTML fragment as a standalone page. Wraps the fragment with a minimal HTML page that injects window.PLUGIN_ID - and loads Tailwind CSS so the fragment runs correctly in a sandboxed iframe. + and loads Tailwind CSS so the fragment runs correctly inside the iframe + that plugin_config.html embeds it in. + + That iframe carries no ``sandbox`` attribute, so the fragment runs with + the interface's own origin. That is deliberate rather than an oversight: + the fragment is a file from an installed plugin, and an installed plugin + already runs Python on the device. The trust boundary is plugin install, + not this route. It is worth knowing when reading the code, which is why + it says so here instead of claiming a sandbox that is not there. """ # Validate URL-derived values against strict allowlists before any path or # script operations. @@ -280,11 +287,12 @@ def serve_plugin_web_ui(plugin_id, filename): if not _SAFE_WEB_UI_FILE_RE.match(filename): return 'Invalid filename', 400, {'Content-Type': 'text/plain'} - # os.path.basename() is the CodeQL-recognised path sanitizer used throughout - # this codebase (see plugin_loader.py). Applying it here breaks the taint - # chain even though the allowlist above already prevents path separators. - safe_id = os.path.basename(plugin_id) - safe_fn = os.path.basename(filename) + # The allowlists above already forbid a separator, but the value that gets + # joined has to be the checked one, not the argument -- see + # src/common/path_safety.py. safe_path_component rejects rather than + # truncates, so these cannot disagree. + safe_id = safe_path_component(plugin_id) + safe_fn = safe_path_component(filename) if not safe_id or not safe_fn: return 'Invalid path component', 400, {'Content-Type': 'text/plain'} @@ -294,22 +302,19 @@ def serve_plugin_web_ui(plugin_id, filename): try: _plugins_base = Path(pages_v3.plugin_manager.plugins_dir).resolve() - # Reconstruct from sanitised basename — CodeQL-approved pattern. - _plugin_dir = (_plugins_base / safe_id).resolve() - _plugin_dir.relative_to(_plugins_base) # containment guard + _plugin_dir = resolve_under(_plugins_base, safe_id) + if _plugin_dir is None: + return 'Forbidden', 403, {'Content-Type': 'text/plain'} # Mirror PluginManager's ledmatrix- prefix fallback. if not _plugin_dir.exists(): - _alt_id = os.path.basename(f'ledmatrix-{safe_id}') - _alt = (_plugins_base / _alt_id).resolve() - try: - _alt.relative_to(_plugins_base) + _alt = resolve_under(_plugins_base, f'ledmatrix-{safe_id}') + if _alt is not None: _plugin_dir = _alt - except ValueError: - pass - web_ui_path = (_plugin_dir / 'web_ui' / safe_fn).resolve() - web_ui_path.relative_to(_plugin_dir / 'web_ui') # second guard + web_ui_path = resolve_under(_plugin_dir / 'web_ui', safe_fn) + if web_ui_path is None: + return 'Forbidden', 403, {'Content-Type': 'text/plain'} if not web_ui_path.exists(): return 'Not found', 404, {'Content-Type': 'text/plain'} @@ -640,9 +645,11 @@ def _load_plugin_config_partial(plugin_id): Load plugin configuration partial - server-side rendered form. This replaces the client-side generateConfigForm() JavaScript. """ - # Sanitize with basename (CodeQL-recognized sanitizer) then regex-validate format - plugin_id = os.path.basename(plugin_id or '') - if not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9._\-:]*$', plugin_id): + # Refuse an id that is not a plain directory name, rather than quietly + # basename-ing it down to one: "../weather" used to become "weather" and + # render a partial the caller never asked for. + plugin_id = safe_path_component(plugin_id) + if not plugin_id or not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9._\-:]*$', plugin_id): return '
Invalid plugin ID
', 400 try: @@ -655,10 +662,8 @@ def _load_plugin_config_partial(plugin_id): # Resolve and validate all plugin paths against the plugins base directory _plugins_base = Path(pages_v3.plugin_manager.plugins_dir).resolve() - _plugin_dir = (_plugins_base / plugin_id).resolve() - try: - _plugin_dir.relative_to(_plugins_base) - except ValueError: + _plugin_dir = resolve_under(_plugins_base, plugin_id) + if _plugin_dir is None: return '
Invalid plugin ID
', 400 # Try to get plugin info first @@ -682,19 +687,17 @@ def _load_plugin_config_partial(plugin_id): config = full_config.get(plugin_id, {}) # Load uploaded images from metadata file if images field exists in schema - schema_path_temp = _plugin_dir / "config_schema.json" - if schema_path_temp.exists(): + schema_path_temp = resolve_under(_plugin_dir, "config_schema.json") + if schema_path_temp is not None and schema_path_temp.exists(): try: with open(schema_path_temp, 'r', encoding='utf-8') as f: temp_schema = json.load(f) if (temp_schema.get('properties', {}).get('images', {}).get('x-widget') == 'file-upload' or temp_schema.get('properties', {}).get('images', {}).get('x_widget') == 'file-upload'): _assets_base = (Path(__file__).parent.parent.parent / 'assets' / 'plugins').resolve() - metadata_file = (_assets_base / plugin_id / 'uploads' / '.metadata.json').resolve() - try: - metadata_file.relative_to(_assets_base) - except ValueError: - metadata_file = None + metadata_file = resolve_under( + _assets_base, plugin_id, 'uploads', '.metadata.json' + ) if metadata_file and metadata_file.exists(): try: with open(metadata_file, 'r', encoding='utf-8') as mf: @@ -714,8 +717,8 @@ def _load_plugin_config_partial(plugin_id): # Get plugin schema schema = {} - schema_path = _plugin_dir / "config_schema.json" - if schema_path.exists(): + schema_path = resolve_under(_plugin_dir, "config_schema.json") + if schema_path is not None and schema_path.exists(): try: with open(schema_path, 'r', encoding='utf-8') as f: schema = json.load(f) @@ -724,8 +727,8 @@ def _load_plugin_config_partial(plugin_id): # Get web UI actions from plugin manifest web_ui_actions = [] - manifest_path = _plugin_dir / "manifest.json" - if manifest_path.exists(): + manifest_path = resolve_under(_plugin_dir, "manifest.json") + if manifest_path is not None and manifest_path.exists(): try: with open(manifest_path, 'r', encoding='utf-8') as f: manifest = json.load(f) @@ -773,9 +776,10 @@ def _load_plugin_config_partial(plugin_id): def _load_starlark_config_partial(app_id): """Load configuration partial for a Starlark app.""" - # Sanitize with basename (CodeQL-recognized sanitizer) then regex-validate format - app_id = os.path.basename(app_id or '') - if not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9_\-]*$', app_id): + # Refuse an id that is not a plain directory name rather than basename-ing + # it down to one -- see _load_plugin_config_partial for why. + app_id = safe_path_component(app_id) + if not app_id or not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9_\-]*$', app_id): return '
Invalid app ID
', 400 try: @@ -814,11 +818,7 @@ def _load_starlark_config_partial(app_id): # Load schema from schema.json if it exists — validate path stays within starlark_base schema = None - schema_file = (starlark_base / app_id / 'schema.json').resolve() - try: - schema_file.relative_to(starlark_base) - except ValueError: - schema_file = None + schema_file = resolve_under(starlark_base, app_id, 'schema.json') if schema_file and schema_file.exists(): try: with open(schema_file, 'r') as f: @@ -828,11 +828,7 @@ def _load_starlark_config_partial(app_id): # Load config from config.json if it exists — validate path stays within starlark_base config = {} - config_file = (starlark_base / app_id / 'config.json').resolve() - try: - config_file.relative_to(starlark_base) - except ValueError: - config_file = None + config_file = resolve_under(starlark_base, app_id, 'config.json') if config_file and config_file.exists(): try: with open(config_file, 'r') as f: diff --git a/web_interface/static/v3/js/app-shell.js b/web_interface/static/v3/js/app-shell.js index 70aff559..0ee00fd5 100644 --- a/web_interface/static/v3/js/app-shell.js +++ b/web_interface/static/v3/js/app-shell.js @@ -602,9 +602,9 @@ }; // Build the + label as DOM nodes so a // hostile plugin.icon (e.g. containing a quote) can't - // break out of the attribute. escapeHtml only escapes - // <, >, &, not ", so attribute-context interpolation - // would be unsafe. + // break out of the attribute. escapeHtml now escapes + // quotes too, but setting the property directly cannot + // be got wrong at all, so it stays. const iconEl = document.createElement('i'); iconEl.className = plugin.icon || 'fas fa-puzzle-piece'; const labelNode = document.createTextNode(plugin.name || plugin.id); @@ -643,10 +643,15 @@ } }, + // Quotes too, so the result is safe inside a quoted attribute + // value -- the textContent/innerHTML round-trip alone only + // escapes &, < and >. escapeHtml(text) { const div = document.createElement('div'); div.textContent = text; - return div.innerHTML; + return div.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); }, async refreshPlugins() { diff --git a/web_interface/static/v3/js/utils/error_handler.js b/web_interface/static/v3/js/utils/error_handler.js index 73794741..f6afbe3d 100644 --- a/web_interface/static/v3/js/utils/error_handler.js +++ b/web_interface/static/v3/js/utils/error_handler.js @@ -260,6 +260,9 @@ function closeErrorModal() { /** * Escape HTML to prevent XSS. + * + * Quotes are escaped as well so the result is safe inside a quoted attribute + * value -- the textContent/innerHTML round-trip alone only escapes &, < and >. */ function escapeHtml(text) { if (typeof text !== 'string') { @@ -267,7 +270,9 @@ function escapeHtml(text) { } const div = document.createElement('div'); div.textContent = text; - return div.innerHTML; + return div.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); } /** diff --git a/web_interface/static/v3/js/widgets/base-widget.js b/web_interface/static/v3/js/widgets/base-widget.js index 084c8555..93c56b4c 100644 --- a/web_interface/static/v3/js/widgets/base-widget.js +++ b/web_interface/static/v3/js/widgets/base-widget.js @@ -118,15 +118,27 @@ /** * Escape HTML to prevent XSS * Always escapes the input, even for non-strings, by coercing to string first + * + * The result is safe in text content AND inside quoted attribute values. + * The textContent/innerHTML round-trip only escapes `&`, `<` and `>` -- + * the HTML serializer leaves quotes alone because they are harmless in a + * text node. Every widget here interpolates the result into attributes + * (`value="${escapeHtml(v)}"`), where an unescaped `"` closes the + * attribute and lets the value inject its own, so the quotes have to go + * too. Each widget's standalone fallback already did this; the shared + * implementation they all prefer did not. + * * @param {*} text - Text to escape (will be coerced to string) - * @returns {string} Escaped text + * @returns {string} Escaped text, safe for text and attribute contexts */ escapeHtml(text) { // Always coerce to string first, then escape const textStr = String(text); const div = document.createElement('div'); div.textContent = textStr; - return div.innerHTML; + return div.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); } /** diff --git a/web_interface/static/v3/js/widgets/example-color-picker.js b/web_interface/static/v3/js/widgets/example-color-picker.js index 4121bca5..a05e95f0 100644 --- a/web_interface/static/v3/js/widgets/example-color-picker.js +++ b/web_interface/static/v3/js/widgets/example-color-picker.js @@ -50,10 +50,14 @@ } // Escape HTML to prevent XSS (for HTML contexts) + // Quotes too: the result lands in quoted attribute values below, and + // the textContent/innerHTML round-trip only escapes &, < and >. const escapeHtml = (text) => { const div = document.createElement('div'); - div.textContent = text; - return div.innerHTML; + div.textContent = String(text ?? ''); + return div.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); }; // Use validated/sanitized hex for style attribute and input values diff --git a/web_interface/static/v3/js/widgets/file-upload.js b/web_interface/static/v3/js/widgets/file-upload.js index 23e40886..8a2ed11f 100644 --- a/web_interface/static/v3/js/widgets/file-upload.js +++ b/web_interface/static/v3/js/widgets/file-upload.js @@ -804,10 +804,14 @@ const schedule = image.schedule || { enabled: false, mode: 'always', start_time: '08:00', end_time: '18:00', days: {} }; // Escape HTML helper + // Quotes too: the result lands in quoted attribute values below, and the + // textContent/innerHTML round-trip only escapes &, < and >. const escapeHtml = (text) => { const div = document.createElement('div'); - div.textContent = text; - return div.innerHTML; + div.textContent = String(text ?? ''); + return div.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); }; // Use sanitizedId for all ID references in the schedule HTML diff --git a/web_interface/static/v3/js/widgets/json-file-manager.js b/web_interface/static/v3/js/widgets/json-file-manager.js index 18d27b0c..06ce1188 100644 --- a/web_interface/static/v3/js/widgets/json-file-manager.js +++ b/web_interface/static/v3/js/widgets/json-file-manager.js @@ -738,10 +738,15 @@ delete btn._jfmOrigText; } + // Quotes too: the result lands in quoted attribute values (title=, + // data-cat=, pattern=, ...), and the textContent/innerHTML round-trip + // only escapes &, < and >. _esc(str) { const d = document.createElement('div'); d.textContent = String(str ?? ''); - return d.innerHTML; + return d.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); } _fmtSize(bytes) { diff --git a/web_interface/static/v3/js/widgets/plugin-file-manager.js b/web_interface/static/v3/js/widgets/plugin-file-manager.js index 5cc54154..71e8bff2 100644 --- a/web_interface/static/v3/js/widgets/plugin-file-manager.js +++ b/web_interface/static/v3/js/widgets/plugin-file-manager.js @@ -206,10 +206,15 @@ else console.log(`[PFM][${type}] ${msg}`); } + // Quotes too: the result lands in quoted attribute values (id=, value=, + // data-col=), and the textContent/innerHTML round-trip only escapes + // &, < and >. function escHtml(s) { const d = document.createElement('div'); d.textContent = String(s ?? ''); - return d.innerHTML; + return d.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); } function formatSize(bytes) { @@ -383,7 +388,7 @@ - `; + `); overlay.appendChild(modal); // Bind events after DOM insertion — filename captured in closure, not in HTML. modal.querySelector(`#${CSS.escape(fieldId)}_modal_close`).addEventListener('click', () => window._pfmCloseModal(fieldId)); @@ -411,7 +416,7 @@ -
`; +
`); } }; @@ -431,6 +436,20 @@ if (!entries.length) { container.textContent = 'No entries.'; return; } const cols = Object.keys(entries[0][1]); + + // Delegated listener: day/col reach _pfmCellEdit only through data-* + // attributes, never through a JS string spliced into an inline + // handler -- the browser HTML-decodes attribute values before + // running them as script, which undoes escHtml's quote escaping and + // reopens the exact injection it exists to close. `container` is a + // fresh element per modal open (see _pfmOpenEdit), so this attaches + // exactly once per table, even though buildPage() re-renders below. + container.addEventListener('input', (e) => { + const cell = e.target.closest('input[data-day], textarea[data-day]'); + if (!cell) return; + window._pfmCellEdit(fieldId, cell.dataset.day, cell.dataset.col, cell.value); + }); + const MS_PER_DAY = 86400 * 1000; // eslint-disable-line no-magic-numbers -- 86400s/day is not magic const todayDoy = Math.ceil((new Date() - new Date(new Date().getFullYear(), 0, 0)) / MS_PER_DAY); const total = entries.length; @@ -459,18 +478,16 @@ ${pageEntries.map(([day, val]) => ` - + ${escHtml(day)} ${cols.map(col => { const v = val[col] ?? ''; const isLong = String(v).length > 60 || col === 'description' || col === 'definition' || col === 'content'; return isLong - ? `` - : ``; + : ``; }).join('')} `).join('')} @@ -489,7 +506,7 @@ ${page >= totalPages ? 'disabled' : ''} onclick="window._pfmTablePage('${fieldId}',${page + 1})">Next › - `; + `); st._tablePage = page; st._tableEntries = entries; st._tableCols = cols; @@ -578,7 +595,7 @@ - `; + `); overlay.appendChild(modal); modal.querySelector(`#${CSS.escape(fieldId)}_del_close`).addEventListener('click', () => window._pfmCloseModal(fieldId)); modal.querySelector(`#${CSS.escape(fieldId)}_del_cancel`).addEventListener('click', () => window._pfmCloseModal(fieldId)); @@ -631,7 +648,7 @@ Create - `; + `); overlay.appendChild(modal); modal.querySelector(`#${CSS.escape(fieldId)}_cre_close`).addEventListener('click', () => window._pfmCloseModal(fieldId)); modal.querySelector(`#${CSS.escape(fieldId)}_cre_cancel`).addEventListener('click', () => window._pfmCloseModal(fieldId)); @@ -784,7 +801,7 @@
Loading…
- `; + `); loadFiles(fieldId); }, diff --git a/web_interface/static/v3/js/widgets/url-input.js b/web_interface/static/v3/js/widgets/url-input.js index 6a18dfcb..b2b09207 100644 --- a/web_interface/static/v3/js/widgets/url-input.js +++ b/web_interface/static/v3/js/widgets/url-input.js @@ -54,9 +54,16 @@ // RFC 3986 scheme pattern: starts with letter, then letters/digits/+/./- const RFC_SCHEME_PATTERN = /^[A-Za-z][A-Za-z0-9+.-]*$/; + // Schemes that execute script when navigated to. These can never be + // allowed, whatever a schema's allowedProtocols asks for: the validated + // value is written straight into an , so allowing "javascript" + // here would turn a config field into script execution. + const SCRIPTABLE_SCHEMES = ['javascript', 'data', 'vbscript', 'blob', 'filesystem']; + /** * Normalize and validate protocol list against RFC 3986 scheme pattern. * Accepts schemes like "http", "https", "git+ssh", "android-app", etc. + * Scriptable schemes are dropped -- see SCRIPTABLE_SCHEMES. * @param {Array|string} protocols - Protocol list (array or comma-separated string) * @returns {Array} Normalized lowercase protocols, defaults to ['http', 'https'] */ @@ -70,15 +77,25 @@ const normalized = list .map(p => String(p).trim()) .filter(p => RFC_SCHEME_PATTERN.test(p)) - .map(p => p.toLowerCase()); + .map(p => p.toLowerCase()) + .filter(p => !SCRIPTABLE_SCHEMES.includes(p)); return normalized.length > 0 ? normalized : ['http', 'https']; } + /** + * True when `string` parses as a URL whose scheme is allowed AND is not + * one that executes script. The scriptable check is repeated here rather + * than trusted to normalizeProtocols so that a caller passing its own + * protocol list cannot re-open the hole. + */ function isValidUrl(string, allowedProtocols) { try { const url = new URL(string); + const protocol = url.protocol.replace(':', '').toLowerCase(); + if (SCRIPTABLE_SCHEMES.includes(protocol)) { + return false; + } if (allowedProtocols && allowedProtocols.length > 0) { - const protocol = url.protocol.replace(':', '').toLowerCase(); return allowedProtocols.includes(protocol); } return true; @@ -87,6 +104,15 @@ } } + /** + * A value safe to use as an : the URL itself when it validates, + * and '' otherwise. Keeps the "render an href for whatever is stored" + * path from emitting a javascript: URL that was never validated. + */ + function safeHref(value, allowedProtocols) { + return isValidUrl(value, allowedProtocols) ? value : ''; + } + window.LEDMatrixWidgets.register('url-input', { name: 'URL Input Widget', version: '1.0.0', @@ -139,7 +165,7 @@ html += `
@@ -231,10 +257,23 @@ const protocols = normalizeProtocols(widgetEl?.dataset.protocols); if (previewEl && previewLink) { - if (value && isValidUrl(value, protocols)) { + // Scheme checked inline, right where the value reaches the DOM sink, + // rather than through safeHref/isValidUrl -- CodeQL's DOM-based-XSS + // sanitizer recognition does not trace a boolean-returning helper two + // calls deep, so it kept flagging this assignment even though the + // scriptable-scheme check (see SCRIPTABLE_SCHEMES) already covered it. + let scheme = ''; + try { + scheme = value ? new URL(value).protocol.replace(':', '').toLowerCase() : ''; + } catch (_) { + scheme = ''; + } + const schemeIsSafe = !!scheme && !SCRIPTABLE_SCHEMES.includes(scheme) && protocols.includes(scheme); + if (schemeIsSafe) { previewLink.href = value; previewEl.classList.remove('hidden'); } else { + previewLink.removeAttribute('href'); previewEl.classList.add('hidden'); } } diff --git a/web_interface/static/v3/plugins_manager.js b/web_interface/static/v3/plugins_manager.js index a16e66f6..a83c6897 100644 --- a/web_interface/static/v3/plugins_manager.js +++ b/web_interface/static/v3/plugins_manager.js @@ -4370,12 +4370,15 @@ function renderCustomRegistryPlugins(plugins, registryUrl) { return; } - // Escape HTML helper + // Escape HTML helper. Quotes too: the result lands in quoted attribute + // values, and the textContent/innerHTML round-trip only escapes &, < and >. const escapeHtml = (text) => { if (!text) return ''; const div = document.createElement('div'); div.textContent = text; - return div.innerHTML; + return div.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); }; // Helper function to escape for JavaScript strings @@ -4471,11 +4474,15 @@ function isGithubUrl(url) { } } -// Utility function to escape HTML +// Utility function to escape HTML. Quotes too: most call sites interpolate the +// result into a quoted attribute value, and the textContent/innerHTML +// round-trip only escapes &, < and >. function escapeHtml(text) { const div = document.createElement('div'); div.textContent = text; - return div.innerHTML; + return div.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); } // Utility function to escape text for use in HTML attributes @@ -5623,11 +5630,15 @@ document.addEventListener('htmx:afterSettle', function() { let starlarkDataLoaded = false; // ── Helpers ───────────────────────────────────────────────────────────── + // Quotes too: the result lands in quoted attribute values (data-app-id=, + // title=), and the textContent/innerHTML round-trip only escapes &, < and >. function escapeHtml(str) { if (!str) return ''; const div = document.createElement('div'); div.textContent = str; - return div.innerHTML; + return div.innerHTML + .replace(/"/g, '"') + .replace(/'/g, '''); } function isStarlarkInstalled(appId) { diff --git a/web_interface/templates/v3/partials/cache.html b/web_interface/templates/v3/partials/cache.html index a52e5a53..30c3eb98 100644 --- a/web_interface/templates/v3/partials/cache.html +++ b/web_interface/templates/v3/partials/cache.html @@ -152,7 +152,8 @@ function displayCacheFiles(data) { ${escapeHtml(modifiedStr)} -