diff --git a/.codacy.yml b/.codacy.yml index 441240fe..7203ef92 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -4,4 +4,3 @@ exclude_paths: - "plugins/**" - "assets/**" - "test/**" - - "scripts/debug/**" diff --git a/.gitattributes b/.gitattributes index dfe07704..39867887 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,8 @@ # Auto detect text files and perform LF normalization * text=auto + +# Files the Pi executes must stay LF even in a Windows checkout with +# core.autocrlf=true: a CRLF shebang line fails with "bad interpreter", +# and systemd rejects CRLF unit files. +*.sh text eol=lf +*.service text eol=lf diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 7c432109..e2dff6f9 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -6,6 +6,10 @@ on: jobs: claude-review: + # Pull requests from forks get no repository secrets, so without this + # guard every outside contributor's PR showed this check red for a reason + # they can't fix. Skipped checks don't block merging. + if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest permissions: contents: read diff --git a/.gitignore b/.gitignore index 86909a99..a9931074 100644 --- a/.gitignore +++ b/.gitignore @@ -3,15 +3,13 @@ __pycache__/ *.py[cod] *$py.class -# Secrets -config/config_secrets.json -# Atomic writes leave these behind when a save or a test is interrupted; -# the suite drops several per run. -config/.config_secrets.json.tmp.* -config/config.json -config/config.json.backup -config/wifi_config.json -config/uninstalled_plugins.json +# Secrets and per-device state. Everything the software writes into config/ +# is local to one device -- config.json, config_secrets.json, wifi_config.json, +# ytm_auth.json (a login session), saved_repositories.json, font_overrides.json, +# and the temp files atomic writes leave behind when interrupted -- so only +# the templates are tracked. Listing files one by one missed several. +config/* +!config/*.template.json credentials.json token.pickle diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10073329..1ff11b58 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,11 @@ repos: types: [python] pass_filenames: false + # Manual only: src/ still has ~500 type errors, so running this on every + # commit would block every contributor. Run it with + # pre-commit run mypy --hook-stage manual + # and don't add new errors in the files you touch. mypy.ini's `files = src` + # is what gives it a target, since pass_filenames is off. - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.8.0 hooks: @@ -45,6 +50,7 @@ repos: args: [--ignore-missing-imports, --no-error-summary] pass_filenames: false files: ^src/ + stages: [manual] - repo: https://github.com/PyCQA/bandit rev: 1.8.3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ff75ac..e523b2cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,13 @@ accepts both, but the store flags the old spelling as deprecated ## Unreleased +- Contributor tooling and docs: + - `mypy.ini` parses again. A multi-line `exclude` and trailing comments on values made mypy refuse the whole file, so none of its settings applied and the pre-commit hook failed with "Missing target". The mypy hook is now manual (`pre-commit run mypy --hook-stage manual`) while the ~500 existing type errors in `src/` are paid down. + - `.gitignore` ignores everything in `config/` except the templates; `ytm_auth.json`, `saved_repositories.json`, `wifi_status.json` and `font_overrides.json` weren't ignored. + - `.sh` and `.service` files are always checked out with LF line endings. + - The Claude code-review check is skipped on pull requests from forks, which get no secrets and always failed it. + - `check_system_compatibility.sh` treats Python 3.13 (what Trixie ships) as supported and anything below 3.10 as an error. + - Doc fixes: emulator guide (Python 3.10+, `emulator_config.json` isn't in the repo), README's nonexistent "API Metrics" feature, a stale route count, and missing index entries for the scroll-performance and offscreen-rendering docs and the frame-soak and render-bench scripts. - Security and input-validation fixes: - Installing from a URL (and a registry install whose manifest renames the plugin) refuses a plugin id that isn't a single safe name, so `../x` can no longer delete and replace a directory outside the plugins directory. - Plugin uninstall and config reset refuse core config sections (`display`, `schedule`, ...) and ids with path parts. Uninstall still cleans the config of a plugin whose directory is already gone. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a450585c..bd1e364d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,10 +58,13 @@ integration tests. 3. **Keep PRs focused.** One conceptual change per PR. If you find adjacent bugs while working, fix them in a separate PR. 4. **Follow the existing code style.** The pre-commit hooks run - `flake8` (E9, F63, F7, F82 plus bugbear `B` checks), `mypy` on - `src/`, `bandit`, and `gitleaks` — install the CLI with + `flake8` (E9, F63, F7, F82 plus bugbear `B` checks), `bandit`, + and `gitleaks` — install the CLI with `python -m pip install pre-commit`, then run - `pre-commit install` so they run on every commit; HTML/JS in + `pre-commit install` so they run on every commit. `mypy` on + `src/` is a manual hook while existing type errors are paid down + (`pre-commit run mypy --hook-stage manual`): please don't add new + errors in the files you touch. HTML/JS in `web_interface/` follows the patterns already in `templates/v3/` and `static/v3/`. 5. **Update documentation** alongside code changes. If you add a diff --git a/README.md b/README.md index 16e66fe7..d628122e 100644 --- a/README.md +++ b/README.md @@ -948,7 +948,7 @@ sudo systemctl enable ledmatrix-web.service - **On-Demand Controls**: Start specific displays (weather, stocks, sports) on demand - **Service Management**: Start/stop the main display service - **System Controls**: Restart, update code, and manage the system -- **API Metrics**: Monitor API usage and system performance +- **System Stats**: CPU, memory and temperature on the Overview tab - **Logs**: View system logs in real-time ### Troubleshooting Web Interface diff --git a/docs/EMULATOR_SETUP_GUIDE.md b/docs/EMULATOR_SETUP_GUIDE.md index d5130ccf..f118dcb0 100644 --- a/docs/EMULATOR_SETUP_GUIDE.md +++ b/docs/EMULATOR_SETUP_GUIDE.md @@ -17,13 +17,13 @@ The LEDMatrix emulator allows you to run and test LEDMatrix displays on your com ## Prerequisites ### System Requirements -- Python 3.7 or higher +- Python 3.10 or higher - Windows, macOS, or Linux - At least 2GB RAM (4GB recommended) - Internet connection for plugin downloads ### Required Software -- Python 3.7+ +- Python 3.10+ - pip (Python package manager) - Git (for plugin management) @@ -50,8 +50,7 @@ pip install -r requirements-emulator.txt ``` This installs: -- `RGBMatrixEmulator` - The core emulation library -- Additional dependencies for display adapters +- `RGBMatrixEmulator` - the emulation library (and whatever it depends on) ### 3. Install Standard Dependencies @@ -63,8 +62,9 @@ pip install -r requirements.txt ### 1. Emulator Configuration File -The emulator uses `emulator_config.json` for configuration. Here's the -default configuration as it ships in the repo: +The emulator uses `emulator_config.json` for configuration. It isn't in +the repo (it's gitignored): RGBMatrixEmulator writes it on first run. +A typical file looks like this: ```json { diff --git a/docs/README.md b/docs/README.md index bd88bdc6..8c5c59c6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,6 +56,8 @@ Going deeper: - [ADVANCED_FEATURES.md](ADVANCED_FEATURES.md) — Vegas scroll, on-demand display, cache management, background services, permissions - [FONT_MANAGER.md](FONT_MANAGER.md) — font system +- [SCROLL_PERFORMANCE.md](SCROLL_PERFORMANCE.md) — how scrolling is paced, and how to make a plugin's marquee smooth +- [OFFSCREEN_RENDERING.md](OFFSCREEN_RENDERING.md) — rendering plugin content off the render thread - [PERMISSIONS.md](PERMISSIONS.md) — file ownership, sudo rules, repair scripts - [MQTT bridge](../integrations/mqtt_bridge/README.md) — control the display from Home Assistant over MQTT diff --git a/docs/REST_API_REFERENCE.md b/docs/REST_API_REFERENCE.md index 2e22a30f..f6a72aa6 100644 --- a/docs/REST_API_REFERENCE.md +++ b/docs/REST_API_REFERENCE.md @@ -46,7 +46,7 @@ the entry below says so. > The three SSE endpoints (`/api/v3/stream/*`) are defined directly on the > Flask app in `app.py` (`stream_stats`, `stream_display`, `stream_logs`). > `test/fixtures/api_v3_url_map.json` is the canonical list of blueprint -> routes (116 URL rules); a test fails if the code and that fixture differ. +> routes; a test fails if the code and that fixture differ. --- diff --git a/mypy.ini b/mypy.ini index e41d4b92..b77e3e7b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,11 @@ [mypy] # Mypy configuration for LEDMatrix +# What a bare `mypy` checks. ini values can't span lines or carry trailing +# comments -- this file used to have both, so mypy refused to read it at all. +files = src +exclude = (^|/)(test|__pycache__)/ + # Python version python_version = 3.10 @@ -25,11 +30,11 @@ warn_unreachable = True # Strict optional checking strict_optional = True -# Disallow untyped definitions -disallow_untyped_defs = False # Set to True once all code is typed +# Disallow untyped definitions (set to True once all code is typed) +disallow_untyped_defs = False -# Disallow untyped calls -disallow_untyped_calls = False # Set to True once all code is typed +# Disallow untyped calls (set to True once all code is typed) +disallow_untyped_calls = False # Check untyped definitions check_untyped_defs = True @@ -96,10 +101,3 @@ ignore_missing_imports = True [mypy-spotipy.*] ignore_missing_imports = True -# Exclude test files and generated files -exclude = (?x)( - ^test/.*| - ^.*/__pycache__/.*| - ^.*\.pyc$ -) - diff --git a/scripts/README.md b/scripts/README.md index d4c4a5be..216e6282 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -31,9 +31,11 @@ display; **diagnostic** — run by hand on a Pi when something is wrong. | `diagnose_web_interface.sh` | diagnostic | Checks why the web interface is not reachable | | `download_pixlet.sh` | keep | Downloads the bundled Pixlet binaries for Starlark apps (also run from the web UI) | | `emergency_reconnect.sh` | diagnostic | Reconnects to your WiFi network if captive-portal testing leaves the Pi offline | +| `frame_soak.py` | diagnostic | Soaks a running display and reports how often frames reached the panel late (docs/SCROLL_PERFORMANCE.md) | | `install_dependencies_apt.py` | keep | Dependency installer that tries apt packages first, then pip (installer Step 7, plugin loader) | | `install_plugin_dependencies.sh` | diagnostic | Installs plugin requirements by hand when the automatic install fails | | `prove_security.py` | keep | Security property checks run by pre-commit | +| `render_bench.py` | diagnostic | Benchmarks the render loop against the panel's real refresh rate on a synthetic strip | | `render_plugin.py` | dev-only | Runs a plugin's `update()` + `display()` and saves the frame as a PNG | | `run_plugin_tests.py` | dev-only | Discovers and runs plugin test suites | | `scroll_speeds.py` | keep | Shows and tries the scroll speeds your panel can display cleanly | diff --git a/scripts/add_defaults_to_schemas.py b/scripts/add_defaults_to_schemas.py index 3ef33e94..6433269a 100755 --- a/scripts/add_defaults_to_schemas.py +++ b/scripts/add_defaults_to_schemas.py @@ -194,7 +194,7 @@ def process_schema_file(schema_path: Path) -> bool: print(f" ✓ Modified {len(modified_fields)} fields") return True else: - print(f" ✓ No changes needed") + print(" ✓ No changes needed") return False diff --git a/scripts/analyze_plugin_schemas.py b/scripts/analyze_plugin_schemas.py index e05f20eb..6720dd8c 100755 --- a/scripts/analyze_plugin_schemas.py +++ b/scripts/analyze_plugin_schemas.py @@ -87,7 +87,6 @@ def find_duplicate_fields(schema: Dict[str, Any], path: str = "") -> List[str]: def validate_schema_syntax(schema_path: Path) -> tuple[bool, List[str]]: """Validate JSON Schema syntax.""" - errors = [] try: with open(schema_path, 'r', encoding='utf-8') as f: schema = json.load(f) @@ -164,7 +163,7 @@ def analyze_schema(schema_path: Path) -> Dict[str, Any]: if "update_interval_seconds" in properties: analysis["update_interval_variant"] = "update_interval_seconds" analysis["naming_issues"].append( - f"Uses 'update_interval_seconds' instead of 'update_interval'" + "Uses 'update_interval_seconds' instead of 'update_interval'" ) else: analysis["missing_common_fields"].append(field_name) @@ -239,7 +238,7 @@ def main(): print(f" Missing common fields: {', '.join(result['missing_common_fields'])}") if result['naming_issues']: - print(f" Naming issues:") + print(" Naming issues:") for issue in result['naming_issues']: print(f" - {issue}") diff --git a/scripts/check_system_compatibility.sh b/scripts/check_system_compatibility.sh index 87538e17..0a8530e6 100755 --- a/scripts/check_system_compatibility.sh +++ b/scripts/check_system_compatibility.sh @@ -112,15 +112,13 @@ if command -v python3 >/dev/null 2>&1; then echo "Python: $PYTHON_VERSION" if [ "$PYTHON_MAJOR" -eq "3" ]; then - if [ "$PYTHON_MINOR" -ge "10" ] && [ "$PYTHON_MINOR" -le "12" ]; then - print_success "Python version is fully supported (3.10-3.12)" - elif [ "$PYTHON_MINOR" -eq "13" ]; then - print_warning "Python 3.13 detected - most packages compatible, but some may have limited testing" - print_warning "Please report any compatibility issues you encounter" + if [ "$PYTHON_MINOR" -ge "10" ] && [ "$PYTHON_MINOR" -le "13" ]; then + print_success "Python version is supported (3.10-3.13)" elif [ "$PYTHON_MINOR" -ge "14" ]; then print_warning "Python 3.${PYTHON_MINOR} is very new - some packages may not be compatible yet" else - print_warning "Python 3.${PYTHON_MINOR} is outdated - upgrade to 3.10+ recommended" + # Pillow 12 and the pinned test tools need 3.10+, so this won't install. + print_error "Python 3.${PYTHON_MINOR} is too old - Python 3.10+ is required" fi else print_error "Python 2.x detected - Python 3.10+ is required" diff --git a/scripts/dev_server.py b/scripts/dev_server.py index 4372f14b..dc4a836a 100644 --- a/scripts/dev_server.py +++ b/scripts/dev_server.py @@ -423,7 +423,7 @@ def main(): global _extra_dirs _extra_dirs = args.extra_dir - print(f"LEDMatrix Dev Preview Server") + print("LEDMatrix Dev Preview Server") print(f"Open http://{args.host}:{args.port} in your browser") print(f"Plugin search dirs: {[str(d) for d in get_search_dirs()]}") print() diff --git a/scripts/run_plugin_tests.py b/scripts/run_plugin_tests.py index fcaa8482..0e802770 100755 --- a/scripts/run_plugin_tests.py +++ b/scripts/run_plugin_tests.py @@ -299,6 +299,5 @@ def main(): if __name__ == '__main__': import importlib.util - from typing import Optional sys.exit(main()) diff --git a/scripts/utils/wifi_monitor_daemon.py b/scripts/utils/wifi_monitor_daemon.py index 414c16dc..397bcf63 100755 --- a/scripts/utils/wifi_monitor_daemon.py +++ b/scripts/utils/wifi_monitor_daemon.py @@ -219,7 +219,7 @@ def main(): parser.add_argument( '--foreground', action='store_true', - help='Run in foreground (for debugging)' + help='Accepted for compatibility; the daemon always runs in the foreground' ) args = parser.parse_args() diff --git a/test/js/README.md b/test/js/README.md index 0c341f52..d4aefa06 100644 --- a/test/js/README.md +++ b/test/js/README.md @@ -25,7 +25,7 @@ HTML and the **real** API rather than fixtures: EMULATOR=true python3 web_interface/app.py # http://localhost:5000 # or point the suites at a device -BASE=http://10.0.10.169:5000 node run_all.js +BASE=http://:5000 node run_all.js ``` `run_all.js` skips the DOM suites (rather than failing) when jsdom is missing or diff --git a/test/js/run_all.js b/test/js/run_all.js index bc95a3b4..e2efb82b 100755 --- a/test/js/run_all.js +++ b/test/js/run_all.js @@ -3,7 +3,7 @@ // // node run_all.js unit suites, plus DOM suites if a // web interface is reachable -// BASE=http://10.0.10.169:5000 node run_all.js point the DOM suites at a rig +// BASE=http://:5000 node run_all.js point the DOM suites at a rig // // Unit suites need nothing but node. The DOM suites need `npm install` (jsdom) // and a running web interface, because they deliberately test against the real