Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
73a5304194 | ||
|
|
05e7c43b27 | ||
|
|
2ffc57cf40 | ||
|
|
aab0e9ade0 | ||
|
|
978a03b42d | ||
|
|
bd9f461f70 | ||
|
|
3b93024993 | ||
|
|
85d321cf33 | ||
|
|
63a233f3ed | ||
|
|
7a9d01342a | ||
|
|
9b2f02681d | ||
|
|
7a6bad29fe | ||
|
|
bea00448d3 | ||
|
|
deaa3d7a98 | ||
|
|
cbb8ec41e8 | ||
|
|
c6ce332d49 | ||
|
|
8e5f66501a | ||
|
|
639e1c3a93 | ||
|
|
6096a22c3d | ||
|
|
fefc2d44a2 | ||
|
|
d297dd6217 | ||
|
|
974d7ea57a | ||
|
|
ab0cfd2362 | ||
|
|
d22d0a3754 | ||
|
|
5beef0aa01 | ||
|
|
cf28a8c0d5 | ||
|
|
a06682981c | ||
|
|
bc027c921d | ||
|
|
e0bd7088fa | ||
|
|
313e35a98f | ||
|
|
122e6d6863 | ||
|
|
d488e8a2ad | ||
|
|
b9dcbb5152 | ||
|
|
f27fd260f7 | ||
|
|
eedf680a8c | ||
|
|
ac3a15bfaa | ||
|
|
4961697251 | ||
|
|
cac9644b6d | ||
|
|
f96fdd9f24 | ||
|
|
35c540d0e0 | ||
|
|
7603909c59 | ||
|
|
34b186125a | ||
|
|
ea95f37d73 | ||
|
|
0c7d03a476 | ||
|
|
321a87f734 | ||
|
|
9930bd33b1 | ||
|
|
713539e491 | ||
|
|
327e87f735 | ||
|
|
b5426da2a7 | ||
|
|
302ab1da4f | ||
|
|
9cd2bd14ce | ||
|
|
53ee184bc5 | ||
|
|
e00d75bbb5 | ||
|
|
33f76b4895 |
@@ -0,0 +1,7 @@
|
||||
---
|
||||
exclude_paths:
|
||||
- "plugin-repos/**"
|
||||
- "plugins/**"
|
||||
- "assets/**"
|
||||
- "test/**"
|
||||
- "scripts/debug/**"
|
||||
@@ -0,0 +1,44 @@
|
||||
name: Claude Code Review
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, ready_for_review, reopened]
|
||||
# Optional: Only run on specific file changes
|
||||
# paths:
|
||||
# - "src/**/*.ts"
|
||||
# - "src/**/*.tsx"
|
||||
# - "src/**/*.js"
|
||||
# - "src/**/*.jsx"
|
||||
|
||||
jobs:
|
||||
claude-review:
|
||||
# Optional: Filter by PR author
|
||||
# if: |
|
||||
# github.event.pull_request.user.login == 'external-contributor' ||
|
||||
# github.event.pull_request.user.login == 'new-developer' ||
|
||||
# github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code Review
|
||||
id: claude-review
|
||||
uses: anthropics/claude-code-action@v1
|
||||
with:
|
||||
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
|
||||
plugins: 'code-review@claude-code-plugins'
|
||||
prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://code.claude.com/docs/en/cli-reference for available options
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
actions: read # Required for Claude to read CI results on PRs
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@v1
|
||||
with:
|
||||
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
|
||||
# This is an optional setting that allows Claude to read CI results on PRs
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
|
||||
# Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
|
||||
# prompt: 'Update the pull request description to include a summary of changes.'
|
||||
|
||||
# Optional: Add claude_args to customize behavior and configuration
|
||||
# See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
|
||||
# or https://code.claude.com/docs/en/cli-reference for available options
|
||||
# claude_args: '--allowed-tools Bash(gh pr *)'
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
name: Security Audit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- 'feat/**'
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
schedule:
|
||||
# Weekly full scan — catches new CVEs in existing deps
|
||||
- cron: '0 6 * * 1'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# SAST — Static Application Security Testing
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
sast:
|
||||
name: Static Analysis (bandit + semgrep)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install SAST tools
|
||||
run: pip install bandit==1.8.3 semgrep
|
||||
|
||||
# Bandit — Python-specific security linter
|
||||
# --exit-zero: findings are warnings, not CI blockers.
|
||||
# The security-report job interprets severity.
|
||||
- name: Run bandit
|
||||
run: |
|
||||
bandit -r src/ web_interface/ \
|
||||
-c bandit.yaml \
|
||||
-f json \
|
||||
-o bandit-results.json \
|
||||
--exit-zero
|
||||
|
||||
# Semgrep — broader pattern-based analysis
|
||||
# || true: prevents network/rate-limit errors from blocking the workflow
|
||||
- name: Run semgrep
|
||||
run: |
|
||||
semgrep --config "p/python" \
|
||||
--config "p/flask" \
|
||||
--json \
|
||||
--output semgrep-results.json \
|
||||
src/ web_interface/ \
|
||||
|| true
|
||||
|
||||
- name: Upload SAST artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: sast-results
|
||||
path: |
|
||||
bandit-results.json
|
||||
semgrep-results.json
|
||||
retention-days: 30
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Dependency Vulnerability Scanning
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
dependency-audit:
|
||||
name: Dependency Audit (pip-audit + safety)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install audit tools
|
||||
run: pip install pip-audit safety
|
||||
|
||||
# Install project deps. Hardware-specific packages (rgbmatrix) will fail
|
||||
# to build on Ubuntu runners — || true handles this gracefully.
|
||||
# pip-audit operates on installed packages; partial install is acceptable.
|
||||
- name: Install project dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt || true
|
||||
pip install -r web_interface/requirements.txt || true
|
||||
pip install -r requirements-emulator.txt || true
|
||||
|
||||
- name: Run pip-audit
|
||||
run: |
|
||||
pip-audit --format json --output pip-audit-results.json || true
|
||||
|
||||
- name: Run safety check
|
||||
run: |
|
||||
safety check --output json > safety-results.json 2>&1 || true
|
||||
|
||||
- name: Upload dependency audit artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: dependency-audit-results
|
||||
path: |
|
||||
pip-audit-results.json
|
||||
safety-results.json
|
||||
retention-days: 30
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Secrets Detection
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
secrets-scan:
|
||||
name: Secrets Scan (gitleaks)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Full history for scanning all commits
|
||||
|
||||
# continue-on-error: config/config_secrets.template.json contains
|
||||
# placeholder strings (YOUR_*) that may trigger gitleaks rules.
|
||||
# The generate_report.py script suppresses these false positives.
|
||||
- name: Run gitleaks
|
||||
uses: gitleaks/gitleaks-action@v2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload secrets scan artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: secrets-scan-results
|
||||
path: results.sarif
|
||||
retention-days: 30
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# LEDMatrix-Specific Security Proofs
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
ledmatrix-security-proofs:
|
||||
name: LEDMatrix Security Proofs
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements.txt || true
|
||||
|
||||
# Script exits 1 only on CRITICAL findings.
|
||||
# Warnings are reported but do not block the workflow.
|
||||
- name: Run security proofs
|
||||
run: |
|
||||
python scripts/prove_security.py \
|
||||
--output security-proofs-results.json \
|
||||
--verbose
|
||||
|
||||
- name: Upload proofs artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: security-proofs-results
|
||||
path: security-proofs-results.json
|
||||
retention-days: 30
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Plugin Security Audit
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
plugin-audit:
|
||||
name: Plugin Security Audit
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
# Script exits 1 only on CRITICAL findings (eval/exec in plugins).
|
||||
# Missing manifest.json etc are warnings.
|
||||
- name: Run plugin audit
|
||||
run: |
|
||||
python scripts/audit_plugins.py \
|
||||
--output plugin-audit-results.json \
|
||||
--verbose
|
||||
|
||||
- name: Upload plugin audit artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: plugin-audit-results
|
||||
path: plugin-audit-results.json
|
||||
retention-days: 30
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
# Aggregate Report
|
||||
# ─────────────────────────────────────────────────────────────────────────
|
||||
security-report:
|
||||
name: Security Report
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- sast
|
||||
- dependency-audit
|
||||
- secrets-scan
|
||||
- ledmatrix-security-proofs
|
||||
- plugin-audit
|
||||
if: always() # Run even if upstream jobs fail or are skipped
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: audit-artifacts/
|
||||
|
||||
- name: Generate consolidated report
|
||||
run: |
|
||||
python scripts/generate_report.py \
|
||||
--artifact-dir audit-artifacts/ \
|
||||
--output security-report.md \
|
||||
--verbose
|
||||
|
||||
- name: Upload consolidated report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: security-report
|
||||
path: security-report.md
|
||||
retention-days: 90
|
||||
|
||||
- name: Comment on PR
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const report = fs.readFileSync('security-report.md', 'utf8');
|
||||
// Use sticky comment — update existing comment rather than adding a new one each run
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
});
|
||||
const botComment = comments.find(c =>
|
||||
c.user.type === 'Bot' && c.body.includes('🔒 Security Audit')
|
||||
);
|
||||
if (botComment) {
|
||||
await github.rest.issues.updateComment({
|
||||
comment_id: botComment.id,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: report,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: report,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
plugin-safety:
|
||||
name: Plugin safety harness + unit tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt -r requirements-test.txt
|
||||
pip install RGBMatrixEmulator
|
||||
|
||||
- name: Run harness + visual rendering tests
|
||||
run: |
|
||||
pytest --no-cov \
|
||||
test/plugins/test_harness.py \
|
||||
test/plugins/test_visual_rendering.py \
|
||||
test/plugins/test_plugin_matrix.py
|
||||
@@ -0,0 +1,46 @@
|
||||
name: Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: pytest (Python ${{ matrix.python-version }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ['3.10', '3.11', '3.12']
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: false # rgbmatrix submodule not needed in EMULATOR mode
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
cache: pip
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
# Optional deps that some test modules import
|
||||
pip install scipy psutil Flask-Limiter
|
||||
|
||||
- name: Run tests
|
||||
env:
|
||||
EMULATOR: "true"
|
||||
run: |
|
||||
pytest \
|
||||
-m "not hardware and not slow" \
|
||||
--tb=short
|
||||
@@ -8,6 +8,7 @@ config/config_secrets.json
|
||||
config/config.json
|
||||
config/config.json.backup
|
||||
config/wifi_config.json
|
||||
config/uninstalled_plugins.json
|
||||
credentials.json
|
||||
token.pickle
|
||||
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# LEDMatrix
|
||||
[](LICENSE)
|
||||
[](https://discord.gg/RdrC37rEag)
|
||||
[](https://github.com/ChuckBuilds/ledmatrix)
|
||||
[](https://app.codacy.com/gh/ChuckBuilds/LEDMatrix/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade)
|
||||
|
||||
|
||||
## Welcome to LEDMatrix!
|
||||
Welcome to the LEDMatrix Project! This open-source project enables you to run an information-rich display on a Raspberry Pi connected to an LED RGB Matrix panel. Whether you want to see your calendar, weather forecasts, sports scores, stock prices, or any other information at a glance, LEDMatrix brings it all together.
|
||||
|
||||
@@ -127,10 +132,15 @@ The system supports live, recent, and upcoming game information for multiple spo
|
||||
| This project can be finnicky! RGB LED Matrix displays are not built the same or to a high-quality standard. We have seen many displays arrive dead or partially working in our discord. Please purchase from a reputable vendor. |
|
||||
|
||||
### Raspberry Pi
|
||||
- Raspberry Pi Zero's don't have enough processing power for this project and the Pi 5 is unsupported due to new GPIO output.
|
||||
- **Raspberry Pi 3B or 4 (NOT RPi 5!)**
|
||||
- Raspberry Pi Zero's don't have enough processing power for this project.
|
||||
- **Raspberry Pi 3B, 4, or 5**
|
||||
[Amazon Affiliate Link – Raspberry Pi 4 4GB RAM](https://amzn.to/4dJixuX)
|
||||
[Amazon Affiliate Link – Raspberry Pi 4 8GB RAM](https://amzn.to/4qbqY7F)
|
||||
- **Pi 5 users**: the installer automatically detects Pi 5 and builds the `rpi-rgb-led-matrix` library with RP1 support. If you previously installed on a Pi 4 and migrated the SD card, or if you see `mmap` errors in the logs, force a fresh library build:
|
||||
```bash
|
||||
sudo RPI_RGB_FORCE_REBUILD=1 ./first_time_install.sh
|
||||
```
|
||||
- Pi 5 config: leave `rp1_rio` at `0` (PIO mode, default) and set `gpio_slowdown` to `1` or `2`.
|
||||
|
||||
|
||||
### RGB Matrix Bonnet / HAT
|
||||
@@ -582,7 +592,7 @@ These settings control runtime behavior and GPIO timing:
|
||||
- **Critical setting**: Must match your Raspberry Pi model for stability
|
||||
- **Raspberry Pi 3**: Use 3
|
||||
- **Raspberry Pi 4**: Use 4
|
||||
- **Raspberry Pi 5**: Use 5 (or higher if needed)
|
||||
- **Raspberry Pi 5**: Use 1–2 in PIO mode (`rp1_rio: 0`, the default); start with `1` and increase if you see flickering
|
||||
- **Raspberry Pi Zero/1**: Use 1-2
|
||||
- Incorrect values can cause display corruption, flickering, or system instability
|
||||
- If you experience issues, try adjusting this value up or down by 1
|
||||
|
||||
|
Before Width: | Height: | Size: 48 KiB After Width: | Height: | Size: 102 KiB |
|
Before Width: | Height: | Size: 90 KiB After Width: | Height: | Size: 111 KiB |
|
Before Width: | Height: | Size: 76 KiB After Width: | Height: | Size: 96 KiB |
|
Before Width: | Height: | Size: 52 KiB After Width: | Height: | Size: 109 KiB |
|
Before Width: | Height: | Size: 43 KiB After Width: | Height: | Size: 98 KiB |
|
Before Width: | Height: | Size: 46 KiB After Width: | Height: | Size: 93 KiB |
|
Before Width: | Height: | Size: 69 KiB After Width: | Height: | Size: 120 KiB |
|
Before Width: | Height: | Size: 46 KiB After Width: | Height: | Size: 55 KiB |
|
Before Width: | Height: | Size: 77 KiB After Width: | Height: | Size: 66 KiB |
|
Before Width: | Height: | Size: 40 KiB After Width: | Height: | Size: 70 KiB |
|
Before Width: | Height: | Size: 20 KiB After Width: | Height: | Size: 58 KiB |
|
Before Width: | Height: | Size: 90 KiB After Width: | Height: | Size: 105 KiB |
|
Before Width: | Height: | Size: 27 KiB After Width: | Height: | Size: 50 KiB |
|
Before Width: | Height: | Size: 42 KiB After Width: | Height: | Size: 87 KiB |
|
Before Width: | Height: | Size: 68 KiB After Width: | Height: | Size: 25 KiB |
|
Before Width: | Height: | Size: 96 KiB After Width: | Height: | Size: 140 KiB |
|
Before Width: | Height: | Size: 91 KiB After Width: | Height: | Size: 102 KiB |
|
Before Width: | Height: | Size: 153 KiB After Width: | Height: | Size: 64 KiB |
|
Before Width: | Height: | Size: 89 KiB After Width: | Height: | Size: 91 KiB |
|
Before Width: | Height: | Size: 101 KiB After Width: | Height: | Size: 54 KiB |
|
Before Width: | Height: | Size: 55 KiB After Width: | Height: | Size: 60 KiB |
|
Before Width: | Height: | Size: 9.8 KiB After Width: | Height: | Size: 29 KiB |
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 30 KiB |
|
Before Width: | Height: | Size: 103 KiB After Width: | Height: | Size: 126 KiB |
|
Before Width: | Height: | Size: 94 KiB After Width: | Height: | Size: 54 KiB |
|
Before Width: | Height: | Size: 92 KiB After Width: | Height: | Size: 93 KiB |
|
Before Width: | Height: | Size: 59 KiB After Width: | Height: | Size: 60 KiB |
|
Before Width: | Height: | Size: 80 KiB After Width: | Height: | Size: 41 KiB |
|
Before Width: | Height: | Size: 38 KiB After Width: | Height: | Size: 77 KiB |
|
Before Width: | Height: | Size: 111 KiB After Width: | Height: | Size: 140 KiB |
|
After Width: | Height: | Size: 32 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 20 KiB |
|
After Width: | Height: | Size: 16 KiB |
@@ -0,0 +1,33 @@
|
||||
# bandit.yaml — LEDMatrix bandit configuration
|
||||
# https://bandit.readthedocs.io/en/latest/config.html
|
||||
#
|
||||
# Skips are justified by the specific codebase context documented below.
|
||||
# Do not remove skips without updating the justification comment.
|
||||
|
||||
skips:
|
||||
# B104: Binding to all interfaces (0.0.0.0)
|
||||
# Intentional — the Flask server binds 0.0.0.0 for LAN access on a Raspberry Pi.
|
||||
# This is not internet-facing and is documented in web_interface/app.py.
|
||||
- B104
|
||||
|
||||
# B603: subprocess call without shell=True
|
||||
# All subprocess.run() calls in this codebase use list arguments (confirmed by
|
||||
# grep — zero uses of shell=True in src/ or web_interface/). List args prevent
|
||||
# shell injection. See src/common/permission_utils.py for the primary usage.
|
||||
- B603
|
||||
|
||||
# B607: Starting a process with a partial executable path
|
||||
# The subprocess calls invoke system utilities (systemctl, sudo, git) by name.
|
||||
# These are fixed-list invocations, not user-controlled, and rely on PATH.
|
||||
- B607
|
||||
|
||||
exclude_dirs:
|
||||
- tests
|
||||
- test
|
||||
- venv
|
||||
- .venv
|
||||
- rpi-rgb-led-matrix-master
|
||||
# prove_security.py intentionally contains detection patterns as string literals
|
||||
# (e.g. "eval(", "exec(") to search for in other files — bandit would flag
|
||||
# these as false positives.
|
||||
- scripts/prove_security.py
|
||||
@@ -1,43 +1,43 @@
|
||||
{
|
||||
"web_display_autostart": true,
|
||||
"schedule": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"mode": "per-day",
|
||||
"start_time": "07:00",
|
||||
"end_time": "23:00",
|
||||
"days": {
|
||||
"monday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "07:00",
|
||||
"end_time": "23:00"
|
||||
},
|
||||
"tuesday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "07:00",
|
||||
"end_time": "23:00"
|
||||
},
|
||||
"wednesday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "07:00",
|
||||
"end_time": "23:00"
|
||||
},
|
||||
"thursday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "07:00",
|
||||
"end_time": "23:00"
|
||||
},
|
||||
"friday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "07:00",
|
||||
"end_time": "23:00"
|
||||
},
|
||||
"saturday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "07:00",
|
||||
"end_time": "23:00"
|
||||
},
|
||||
"sunday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "07:00",
|
||||
"end_time": "23:00"
|
||||
}
|
||||
@@ -51,46 +51,46 @@
|
||||
"end_time": "07:00",
|
||||
"days": {
|
||||
"monday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "20:00",
|
||||
"end_time": "07:00"
|
||||
},
|
||||
"tuesday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "20:00",
|
||||
"end_time": "07:00"
|
||||
},
|
||||
"wednesday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "20:00",
|
||||
"end_time": "07:00"
|
||||
},
|
||||
"thursday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "20:00",
|
||||
"end_time": "07:00"
|
||||
},
|
||||
"friday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "20:00",
|
||||
"end_time": "07:00"
|
||||
},
|
||||
"saturday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "20:00",
|
||||
"end_time": "07:00"
|
||||
},
|
||||
"sunday": {
|
||||
"enabled": true,
|
||||
"enabled": false,
|
||||
"start_time": "20:00",
|
||||
"end_time": "07:00"
|
||||
}
|
||||
}
|
||||
},
|
||||
"timezone": "America/Chicago",
|
||||
"timezone": "America/New_York",
|
||||
"location": {
|
||||
"city": "Dallas",
|
||||
"state": "Texas",
|
||||
"city": "Tampa",
|
||||
"state": "Florida",
|
||||
"country": "US"
|
||||
},
|
||||
"display": {
|
||||
@@ -115,6 +115,11 @@
|
||||
"gpio_slowdown": 3,
|
||||
"rp1_rio": 0
|
||||
},
|
||||
"double_sided": {
|
||||
"enabled": false,
|
||||
"copies": 2,
|
||||
"axis": "horizontal"
|
||||
},
|
||||
"display_durations": {},
|
||||
"use_short_date_format": true,
|
||||
"vegas_scroll": {
|
||||
|
||||
@@ -248,7 +248,6 @@ test/
|
||||
├── test_config_service.py # Config service tests
|
||||
├── test_config_validation_edge_cases.py # Config edge cases
|
||||
├── test_font_manager.py # Font manager tests
|
||||
├── test_layout_manager.py # Layout manager tests
|
||||
├── test_text_helper.py # Text helper tests
|
||||
├── test_error_handling.py # Error handling tests
|
||||
├── test_error_aggregator.py # Error aggregation tests
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# Plugin Safety Harness
|
||||
|
||||
Renders a plugin across **every declared screen (mode)** and **a spread of
|
||||
matrix sizes**, and fails if any combination crashes, draws past the panel edge,
|
||||
or — for plugins that ship golden images — drifts visually. The goal: change a
|
||||
plugin without breaking a size or screen you didn't think to test.
|
||||
|
||||
## Sizes: a sample, not a fixed list
|
||||
|
||||
There is **no fixed set of supported panel sizes** — an RGB matrix build can be
|
||||
any width/height and configuration (square, rectangle, 2×2, 4×4, 8×2, long
|
||||
strips, tall stacks). Plugins are expected to read dimensions dynamically
|
||||
(`self.display_manager.matrix.width/height`) and lay themselves out
|
||||
accordingly, so a hardcoded coordinate or unscaled font shows up as a failure
|
||||
here.
|
||||
|
||||
The harness therefore renders against a **representative sample** that spans the
|
||||
axes of variation (`DEFAULT_TEST_SIZES` in `src/plugin_system/testing/sizes.py`),
|
||||
not an authoritative list:
|
||||
|
||||
Each module is 64×32; entries are real panel-grid arrangements (cols × rows):
|
||||
|
||||
| Size | Grid | Why it's in the sample |
|
||||
|---------|------|--------------------------------------------|
|
||||
| 64×32 | 1×1 | single panel — tightest common rectangle |
|
||||
| 128×32 | 2×1 | the baseline most plugins are tuned for |
|
||||
| 64×64 | 1×2 | stacked — tall-narrow centering |
|
||||
| 128×64 | 2×2 | block — icon scaling / vertical centering |
|
||||
| 256×32 | 4×1 | long strip — wide horizontal layout |
|
||||
| 128×96 | 2×3 | tall — vertical overflow |
|
||||
| 256×128 | 4×4 | large block — both dimensions big at once |
|
||||
|
||||
**Override the sizes entirely** to test your actual hardware (or any shape):
|
||||
|
||||
```bash
|
||||
# CLI — one-off:
|
||||
python scripts/check_plugin.py --plugin clock-simple --sizes 8x16,64x64,256x32
|
||||
|
||||
# pytest — force every plugin onto your panel(s):
|
||||
LEDMATRIX_TEST_SIZES="8x16,128x128" pytest test/plugins/test_plugin_matrix.py
|
||||
|
||||
# Per-plugin — declare the shapes a plugin targets in its test/harness.json:
|
||||
# { "sizes": [[8, 16], [64, 64]] }
|
||||
```
|
||||
|
||||
Precedence: `LEDMATRIX_TEST_SIZES` env (global) → per-plugin `harness.json`
|
||||
`sizes` → the default sample. Bounds checking adapts to whatever sizes a run
|
||||
uses — the backing canvas is padded out to the **largest** panel in the run, so
|
||||
a coordinate meant for a big build is still caught when rendering a small one.
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# Functional + bounds check across all sizes/screens:
|
||||
python scripts/check_plugin.py --plugin clock-simple
|
||||
|
||||
# Every discovered plugin:
|
||||
python scripts/check_plugin.py --all
|
||||
|
||||
# Dump PNGs to eyeball each size/screen:
|
||||
python scripts/check_plugin.py --plugin ledmatrix-weather --out-dir /tmp/preview
|
||||
```
|
||||
|
||||
Exit code is non-zero if any `(plugin, size, screen)` fails. Plugins are
|
||||
discovered in `plugin-repos/` and `plugins/` (override with `--plugin-dir`).
|
||||
|
||||
## What it checks (Phase 1 — always on)
|
||||
|
||||
1. **Loads** and builds its mode list.
|
||||
2. **Renders every screen** at every size without raising. `update()` may fail
|
||||
(no network in CI) and is tolerated; a crash in `display()` is a failure —
|
||||
`display()` must handle the no-data state.
|
||||
3. **Bounds**: nothing is drawn past the right/bottom edge. Implemented by
|
||||
`BoundsCheckingDisplayManager`, which backs the declared panel with an
|
||||
oversized canvas and flags any pixels that land in the margin. (Left/top
|
||||
overflow at negative coordinates and BDF text are not flagged — golden images
|
||||
cover those.)
|
||||
|
||||
## Golden images (Phase 2 — opt-in per plugin)
|
||||
|
||||
A plugin opts in by committing reference PNGs and (usually) a small harness spec:
|
||||
|
||||
```
|
||||
plugins/<id>/test/harness.json # how to render deterministically
|
||||
plugins/<id>/test/fixtures/mock.json # optional cached data
|
||||
plugins/<id>/test/golden/<WxH>/<mode>.png
|
||||
```
|
||||
|
||||
`test/harness.json` keys (all optional):
|
||||
|
||||
```json
|
||||
{
|
||||
"config": { "timezone": "UTC" },
|
||||
"mock_data": "fixtures/mock.json",
|
||||
"freeze_time": "2025-08-01 15:25:00",
|
||||
"skip_update": false,
|
||||
"sizes": [[128, 32], [128, 64]]
|
||||
}
|
||||
```
|
||||
|
||||
Generate / refresh goldens after an intentional visual change, then review the
|
||||
diff before committing:
|
||||
|
||||
```bash
|
||||
python scripts/check_plugin.py --plugin clock-simple --update-golden \
|
||||
--config '{"timezone":"UTC"}' --freeze-time "2025-08-01 15:25:00"
|
||||
```
|
||||
|
||||
Comparison is exact by default (`compare_images` in `harness.py` accepts a
|
||||
tolerance for known anti-aliasing noise). Determinism requires a pinned Pillow
|
||||
and the bundled fonts — keep both stable when regenerating goldens.
|
||||
|
||||
## Tests & CI
|
||||
|
||||
- `test/plugins/test_harness.py` — unit tests for bounds detection, image
|
||||
comparison, and mode enumeration (run anywhere).
|
||||
- `test/plugins/test_plugin_matrix.py` — parametrized over discovered plugins ×
|
||||
sizes × screens; honors each plugin's `test/harness.json` and goldens. Skips
|
||||
when no plugins are present (e.g. a fresh core checkout); set
|
||||
`LEDMATRIX_REQUIRE_PLUGINS=1` in a pipeline where plugins must be present to
|
||||
turn an empty discovery into a hard failure instead. Point it at the monorepo
|
||||
with `LEDMATRIX_PLUGINS_DIR=/path/to/ledmatrix-plugins/plugins`.
|
||||
- `.github/workflows/test.yml` — runs the harness + visual tests on every PR.
|
||||
|
||||
The plugin monorepo has its own `Plugin Safety` workflow that runs this harness
|
||||
against changed plugins on every PR.
|
||||
|
||||
## Developer workflow
|
||||
|
||||
1. Change the plugin on a branch.
|
||||
2. `python scripts/check_plugin.py --plugin <id> --out-dir /tmp/preview` and
|
||||
eyeball the PNGs.
|
||||
3. Intentional visual change? `--update-golden`, review diffs, commit goldens.
|
||||
4. (Monorepo) bump `manifest.json` version and let the pre-commit hook sync
|
||||
`plugins.json`.
|
||||
5. Push — CI re-runs the harness across all sizes and gates the PR.
|
||||
@@ -10,6 +10,98 @@ The LEDMatrix Widget Registry system allows plugins to use reusable UI component
|
||||
|
||||
## Available Core Widgets
|
||||
|
||||
### Plugin File Manager Widget (`plugin-file-manager`)
|
||||
|
||||
Full inline file management UI for plugins that manage files via the `web_ui_actions` system. Renders a card grid, upload zone, create/delete modals, and an entry table editor — entirely inline, no iframe.
|
||||
|
||||
`plugin_id` is **automatically injected** from template context. File operations call `/api/v3/plugins/action` immediately on user action; no Save Configuration needed.
|
||||
|
||||
**Schema Configuration:**
|
||||
```json
|
||||
{
|
||||
"file_manager": {
|
||||
"type": "null",
|
||||
"title": "Data Files",
|
||||
"x-widget": "plugin-file-manager",
|
||||
"x-widget-config": {
|
||||
"actions": {
|
||||
"list": "list-files",
|
||||
"get": "get-file",
|
||||
"save": "save-file",
|
||||
"upload": "upload-file",
|
||||
"delete": "delete-file",
|
||||
"create": "create-file",
|
||||
"toggle": "toggle-category"
|
||||
},
|
||||
"upload_hint": "JSON files with day numbers 1–365 as keys",
|
||||
"directory_label": "my_data/",
|
||||
"create_fields": [
|
||||
{ "key": "category_name", "label": "Category Name",
|
||||
"placeholder": "e.g., my_words", "pattern": "^[a-z0-9_]+$",
|
||||
"hint": "Lowercase letters, numbers, underscores" },
|
||||
{ "key": "display_name", "label": "Display Name",
|
||||
"placeholder": "e.g., My Words", "hint": "Optional" }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**`list` is required** — the widget calls it on render to populate the file grid; omitting it leaves the widget stuck in a loading state. All other actions are optional — omit any key to hide its UI element (e.g., no `create` = no New File button, no `toggle` = no enable/disable switch).
|
||||
|
||||
The edit view auto-detects whether file content is tabular (object-of-objects with uniform keys) and shows a paginated table editor with inline cells. Otherwise falls back to a JSON textarea.
|
||||
|
||||
**Used by:** of-the-day
|
||||
|
||||
---
|
||||
|
||||
### Time Picker Widget (`time-picker`)
|
||||
|
||||
Single time selection using the browser's native time input. Returns a string in `HH:MM` (24-hour) format. Generic — works in any plugin without configuration.
|
||||
|
||||
**Schema Configuration:**
|
||||
```json
|
||||
{
|
||||
"target_time": {
|
||||
"type": "string",
|
||||
"x-widget": "time-picker",
|
||||
"default": "00:00",
|
||||
"x-options": {
|
||||
"placeholder": "Select time",
|
||||
"clearable": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Used by:** countdown
|
||||
|
||||
---
|
||||
|
||||
### File Upload Single Widget (`file-upload-single`)
|
||||
|
||||
Single-image upload for string fields. Uploads to the plugin's asset folder (`assets/plugins/<plugin_id>/uploads/`) and sets the string field value to the returned relative path. Shows a thumbnail preview and a clear button. The `plugin_id` is **automatically injected** from the template context — no need to specify it in the schema.
|
||||
|
||||
**Schema Configuration:**
|
||||
```json
|
||||
{
|
||||
"image_path": {
|
||||
"type": "string",
|
||||
"x-widget": "file-upload-single",
|
||||
"x-upload-config": {
|
||||
"allowed_types": ["image/png", "image/jpeg", "image/bmp", "image/gif"],
|
||||
"max_size_mb": 5
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Note: Unlike `file-upload` (array-level), this widget is for a single `string` field. It is ideal for per-item images inside `array-table` rows.
|
||||
|
||||
**Used by:** countdown
|
||||
|
||||
---
|
||||
|
||||
### File Upload Widget (`file-upload`)
|
||||
|
||||
Upload and manage image files with drag-and-drop support, preview, delete, and scheduling.
|
||||
|
||||
@@ -15,8 +15,8 @@ on_error() {
|
||||
echo "✗ An error occurred during: $CURRENT_STEP (line $line_no, exit $exit_code)" >&2
|
||||
if [ -n "${LOG_FILE:-}" ]; then
|
||||
echo "See the log for details: $LOG_FILE" >&2
|
||||
echo "-- Last 50 lines from log --" >&2
|
||||
tail -n 50 "$LOG_FILE" >&2 || true
|
||||
echo "-- Last 100 lines from log --" >&2
|
||||
tail -n 100 "$LOG_FILE" >&2 || true
|
||||
fi
|
||||
echo "\nCommon fixes:" >&2
|
||||
echo "- Ensure the Pi is online (try: ping -c1 8.8.8.8)." >&2
|
||||
@@ -36,9 +36,17 @@ if [ -r /proc/device-tree/model ]; then
|
||||
DEVICE_MODEL=$(tr -d '\0' </proc/device-tree/model)
|
||||
echo "Detected device: $DEVICE_MODEL"
|
||||
else
|
||||
DEVICE_MODEL=""
|
||||
echo "⚠ Could not detect Raspberry Pi model (continuing anyway)"
|
||||
fi
|
||||
|
||||
# Detect Pi 5 for hardware-specific install decisions (RP1 library verification)
|
||||
IS_PI5=0
|
||||
if echo "${DEVICE_MODEL:-}" | grep -qi "Raspberry Pi 5"; then
|
||||
IS_PI5=1
|
||||
echo "Raspberry Pi 5 detected — will verify RP1 library support."
|
||||
fi
|
||||
|
||||
# Check OS version - must be Raspberry Pi OS Lite (Trixie)
|
||||
echo ""
|
||||
echo "Checking operating system requirements..."
|
||||
@@ -194,8 +202,33 @@ retry() {
|
||||
done
|
||||
}
|
||||
|
||||
apt_update() { retry apt update; }
|
||||
apt_install() { retry apt install -y "$@"; }
|
||||
# Wait for another apt/dpkg process (commonly unattended-upgrades running
|
||||
# shortly after first boot) to release its lock before we try apt ourselves.
|
||||
# Without this, apt_update/apt_install can fail outright in the first couple
|
||||
# minutes after a fresh Pi OS boot with a generic "Command failed after 3
|
||||
# attempts" error.
|
||||
wait_for_apt_lock() {
|
||||
command -v flock >/dev/null 2>&1 || return 0
|
||||
local lock_file="/var/lib/dpkg/lock-frontend"
|
||||
local max_wait=180
|
||||
local waited=0
|
||||
local printed=0
|
||||
while ! flock -n "$lock_file" -c true 2>/dev/null; do
|
||||
if [ "$printed" -eq 0 ]; then
|
||||
echo "⚠ Waiting for another apt/dpkg process to finish (e.g. unattended-upgrades on first boot)..."
|
||||
printed=1
|
||||
fi
|
||||
if [ "$waited" -ge "$max_wait" ]; then
|
||||
echo "⚠ Still waiting after ${max_wait}s; proceeding anyway."
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
waited=$((waited+5))
|
||||
done
|
||||
}
|
||||
|
||||
apt_update() { wait_for_apt_lock; retry apt-get -o DPkg::Lock::Timeout=180 update; }
|
||||
apt_install() { wait_for_apt_lock; retry apt-get -o DPkg::Lock::Timeout=180 install -y "$@"; }
|
||||
apt_remove() { apt-get remove -y "$@" || true; }
|
||||
|
||||
check_network() {
|
||||
@@ -214,6 +247,22 @@ check_network() {
|
||||
exit 1
|
||||
}
|
||||
|
||||
check_disk_space() {
|
||||
command -v df >/dev/null 2>&1 || return 0
|
||||
local available_mb
|
||||
available_mb=$(df -m "$PROJECT_ROOT_DIR" | awk 'NR==2{print $4}')
|
||||
available_mb=${available_mb:-0}
|
||||
if [ "$available_mb" -lt 500 ]; then
|
||||
echo "✗ ERROR: Insufficient disk space: ${available_mb}MB available (need at least 500MB)"
|
||||
echo " Free up space first, e.g.: sudo apt clean && sudo apt autoremove"
|
||||
exit 1
|
||||
elif [ "$available_mb" -lt 1024 ]; then
|
||||
echo "⚠ Limited disk space: ${available_mb}MB available (recommend at least 1GB for the rpi-rgb-led-matrix build in Step 6)"
|
||||
else
|
||||
echo "✓ Disk space sufficient: ${available_mb}MB available"
|
||||
fi
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "This script will perform the following steps:"
|
||||
echo "1. Install system dependencies"
|
||||
@@ -263,8 +312,9 @@ CURRENT_STEP="Install system dependencies"
|
||||
echo "Step 1: Installing system dependencies..."
|
||||
echo "----------------------------------------"
|
||||
|
||||
# Ensure network is available before APT operations
|
||||
# Pre-flight checks before APT operations
|
||||
check_network
|
||||
check_disk_space
|
||||
|
||||
# Update package list
|
||||
apt_update
|
||||
@@ -676,7 +726,11 @@ if [ -f "$PROJECT_ROOT_DIR/requirements.txt" ]; then
|
||||
|
||||
if command -v timeout >/dev/null 2>&1; then
|
||||
# Use timeout if available (10 minutes = 600 seconds)
|
||||
if timeout 600 python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --verbose "$line" > "$INSTALL_OUTPUT" 2>&1; then
|
||||
# --ignore-installed: apt-managed packages (e.g. python3-requests)
|
||||
# ship no pip RECORD file, so upgrading them would otherwise abort
|
||||
# with "uninstall-no-record-file"; this lays the new version down
|
||||
# alongside instead of trying to uninstall the apt copy first.
|
||||
if timeout 600 python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --ignore-installed --verbose "$line" > "$INSTALL_OUTPUT" 2>&1; then
|
||||
INSTALL_SUCCESS=true
|
||||
else
|
||||
EXIT_CODE=$?
|
||||
@@ -684,7 +738,7 @@ if [ -f "$PROJECT_ROOT_DIR/requirements.txt" ]; then
|
||||
echo "✗ Timeout (10 minutes) installing: $line"
|
||||
echo " This package may require building from source, which can be slow on Raspberry Pi."
|
||||
echo " You can try installing it manually later with:"
|
||||
echo " python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --verbose '$line'"
|
||||
echo " python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --ignore-installed --verbose '$line'"
|
||||
else
|
||||
echo "✗ Failed to install: $line (exit code: $EXIT_CODE)"
|
||||
fi
|
||||
@@ -692,7 +746,7 @@ if [ -f "$PROJECT_ROOT_DIR/requirements.txt" ]; then
|
||||
else
|
||||
# No timeout command available, install without timeout
|
||||
echo " Note: timeout command not available, installation may take a while..."
|
||||
if python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --verbose "$line" > "$INSTALL_OUTPUT" 2>&1; then
|
||||
if python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --ignore-installed --verbose "$line" > "$INSTALL_OUTPUT" 2>&1; then
|
||||
INSTALL_SUCCESS=true
|
||||
else
|
||||
EXIT_CODE=$?
|
||||
@@ -744,7 +798,7 @@ if [ -f "$PROJECT_ROOT_DIR/requirements.txt" ]; then
|
||||
echo " 1. Ensure you have enough disk space: df -h"
|
||||
echo " 2. Check available memory: free -h"
|
||||
echo " 3. Try installing failed packages individually with verbose output:"
|
||||
echo " python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --verbose <package>"
|
||||
echo " python3 -m pip install --break-system-packages --no-cache-dir --prefer-binary --ignore-installed --verbose <package>"
|
||||
echo " 4. For packages that build from source (like numpy), consider:"
|
||||
echo " - Installing pre-built wheels: python3 -m pip install --only-binary :all: <package>"
|
||||
echo " - Or installing via apt if available: sudo apt install python3-<package>"
|
||||
@@ -766,7 +820,10 @@ echo ""
|
||||
# Install web interface dependencies
|
||||
echo "Installing web interface dependencies..."
|
||||
if [ -f "$PROJECT_ROOT_DIR/web_interface/requirements.txt" ]; then
|
||||
if python3 -m pip install --break-system-packages --prefer-binary -r "$PROJECT_ROOT_DIR/web_interface/requirements.txt"; then
|
||||
# --ignore-installed: apt-managed packages (e.g. python3-requests) ship no
|
||||
# pip RECORD file, so upgrading them to the version pinned here would
|
||||
# otherwise abort the whole install with "uninstall-no-record-file".
|
||||
if python3 -m pip install --break-system-packages --prefer-binary --ignore-installed -r "$PROJECT_ROOT_DIR/web_interface/requirements.txt"; then
|
||||
echo "✓ Web interface dependencies installed"
|
||||
# Create marker file to indicate dependencies are installed
|
||||
touch "$PROJECT_ROOT_DIR/.web_deps_installed"
|
||||
@@ -783,29 +840,54 @@ CURRENT_STEP="Build and install rpi-rgb-led-matrix"
|
||||
echo "Step 6: Building and installing rpi-rgb-led-matrix..."
|
||||
echo "-----------------------------------------------------"
|
||||
|
||||
# If already installed and not forcing rebuild, skip expensive build
|
||||
# On Pi 5, also check that the installed library has rp1_rio support.
|
||||
# A library built before Pi 5 support was added imports fine but maps to the
|
||||
# Pi 3 peripheral bus address (0x3f000000) instead of the RP1 chip at runtime.
|
||||
_HAS_RP1=0
|
||||
if python3 -c 'from rgbmatrix import RGBMatrixOptions; assert hasattr(RGBMatrixOptions(), "rp1_rio")' >/dev/null 2>&1; then
|
||||
_HAS_RP1=1
|
||||
fi
|
||||
|
||||
_SKIP_BUILD=0
|
||||
if python3 -c 'from rgbmatrix import RGBMatrix, RGBMatrixOptions' >/dev/null 2>&1 && [ "${RPI_RGB_FORCE_REBUILD:-0}" != "1" ]; then
|
||||
echo "rgbmatrix Python package already available; skipping build (set RPI_RGB_FORCE_REBUILD=1 to force rebuild)."
|
||||
if [ "$IS_PI5" = "1" ] && [ "$_HAS_RP1" = "0" ]; then
|
||||
echo "⚠ Pi 5 detected: installed rgbmatrix lacks rp1_rio support (older build)."
|
||||
echo " Forcing rebuild to get Pi 5 RP1 support..."
|
||||
else
|
||||
_SKIP_BUILD=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$_SKIP_BUILD" = "1" ]; then
|
||||
_skip_suffix=""
|
||||
if [ "$IS_PI5" = "1" ]; then _skip_suffix=" with Pi 5 RP1 support"; fi
|
||||
echo "rgbmatrix already installed${_skip_suffix}; skipping build (set RPI_RGB_FORCE_REBUILD=1 to force rebuild)."
|
||||
else
|
||||
# Ensure rpi-rgb-led-matrix submodule is initialized
|
||||
# Wrapper used with retry(): removes any partial clone dir before each attempt
|
||||
# so git clone doesn't fail with "destination path already exists".
|
||||
_clone_rpi_rgb() {
|
||||
rm -rf "$PROJECT_ROOT_DIR/rpi-rgb-led-matrix-master"
|
||||
git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master
|
||||
}
|
||||
if [ ! -d "$PROJECT_ROOT_DIR/rpi-rgb-led-matrix-master" ]; then
|
||||
echo "rpi-rgb-led-matrix-master not found. Initializing git submodule..."
|
||||
cd "$PROJECT_ROOT_DIR"
|
||||
|
||||
|
||||
# Try to initialize submodule if .gitmodules exists
|
||||
if [ -f "$PROJECT_ROOT_DIR/.gitmodules" ] && grep -q "rpi-rgb-led-matrix" "$PROJECT_ROOT_DIR/.gitmodules"; then
|
||||
echo "Initializing rpi-rgb-led-matrix submodule..."
|
||||
if ! git submodule update --init --recursive rpi-rgb-led-matrix-master 2>&1; then
|
||||
if ! retry git submodule update --init --recursive rpi-rgb-led-matrix-master; then
|
||||
echo "⚠ Submodule init failed, cloning directly from GitHub..."
|
||||
git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master
|
||||
retry _clone_rpi_rgb
|
||||
fi
|
||||
else
|
||||
# Fallback: clone directly if submodule not configured
|
||||
echo "Submodule not configured, cloning directly from GitHub..."
|
||||
git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master
|
||||
retry _clone_rpi_rgb
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
# Build and install rpi-rgb-led-matrix Python bindings
|
||||
if [ -d "$PROJECT_ROOT_DIR/rpi-rgb-led-matrix-master" ]; then
|
||||
# Check if submodule is properly initialized (not empty)
|
||||
@@ -814,23 +896,34 @@ else
|
||||
cd "$PROJECT_ROOT_DIR"
|
||||
rm -rf rpi-rgb-led-matrix-master
|
||||
if [ -f "$PROJECT_ROOT_DIR/.gitmodules" ] && grep -q "rpi-rgb-led-matrix" "$PROJECT_ROOT_DIR/.gitmodules"; then
|
||||
git submodule update --init --recursive rpi-rgb-led-matrix-master
|
||||
retry git submodule update --init --recursive rpi-rgb-led-matrix-master
|
||||
else
|
||||
git clone https://github.com/hzeller/rpi-rgb-led-matrix.git rpi-rgb-led-matrix-master
|
||||
retry _clone_rpi_rgb
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
pushd "$PROJECT_ROOT_DIR/rpi-rgb-led-matrix-master" >/dev/null
|
||||
echo "Installing rpi-rgb-led-matrix Python package (scikit-build-core + cmake)..."
|
||||
echo " Build deps required: python-dev-is-python3 cmake"
|
||||
echo " This compiles C++ — may take 2-5 minutes on Pi 4/5..."
|
||||
if ! python3 -m pip install --break-system-packages .; then
|
||||
BUILD_OUTPUT=$(mktemp)
|
||||
BUILD_SUCCESS=false
|
||||
if python3 -m pip install --break-system-packages . > "$BUILD_OUTPUT" 2>&1; then
|
||||
BUILD_SUCCESS=true
|
||||
fi
|
||||
cat "$BUILD_OUTPUT" >> "$LOG_FILE"
|
||||
if [ "$BUILD_SUCCESS" != true ]; then
|
||||
echo "✗ Failed to install rpi-rgb-led-matrix Python package"
|
||||
echo " Ensure build tools are installed:"
|
||||
echo " sudo apt install -y python-dev-is-python3 cmake build-essential"
|
||||
echo ""
|
||||
echo "-- Last 50 lines of build output --"
|
||||
tail -n 50 "$BUILD_OUTPUT"
|
||||
rm -f "$BUILD_OUTPUT"
|
||||
popd >/dev/null
|
||||
exit 1
|
||||
fi
|
||||
rm -f "$BUILD_OUTPUT"
|
||||
popd >/dev/null
|
||||
else
|
||||
echo "✗ rpi-rgb-led-matrix-master directory not found at $PROJECT_ROOT_DIR"
|
||||
@@ -852,6 +945,17 @@ except Exception as e:
|
||||
PY
|
||||
then
|
||||
echo "✓ rpi-rgb-led-matrix installed and verified"
|
||||
# Pi 5: confirm the freshly-built library has rp1_rio support
|
||||
if [ "$IS_PI5" = "1" ]; then
|
||||
if python3 -c 'from rgbmatrix import RGBMatrixOptions; assert hasattr(RGBMatrixOptions(), "rp1_rio")' >/dev/null 2>&1; then
|
||||
echo "✓ Pi 5 RP1 (rp1_rio) support confirmed"
|
||||
else
|
||||
echo "⚠ rp1_rio not found after rebuild — the submodule may be an older version."
|
||||
echo " Try updating the submodule and rebuilding:"
|
||||
echo " git submodule update --remote rpi-rgb-led-matrix-master"
|
||||
echo " sudo RPI_RGB_FORCE_REBUILD=1 ./first_time_install.sh"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "✗ rpi-rgb-led-matrix import test failed"
|
||||
exit 1
|
||||
@@ -874,11 +978,15 @@ else
|
||||
# Try to install dependencies using the smart installer if available
|
||||
if [ -f "$PROJECT_ROOT_DIR/scripts/install_dependencies_apt.py" ]; then
|
||||
echo "Using smart dependency installer..."
|
||||
python3 "$PROJECT_ROOT_DIR/scripts/install_dependencies_apt.py"
|
||||
# -u: unbuffered stdout/stderr so output is captured in $LOG_FILE in
|
||||
# real time and in order relative to this script's own echo statements
|
||||
python3 -u "$PROJECT_ROOT_DIR/scripts/install_dependencies_apt.py"
|
||||
else
|
||||
echo "Using pip to install dependencies..."
|
||||
if [ -f "$PROJECT_ROOT_DIR/requirements_web_v2.txt" ]; then
|
||||
python3 -m pip install --break-system-packages --prefer-binary -r requirements_web_v2.txt
|
||||
# --ignore-installed: see the Step 5 web_interface/requirements.txt
|
||||
# install above — same apt/pip RECORD-file conflict applies here.
|
||||
python3 -m pip install --break-system-packages --prefer-binary --ignore-installed -r requirements_web_v2.txt
|
||||
else
|
||||
echo "⚠ requirements_web_v2.txt not found; skipping web dependency install"
|
||||
fi
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/7-segment-clock
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/baseball-scoreboard
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/basketball-scoreboard
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/calendar
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/christmas-countdown
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/clock-simple
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/countdown
|
||||
@@ -0,0 +1 @@
|
||||
/home/chuck/Github/ledmatrix-plugins/plugins/f1-scoreboard
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/football-scoreboard
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/hello-world
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/hockey-scoreboard
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/ledmatrix-flights
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/ledmatrix-leaderboard
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/ledmatrix-music
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/ledmatrix-stocks
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/ledmatrix-weather
|
||||
|
After Width: | Height: | Size: 476 B |
|
After Width: | Height: | Size: 459 B |
|
After Width: | Height: | Size: 545 B |
|
After Width: | Height: | Size: 496 B |
|
After Width: | Height: | Size: 561 B |
|
After Width: | Height: | Size: 538 B |
|
After Width: | Height: | Size: 521 B |
@@ -197,7 +197,7 @@ class MarchMadnessPlugin(BasePlugin):
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _load_round_logos(self) -> None:
|
||||
logo_dir = Path("assets/sports/ncaa_logos")
|
||||
logo_dir = Path(__file__).parent / "assets" / "logos"
|
||||
for round_key, filename in ROUND_LOGO_FILES.items():
|
||||
path = logo_dir / filename
|
||||
try:
|
||||
@@ -212,7 +212,7 @@ class MarchMadnessPlugin(BasePlugin):
|
||||
except Exception:
|
||||
self.logger.exception(f"Unexpected error loading round logo {filename}")
|
||||
|
||||
# March Madness logo
|
||||
# March Madness banner logo (also in plugin assets)
|
||||
mm_path = logo_dir / "MARCH_MADNESS.png"
|
||||
try:
|
||||
img = Image.open(mm_path).convert("RGBA")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
requests>=2.33.0
|
||||
urllib3>=2.2.2
|
||||
urllib3>=2.6.3
|
||||
Pillow>=12.2.0
|
||||
pytz>=2022.1
|
||||
numpy>=1.24.0
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/mqtt-notifications
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/news
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/odds-ticker
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/of-the-day
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/olympics
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/soccer-scoreboard
|
||||
@@ -22,5 +22,6 @@
|
||||
"Pillow>=10.0.0",
|
||||
"PyYAML>=6.0",
|
||||
"requests>=2.31.0"
|
||||
]
|
||||
],
|
||||
"local_only": true
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/static-image
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/stock-news
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/text-display
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/ufc-scoreboard
|
||||
@@ -0,0 +1 @@
|
||||
../../ledmatrix-plugins/plugins/youtube-stats
|
||||
@@ -0,0 +1,9 @@
|
||||
# Test-only dependencies for the plugin safety harness and pytest suite.
|
||||
# Install alongside requirements.txt: pip install -r requirements.txt -r requirements-test.txt
|
||||
#
|
||||
# pytest, pytest-cov, pytest-mock, and jsonschema are already pinned (with
|
||||
# major-version caps) in requirements.txt, so they are intentionally NOT
|
||||
# repeated here — re-pinning pytest to <9 collided with requirements.txt's
|
||||
# pytest>=9.0.3,<10 and made the two files impossible to install together.
|
||||
# Only declare what requirements.txt doesn't already provide.
|
||||
freezegun>=1.2,<2 # deterministic time for golden-image tests
|
||||
@@ -8,16 +8,11 @@ numpy>=1.24.0 # For fast array operations in ScrollHelper (compatible with 2.x)
|
||||
|
||||
# Timezone handling
|
||||
pytz>=2024.2,<2025.0 # Updated for latest timezone data
|
||||
timezonefinder>=6.5.0,<7.0.0 # Updated for better performance and accuracy
|
||||
geopy>=2.4.1,<3.0.0
|
||||
|
||||
# HTTP requests
|
||||
requests>=2.33.0,<3.0.0
|
||||
|
||||
# Google API integration
|
||||
google-auth-oauthlib>=1.2.0,<2.0.0
|
||||
google-auth-httplib2>=0.2.0,<1.0.0
|
||||
google-api-python-client>=2.147.0,<3.0.0
|
||||
|
||||
# Font rendering
|
||||
freetype-py>=2.5.1,<3.0.0
|
||||
@@ -29,10 +24,8 @@ spotipy>=2.25.2,<3.0.0
|
||||
Flask>=3.1.3,<4.0.0
|
||||
|
||||
# Text processing
|
||||
unidecode>=1.3.8,<2.0.0
|
||||
|
||||
# Calendar integration
|
||||
icalevents>=0.1.27,<1.0.0
|
||||
|
||||
# WebSocket support
|
||||
python-socketio>=5.14.0,<6.0.0
|
||||
@@ -43,6 +36,9 @@ websocket-client>=1.8.0,<2.0.0
|
||||
# JSON Schema validation
|
||||
jsonschema>=4.20.0,<5.0.0
|
||||
|
||||
# Requirement specifier parsing (plugin dependency satisfaction checks)
|
||||
packaging>=23.0,<27.0
|
||||
|
||||
# Testing dependencies
|
||||
pytest>=9.0.3,<10.0.0
|
||||
pytest-cov>=4.1.0,<5.0.0
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
LEDMatrix Plugin Security Auditor
|
||||
|
||||
Performs AST-based security analysis of all Python files in plugin directories.
|
||||
Designed to run in CI — exits non-zero on CRITICAL findings only.
|
||||
|
||||
Usage:
|
||||
python scripts/audit_plugins.py
|
||||
python scripts/audit_plugins.py --verbose
|
||||
python scripts/audit_plugins.py --plugin hello-world
|
||||
python scripts/audit_plugins.py --output results.json
|
||||
"""
|
||||
|
||||
import ast
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass, asdict
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
PLUGIN_BASE_DIRS = [
|
||||
PROJECT_ROOT / "plugins",
|
||||
PROJECT_ROOT / "plugin-repos",
|
||||
]
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Finding dataclass
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class Finding:
|
||||
plugin_id: str
|
||||
file: str
|
||||
line: int
|
||||
severity: str # CRITICAL | WARNING | INFO
|
||||
rule: str
|
||||
message: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# AST visitor
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
class _PluginVisitor(ast.NodeVisitor):
|
||||
"""Collect security findings from a single plugin Python file."""
|
||||
|
||||
def __init__(self, filepath: Path, plugin_id: str):
|
||||
self.filepath = filepath
|
||||
self.plugin_id = plugin_id
|
||||
self.findings: list[Finding] = []
|
||||
|
||||
def _add(self, node: ast.AST, severity: str, rule: str, message: str) -> None:
|
||||
self.findings.append(Finding(
|
||||
plugin_id=self.plugin_id,
|
||||
file=str(self.filepath.relative_to(PROJECT_ROOT)),
|
||||
line=getattr(node, "lineno", 0),
|
||||
severity=severity,
|
||||
rule=rule,
|
||||
message=message,
|
||||
))
|
||||
|
||||
def visit_Call(self, node: ast.Call) -> None:
|
||||
# eval() / exec() — arbitrary code execution
|
||||
if isinstance(node.func, ast.Name):
|
||||
if node.func.id == "eval":
|
||||
self._add(node, "CRITICAL", "PLUGIN-001",
|
||||
"eval() call — arbitrary code execution risk")
|
||||
elif node.func.id == "exec":
|
||||
self._add(node, "CRITICAL", "PLUGIN-002",
|
||||
"exec() call — arbitrary code execution risk")
|
||||
elif node.func.id == "compile":
|
||||
self._add(node, "WARNING", "PLUGIN-003",
|
||||
"compile() call — dynamic code compilation")
|
||||
|
||||
# subprocess.*(shell=True)
|
||||
if isinstance(node.func, ast.Attribute):
|
||||
is_subprocess = (
|
||||
isinstance(node.func.value, ast.Name) and
|
||||
node.func.value.id == "subprocess" and
|
||||
node.func.attr in ("run", "call", "Popen", "check_call", "check_output")
|
||||
)
|
||||
if is_subprocess:
|
||||
for kw in node.keywords:
|
||||
if (kw.arg == "shell" and
|
||||
isinstance(kw.value, ast.Constant) and
|
||||
kw.value.value is True):
|
||||
self._add(node, "WARNING", "PLUGIN-004",
|
||||
f"subprocess.{node.func.attr}(shell=True) — "
|
||||
f"shell injection risk if args include user input")
|
||||
|
||||
# os.system() — shell execution
|
||||
is_os_system = (
|
||||
isinstance(node.func.value, ast.Name) and
|
||||
node.func.value.id == "os" and
|
||||
node.func.attr == "system"
|
||||
)
|
||||
if is_os_system:
|
||||
self._add(node, "WARNING", "PLUGIN-005",
|
||||
"os.system() call — prefer subprocess with list args")
|
||||
|
||||
self.generic_visit(node)
|
||||
|
||||
def visit_Import(self, node: ast.Import) -> None:
|
||||
for alias in node.names:
|
||||
self._check_import(node, alias.name)
|
||||
self.generic_visit(node)
|
||||
|
||||
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
||||
if node.module:
|
||||
self._check_import(node, node.module)
|
||||
self.generic_visit(node)
|
||||
|
||||
def _check_import(self, node: ast.AST, module_name: str) -> None:
|
||||
dangerous = {
|
||||
"ctypes": ("WARNING", "PLUGIN-010", "ctypes import — native code execution"),
|
||||
"cffi": ("WARNING", "PLUGIN-011", "cffi import — native code execution"),
|
||||
"pickle": ("WARNING", "PLUGIN-012",
|
||||
"pickle import — deserialization can execute arbitrary code"),
|
||||
"marshal": ("WARNING", "PLUGIN-013",
|
||||
"marshal import — deserialization risk"),
|
||||
}
|
||||
for mod, (severity, rule, msg) in dangerous.items():
|
||||
if module_name == mod or module_name.startswith(mod + "."):
|
||||
self._add(node, severity, rule, msg)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Per-plugin audit
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def audit_plugin(plugin_dir: Path) -> list[Finding]:
|
||||
"""Audit a single plugin directory. Returns all findings."""
|
||||
findings: list[Finding] = []
|
||||
plugin_id = plugin_dir.name
|
||||
|
||||
# Check for required files
|
||||
for required_file, rule, msg in [
|
||||
("manifest.json", "PLUGIN-020",
|
||||
"manifest.json missing — plugin may be incomplete"),
|
||||
("config_schema.json", "PLUGIN-021",
|
||||
"config_schema.json missing — no input validation schema declared"),
|
||||
]:
|
||||
if not (plugin_dir / required_file).exists():
|
||||
findings.append(Finding(
|
||||
plugin_id=plugin_id,
|
||||
file=str((plugin_dir / required_file).relative_to(PROJECT_ROOT)),
|
||||
line=0,
|
||||
severity="WARNING",
|
||||
rule=rule,
|
||||
message=msg,
|
||||
))
|
||||
|
||||
# AST analysis of all Python files
|
||||
for py_file in sorted(plugin_dir.rglob("*.py")):
|
||||
try:
|
||||
source = py_file.read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=str(py_file))
|
||||
visitor = _PluginVisitor(py_file, plugin_id)
|
||||
visitor.visit(tree)
|
||||
findings.extend(visitor.findings)
|
||||
except SyntaxError as exc:
|
||||
findings.append(Finding(
|
||||
plugin_id=plugin_id,
|
||||
file=str(py_file.relative_to(PROJECT_ROOT)),
|
||||
line=getattr(exc, "lineno", 0) or 0,
|
||||
severity="WARNING",
|
||||
rule="PLUGIN-030",
|
||||
message=f"Python syntax error — cannot be parsed: {exc}",
|
||||
))
|
||||
except OSError as exc:
|
||||
findings.append(Finding(
|
||||
plugin_id=plugin_id,
|
||||
file=str(py_file.relative_to(PROJECT_ROOT)),
|
||||
line=0,
|
||||
severity="INFO",
|
||||
rule="PLUGIN-031",
|
||||
message=f"Could not read file: {exc}",
|
||||
))
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Main
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="LEDMatrix plugin security auditor",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--plugin", "-p", default=None,
|
||||
help="Audit a specific plugin ID only")
|
||||
parser.add_argument("--output", "-o", default=None,
|
||||
help="Write JSON results to this file")
|
||||
parser.add_argument("--verbose", "-v", action="store_true",
|
||||
help="Show all findings, not just summary")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("=" * 60)
|
||||
print("LEDMatrix Plugin Security Audit")
|
||||
print(f"Project root: {PROJECT_ROOT}")
|
||||
print("=" * 60)
|
||||
|
||||
all_findings: list[Finding] = []
|
||||
plugins_scanned = 0
|
||||
|
||||
for base_dir in PLUGIN_BASE_DIRS:
|
||||
if not base_dir.exists():
|
||||
if args.verbose:
|
||||
print(f" ⏭️ Skipping {base_dir.name}/ (directory not found)")
|
||||
continue
|
||||
|
||||
base_label = base_dir.relative_to(PROJECT_ROOT)
|
||||
print(f"\n Scanning {base_label}/")
|
||||
|
||||
for plugin_dir in sorted(base_dir.iterdir()):
|
||||
if not plugin_dir.is_dir():
|
||||
continue
|
||||
if plugin_dir.name.startswith((".", "_")):
|
||||
continue
|
||||
if args.plugin and plugin_dir.name != args.plugin:
|
||||
continue
|
||||
|
||||
findings = audit_plugin(plugin_dir)
|
||||
all_findings.extend(findings)
|
||||
plugins_scanned += 1
|
||||
|
||||
critical = [f for f in findings if f.severity == "CRITICAL"]
|
||||
warnings = [f for f in findings if f.severity == "WARNING"]
|
||||
|
||||
if critical:
|
||||
icon, label = "🚨", "CRITICAL"
|
||||
elif warnings:
|
||||
icon, label = "⚠️ ", "WARN "
|
||||
else:
|
||||
icon, label = "✅", "PASS "
|
||||
|
||||
print(f" {icon} [{label}] {plugin_dir.name}"
|
||||
f" — {len(critical)} critical, {len(warnings)} warnings")
|
||||
|
||||
if args.verbose:
|
||||
for f in findings:
|
||||
severity_icon = {"CRITICAL": "🚨", "WARNING": "⚠️ ", "INFO": "ℹ️ "}.get(
|
||||
f.severity, " "
|
||||
)
|
||||
print(f" {severity_icon} {f.rule} {f.file}:{f.line} — {f.message}")
|
||||
|
||||
# Summary
|
||||
critical_findings = [f for f in all_findings if f.severity == "CRITICAL"]
|
||||
warning_findings = [f for f in all_findings if f.severity == "WARNING"]
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f" Plugins scanned : {plugins_scanned}")
|
||||
print(f" CRITICAL : {len(critical_findings)}")
|
||||
print(f" WARNING : {len(warning_findings)}")
|
||||
|
||||
if critical_findings:
|
||||
print("\n 🚨 CRITICAL findings:")
|
||||
for f in critical_findings:
|
||||
print(f" {f.plugin_id} | {Path(f.file).name}:{f.line} | {f.message}")
|
||||
|
||||
# Write JSON output
|
||||
if args.output:
|
||||
output_data = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"plugins_scanned": plugins_scanned,
|
||||
"summary": {
|
||||
"critical": len(critical_findings),
|
||||
"warnings": len(warning_findings),
|
||||
},
|
||||
"findings": [f.to_dict() for f in all_findings],
|
||||
}
|
||||
Path(args.output).write_text(
|
||||
json.dumps(output_data, indent=2), encoding="utf-8"
|
||||
)
|
||||
print(f"\n Results written to: {args.output}")
|
||||
|
||||
if critical_findings:
|
||||
print("\n 🚨 Blocking — CRITICAL issues must be resolved")
|
||||
return 1
|
||||
|
||||
print("\n ✅ No critical issues found")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Plugin safety checker.
|
||||
|
||||
Renders a plugin across every declared screen (mode) and every supported matrix
|
||||
size, and fails if any screen crashes, overflows the panel, or (for plugins with
|
||||
committed golden images) drifts visually.
|
||||
|
||||
Usage:
|
||||
# Functional + bounds check across all sizes/modes:
|
||||
python scripts/check_plugin.py --plugin clock-simple
|
||||
|
||||
# Every discovered plugin:
|
||||
python scripts/check_plugin.py --all
|
||||
|
||||
# Dump PNGs for each size/mode so you can eyeball them:
|
||||
python scripts/check_plugin.py --plugin ledmatrix-weather --out-dir /tmp/preview
|
||||
|
||||
# Refresh committed golden images after an intentional visual change:
|
||||
python scripts/check_plugin.py --plugin clock-simple --update-golden \
|
||||
--mock-data plugins/clock-simple/test/fixtures/mock.json
|
||||
|
||||
Exit code is non-zero if any (plugin, size, mode) fails.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
os.environ['EMULATOR'] = 'true'
|
||||
|
||||
from src.logging_config import get_logger # noqa: E402
|
||||
from src.plugin_system.testing.loading import ( # noqa: E402
|
||||
find_plugin_dir, load_config_defaults, load_harness_spec,
|
||||
)
|
||||
from src.plugin_system.testing.harness import ( # noqa: E402
|
||||
RenderResult, render_plugin_matrix, compare_to_goldens, write_goldens,
|
||||
)
|
||||
from src.plugin_system.testing.sizes import ( # noqa: E402
|
||||
parse_size_token, resolve_test_sizes, safe_mode_filename, size_label,
|
||||
)
|
||||
|
||||
logger = get_logger("[Check Plugin]")
|
||||
|
||||
DEFAULT_SEARCH_DIRS = [
|
||||
str(PROJECT_ROOT / 'plugins'),
|
||||
str(PROJECT_ROOT / 'plugin-repos'),
|
||||
]
|
||||
|
||||
|
||||
def discover_plugins(search_dirs: List[str]) -> List[str]:
|
||||
"""All plugin ids found across the search dirs (dirs containing manifest.json)."""
|
||||
found = []
|
||||
for d in search_dirs:
|
||||
base = Path(d)
|
||||
if not base.exists():
|
||||
continue
|
||||
for child in sorted(base.iterdir()):
|
||||
if (child / 'manifest.json').exists() and child.name not in found:
|
||||
found.append(child.name)
|
||||
return found
|
||||
|
||||
|
||||
def parse_sizes(spec: Optional[str]):
|
||||
if not spec:
|
||||
return None
|
||||
sizes = []
|
||||
for token in spec.split(','):
|
||||
if not token.strip():
|
||||
continue
|
||||
try:
|
||||
sizes.append(parse_size_token(token))
|
||||
except ValueError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
return sizes
|
||||
|
||||
|
||||
def check_one(plugin_id: str, search_dirs: List[str], sizes, mock_data: Dict,
|
||||
config: Dict, run_update: bool, out_dir: Optional[Path],
|
||||
update_golden: bool, golden_dir_override: Optional[Path],
|
||||
freeze_time: Optional[str]) -> List[RenderResult]:
|
||||
plugin_dir = find_plugin_dir(plugin_id, search_dirs)
|
||||
if not plugin_dir:
|
||||
logger.error("Plugin '%s' not found in: %s", plugin_id, search_dirs)
|
||||
return [RenderResult(plugin_id, 0, 0, "<not-found>", error="plugin directory not found")]
|
||||
|
||||
# Per-plugin test/harness.json holds the deterministic settings the committed
|
||||
# goldens were generated with (config, mock data, frozen time, sizes). Load
|
||||
# them so the CLI/CI render reproduces the golden the same way the pytest
|
||||
# matrix path does; explicit CLI flags still override the file.
|
||||
spec = load_harness_spec(plugin_dir)
|
||||
|
||||
# config_schema defaults (real-install behavior), then harness.json config,
|
||||
# then CLI --config — most specific wins.
|
||||
full_config = {"enabled": True}
|
||||
full_config.update(load_config_defaults(plugin_dir))
|
||||
full_config.update(spec.get("config", {}))
|
||||
full_config.update(config)
|
||||
|
||||
# Precedence: CLI flag > LEDMATRIX_TEST_SIZES env > harness.json > default.
|
||||
effective_sizes = sizes if sizes else resolve_test_sizes(spec.get("sizes"))
|
||||
# CLI value wins when provided, else fall back to the harness.json setting.
|
||||
effective_mock_data = mock_data or spec.get("mock_data_contents", {})
|
||||
effective_freeze = freeze_time or spec.get("freeze_time")
|
||||
effective_run_update = run_update and not spec.get("skip_update", False)
|
||||
|
||||
results = render_plugin_matrix(
|
||||
plugin_id=plugin_id, plugin_dir=plugin_dir, config=full_config,
|
||||
mock_data=effective_mock_data, sizes=effective_sizes,
|
||||
run_update=effective_run_update, freeze_time=effective_freeze,
|
||||
)
|
||||
|
||||
golden_dir = golden_dir_override or (plugin_dir / 'test' / 'golden')
|
||||
if update_golden:
|
||||
written = write_goldens(results, golden_dir)
|
||||
logger.info("Wrote %d golden image(s) for %s to %s", written, plugin_id, golden_dir)
|
||||
else:
|
||||
compare_to_goldens(results, golden_dir)
|
||||
|
||||
if out_dir:
|
||||
for r in results:
|
||||
if r.image is None:
|
||||
continue
|
||||
dest = out_dir / plugin_id / size_label(r.width, r.height)
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
r.image.save(dest / f"{safe_mode_filename(r.mode)}.png", format="PNG")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def print_report(all_results: Dict[str, List[RenderResult]]) -> bool:
|
||||
"""Print a per-plugin grid. Returns True if everything passed."""
|
||||
everything_ok = True
|
||||
for plugin_id, results in all_results.items():
|
||||
print(f"\n=== {plugin_id} ===")
|
||||
for r in results:
|
||||
if r.ok:
|
||||
status = "PASS"
|
||||
detail = ""
|
||||
if r.golden_checked:
|
||||
detail = " (golden ✓)"
|
||||
if r.update_error is not None:
|
||||
detail += f" (update warn: {r.update_error})"
|
||||
else:
|
||||
everything_ok = False
|
||||
if r.error is not None:
|
||||
status, detail = "FAIL", f" error={r.error}"
|
||||
elif r.overflow is not None:
|
||||
status, detail = "FAIL", f" overflow bbox={r.overflow}"
|
||||
elif r.golden_ok is False:
|
||||
status = "FAIL"
|
||||
detail = f" golden drift: {r.golden_diff_pixels}px (max Δ={r.golden_max_delta})"
|
||||
else:
|
||||
status, detail = "FAIL", ""
|
||||
print(f" [{status}] {r.size_label:>7} {r.mode}{detail}")
|
||||
print()
|
||||
return everything_ok
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check a plugin renders safely across sizes & screens")
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--plugin', '-p', help='Plugin id to check')
|
||||
group.add_argument('--all', action='store_true', help='Check every discovered plugin')
|
||||
parser.add_argument('--plugin-dir', '-d', default=None, help='Directory to search for plugins')
|
||||
parser.add_argument('--sizes', default=None, help='Comma-separated WxH list (default: all supported)')
|
||||
parser.add_argument('--config', '-c', default='{}', help='Plugin config overrides as JSON')
|
||||
parser.add_argument('--mock-data', '-m', default=None, help='Path to JSON file with mock cache data')
|
||||
parser.add_argument('--out-dir', '-o', default=None, help='Also dump rendered PNGs here')
|
||||
parser.add_argument('--skip-update', action='store_true', help='Skip calling update()')
|
||||
parser.add_argument('--update-golden', action='store_true', help='Write/refresh golden images')
|
||||
parser.add_argument('--golden-dir', default=None, help='Override golden dir (default: <plugin>/test/golden)')
|
||||
parser.add_argument('--freeze-time', default=None,
|
||||
help='Freeze wall clock, e.g. "2025-08-01 15:25:00" (for time-dependent plugins)')
|
||||
args = parser.parse_args()
|
||||
|
||||
search_dirs = [args.plugin_dir] if args.plugin_dir else DEFAULT_SEARCH_DIRS
|
||||
sizes = parse_sizes(args.sizes)
|
||||
|
||||
try:
|
||||
config = json.loads(args.config)
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error("Invalid --config JSON: %s", e)
|
||||
return 2
|
||||
if not isinstance(config, dict):
|
||||
logger.error("--config must be a JSON object, got %s", type(config).__name__)
|
||||
return 2
|
||||
|
||||
mock_data = {}
|
||||
if args.mock_data:
|
||||
mock_path = Path(args.mock_data)
|
||||
if not mock_path.exists():
|
||||
logger.error("Mock data file not found: %s", args.mock_data)
|
||||
return 2
|
||||
with open(mock_path) as f:
|
||||
mock_data = json.load(f)
|
||||
if not isinstance(mock_data, dict):
|
||||
logger.error("--mock-data must be a JSON object (key -> cache value), got %s",
|
||||
type(mock_data).__name__)
|
||||
return 2
|
||||
|
||||
plugin_ids = discover_plugins(search_dirs) if args.all else [args.plugin]
|
||||
if not plugin_ids:
|
||||
logger.error("No plugins found in: %s", search_dirs)
|
||||
return 2
|
||||
|
||||
out_dir = Path(args.out_dir) if args.out_dir else None
|
||||
golden_dir_override = Path(args.golden_dir) if args.golden_dir else None
|
||||
|
||||
all_results: Dict[str, List[RenderResult]] = {}
|
||||
for plugin_id in plugin_ids:
|
||||
all_results[plugin_id] = check_one(
|
||||
plugin_id=plugin_id, search_dirs=search_dirs, sizes=sizes,
|
||||
mock_data=mock_data, config=config, run_update=not args.skip_update,
|
||||
out_dir=out_dir, update_golden=args.update_golden,
|
||||
golden_dir_override=golden_dir_override, freeze_time=args.freeze_time,
|
||||
)
|
||||
|
||||
# When refreshing goldens we skip drift comparison, but a crash or overflow
|
||||
# still means the plugin is broken — never let --update-golden mask that.
|
||||
ok = print_report(all_results)
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Clear all plugin dependency markers to force fresh dependency check
|
||||
# Useful after updating plugins or troubleshooting dependency issues
|
||||
|
||||
echo "Clearing plugin dependency markers..."
|
||||
|
||||
# Check both possible cache locations
|
||||
CACHE_DIRS=(
|
||||
"/var/cache/ledmatrix"
|
||||
"$HOME/.cache/ledmatrix"
|
||||
)
|
||||
|
||||
for CACHE_DIR in "${CACHE_DIRS[@]}"; do
|
||||
if [ -d "$CACHE_DIR" ]; then
|
||||
echo "Checking $CACHE_DIR..."
|
||||
marker_count=$(find "$CACHE_DIR" -name "plugin_*_deps_installed" 2>/dev/null | wc -l)
|
||||
if [ "$marker_count" -gt 0 ]; then
|
||||
echo "Found $marker_count dependency marker(s) in $CACHE_DIR"
|
||||
find "$CACHE_DIR" -name "plugin_*_deps_installed" -delete
|
||||
echo "Cleared $marker_count marker(s)"
|
||||
else
|
||||
echo "No dependency markers found in $CACHE_DIR"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Done! Dependency markers cleared."
|
||||
echo "Next startup will check and install dependencies as needed."
|
||||
|
||||
@@ -67,8 +67,9 @@ def main():
|
||||
print(" 📍 Will run on: http://0.0.0.0:5000")
|
||||
print(" ⏹️ Press Ctrl+C to stop")
|
||||
|
||||
# Run the app (this should start the server)
|
||||
app.run(host='0.0.0.0', port=5000, debug=True)
|
||||
# Run the app (debug mode controlled by env var to satisfy security scanners)
|
||||
_debug = os.environ.get('LEDMATRIX_FLASK_DEBUG', '0') == '1'
|
||||
app.run(host='0.0.0.0', port=5000, debug=_debug)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n ⏹️ Server stopped by user")
|
||||
|
||||
@@ -55,7 +55,7 @@ def main():
|
||||
failures += not check("draw.textbbox",
|
||||
lambda: draw.textbbox((0, 0), "Test", font=font))
|
||||
|
||||
print("\nResampling (used in logo_helper, image_utils, sports base):")
|
||||
print("\nResampling (used in logo_helper, sports base):")
|
||||
logo = Image.new('RGBA', (200, 200), (255, 128, 0, 200))
|
||||
failures += not check("Image.Resampling.LANCZOS exists",
|
||||
lambda: str(Image.Resampling.LANCZOS))
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/bin/bash
|
||||
# safe_pip_install.sh — Install a requirements.txt as root after validating
|
||||
# that the resolved path is the project's own requirements.txt or a plugin's
|
||||
# requirements.txt under plugin-repos/ or plugins/.
|
||||
#
|
||||
# This script is intended to be called via sudo from the web interface, so
|
||||
# that packages a plugin declares end up visible to ledmatrix.service (which
|
||||
# runs as root) rather than only to whichever non-root user runs the web
|
||||
# interface. Plugin code already runs as root once loaded, so installing its
|
||||
# declared dependencies as root is not a new trust boundary.
|
||||
#
|
||||
# Usage: safe_pip_install.sh <requirements_txt_path>
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 <requirements_txt_path>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARGET="$1"
|
||||
|
||||
# Determine the project root (parent of scripts/fix_perms/)
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
# Allowed locations (resolved, no trailing slash):
|
||||
# - the project's own requirements.txt
|
||||
# - any requirements.txt under plugin-repos/ or plugins/
|
||||
ALLOWED_EXACT="$(realpath --canonicalize-missing "$PROJECT_ROOT/requirements.txt")"
|
||||
ALLOWED_BASES=(
|
||||
"$(realpath --canonicalize-missing "$PROJECT_ROOT/plugin-repos")"
|
||||
"$(realpath --canonicalize-missing "$PROJECT_ROOT/plugins")"
|
||||
)
|
||||
|
||||
# Resolve the target path (follow symlinks); works even if it doesn't exist.
|
||||
RESOLVED_TARGET="$(realpath --canonicalize-missing "$TARGET")"
|
||||
|
||||
# Must be named requirements.txt — never install from an arbitrary file.
|
||||
if [ "$(basename "$RESOLVED_TARGET")" != "requirements.txt" ]; then
|
||||
echo "DENIED: $RESOLVED_TARGET is not a requirements.txt file" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
ALLOWED=false
|
||||
if [ "$RESOLVED_TARGET" = "$ALLOWED_EXACT" ]; then
|
||||
ALLOWED=true
|
||||
else
|
||||
for BASE in "${ALLOWED_BASES[@]}"; do
|
||||
if [[ "$RESOLVED_TARGET" == "$BASE/"* ]]; then
|
||||
ALLOWED=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$ALLOWED" = false ]; then
|
||||
echo "DENIED: $RESOLVED_TARGET is not an allowed requirements.txt location" >&2
|
||||
echo "Allowed: $ALLOWED_EXACT, or any requirements.txt under: ${ALLOWED_BASES[*]}" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ ! -f "$RESOLVED_TARGET" ]; then
|
||||
echo "ERROR: $RESOLVED_TARGET does not exist" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
PYTHON_PATH="$(command -v python3)"
|
||||
# --ignore-installed: root's site-packages often has apt/dpkg-managed copies
|
||||
# of common libraries (requests, urllib3, ...) with no pip RECORD file, which
|
||||
# pip refuses to uninstall in place ("Cannot uninstall: no RECORD file was
|
||||
# found"). This tells pip to install the newer version alongside rather than
|
||||
# aborting the whole requirements.txt install over one such conflict.
|
||||
exec "$PYTHON_PATH" -m pip install --break-system-packages --ignore-installed -r "$RESOLVED_TARGET"
|
||||
@@ -0,0 +1,297 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Security Report Generator
|
||||
|
||||
Aggregates JSON output from all CI security audit jobs into a single
|
||||
Markdown report suitable for PR comments and artifact storage.
|
||||
|
||||
Expected artifact layout (from actions/download-artifact@v4):
|
||||
<artifact-dir>/
|
||||
sast-results/
|
||||
bandit-results.json
|
||||
semgrep-results.json
|
||||
dependency-audit-results/
|
||||
pip-audit-results.json
|
||||
safety-results.json
|
||||
secrets-scan-results/
|
||||
gitleaks-results.json
|
||||
security-proofs-results/
|
||||
security-proofs-results.json
|
||||
plugin-audit-results/
|
||||
plugin-audit-results.json
|
||||
|
||||
Usage:
|
||||
python scripts/generate_report.py --artifact-dir audit-artifacts/ --output report.md
|
||||
python scripts/generate_report.py --artifact-dir audit-artifacts/ --output report.md --verbose
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
# Gitleaks matches containing these strings are template placeholders, not real secrets
|
||||
_GITLEAKS_SUPPRESS = [
|
||||
"YOUR_",
|
||||
"PLACEHOLDER",
|
||||
"_HERE",
|
||||
"example.com",
|
||||
"config_secrets.template",
|
||||
]
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _load(path: Path) -> dict | list | None:
|
||||
"""Load JSON file, returning None on any error."""
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, FileNotFoundError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def _md_table_row(*cells: str) -> str:
|
||||
return "| " + " | ".join(str(c) for c in cells) + " |"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Per-tool summarizers
|
||||
# Returns: (markdown_lines: list[str], critical_count: int)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _summarize_bandit(artifact_dir: Path) -> tuple[list[str], int]:
|
||||
data = _load(artifact_dir / "sast-results" / "bandit-results.json")
|
||||
if data is None:
|
||||
return ["_bandit results not available_"], 0
|
||||
|
||||
results = data.get("results", [])
|
||||
high = [r for r in results if r.get("issue_severity") == "HIGH"]
|
||||
medium = [r for r in results if r.get("issue_severity") == "MEDIUM"]
|
||||
low = [r for r in results if r.get("issue_severity") == "LOW"]
|
||||
|
||||
lines = [
|
||||
f"**Bandit**: {len(high)} HIGH · {len(medium)} MEDIUM · {len(low)} LOW"
|
||||
]
|
||||
|
||||
if high:
|
||||
lines += [
|
||||
"",
|
||||
"| Severity | File | Line | Issue |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
for r in high[:10]:
|
||||
fname = Path(r.get("filename", "")).name
|
||||
lines.append(_md_table_row(
|
||||
"HIGH", f"`{fname}`",
|
||||
str(r.get("line_number", "?")),
|
||||
r.get("issue_text", "")
|
||||
))
|
||||
if len(high) > 10:
|
||||
lines.append(f"_… and {len(high) - 10} more HIGH findings_")
|
||||
|
||||
return lines, len(high)
|
||||
|
||||
|
||||
def _summarize_pip_audit(artifact_dir: Path) -> tuple[list[str], int]:
|
||||
data = _load(artifact_dir / "dependency-audit-results" / "pip-audit-results.json")
|
||||
if data is None:
|
||||
return ["_pip-audit results not available_"], 0
|
||||
|
||||
# pip-audit JSON format: {"dependencies": [{"name": ..., "vulns": [...]}]}
|
||||
vulns: list[dict] = []
|
||||
for dep in data.get("dependencies", []):
|
||||
for v in dep.get("vulns", []):
|
||||
vulns.append({"package": dep.get("name", "?"), **v})
|
||||
|
||||
lines = [f"**pip-audit**: {len(vulns)} vulnerabilities found"]
|
||||
|
||||
if vulns:
|
||||
lines += ["", "| Package | ID | Fix |", "| --- | --- | --- |"]
|
||||
for v in vulns[:10]:
|
||||
fix = v.get("fix_versions", ["none"])
|
||||
fix_str = ", ".join(fix) if fix else "none"
|
||||
lines.append(_md_table_row(
|
||||
v.get("package", "?"),
|
||||
v.get("id", "?"),
|
||||
fix_str,
|
||||
))
|
||||
|
||||
# Treat known vulnerabilities as warnings, not critical (they may be unavoidable)
|
||||
return lines, 0
|
||||
|
||||
|
||||
def _summarize_gitleaks(artifact_dir: Path) -> tuple[list[str], int]:
|
||||
data = _load(artifact_dir / "secrets-scan-results" / "gitleaks-results.json")
|
||||
if data is None:
|
||||
return ["_gitleaks results not available_"], 0
|
||||
|
||||
if not isinstance(data, list):
|
||||
data = []
|
||||
|
||||
real_findings = []
|
||||
suppressed = 0
|
||||
for finding in data:
|
||||
secret_val = str(finding.get("Secret", "") or finding.get("Match", ""))
|
||||
if any(p in secret_val for p in _GITLEAKS_SUPPRESS):
|
||||
suppressed += 1
|
||||
else:
|
||||
real_findings.append(finding)
|
||||
|
||||
lines = [
|
||||
f"**Gitleaks**: {len(real_findings)} finding(s) "
|
||||
f"({suppressed} suppressed as template placeholders)"
|
||||
]
|
||||
|
||||
if real_findings:
|
||||
lines += ["", "| Rule | File | Line | Description |", "| --- | --- | --- | --- |"]
|
||||
for f in real_findings[:10]:
|
||||
fname = Path(f.get("File", "")).name
|
||||
lines.append(_md_table_row(
|
||||
f.get("RuleID", "?"),
|
||||
f"`{fname}`",
|
||||
str(f.get("StartLine", "?")),
|
||||
f.get("Description", ""),
|
||||
))
|
||||
|
||||
critical = len(real_findings) # any real secret is critical
|
||||
return lines, critical
|
||||
|
||||
|
||||
def _summarize_security_proofs(artifact_dir: Path) -> tuple[list[str], int]:
|
||||
data = _load(artifact_dir / "security-proofs-results" / "security-proofs-results.json")
|
||||
if data is None:
|
||||
return ["_security proofs results not available_"], 0
|
||||
|
||||
if not isinstance(data, list):
|
||||
data = []
|
||||
|
||||
critical = [r for r in data if r.get("severity") == "CRITICAL"]
|
||||
warnings = [r for r in data if r.get("severity") == "WARNING"]
|
||||
passed = [r for r in data if r.get("severity") == "PASS"]
|
||||
skipped = [r for r in data if r.get("severity") == "SKIP"]
|
||||
|
||||
lines = [
|
||||
f"**Security Proofs**: "
|
||||
f"{len(passed)} PASS · {len(warnings)} WARN · "
|
||||
f"{len(critical)} CRITICAL · {len(skipped)} SKIP",
|
||||
"",
|
||||
]
|
||||
|
||||
_icon = {"PASS": "✅", "INFO": "ℹ️", "WARNING": "⚠️",
|
||||
"CRITICAL": "🚨", "SKIP": "⏭️"}
|
||||
for r in data:
|
||||
icon = _icon.get(r.get("severity", ""), "❓")
|
||||
lines.append(
|
||||
f"- {icon} **{r.get('test_id', '?')}**: {r.get('message', '')}"
|
||||
)
|
||||
if r.get("details") and r.get("severity") in ("CRITICAL", "WARNING"):
|
||||
lines.append(f" - _{r['details']}_")
|
||||
|
||||
return lines, len(critical)
|
||||
|
||||
|
||||
def _summarize_plugin_audit(artifact_dir: Path) -> tuple[list[str], int]:
|
||||
data = _load(artifact_dir / "plugin-audit-results" / "plugin-audit-results.json")
|
||||
if data is None:
|
||||
return ["_plugin audit results not available_"], 0
|
||||
|
||||
summary = data.get("summary", {})
|
||||
findings = data.get("findings", [])
|
||||
critical_findings = [f for f in findings if f.get("severity") == "CRITICAL"]
|
||||
warning_findings = [f for f in findings if f.get("severity") == "WARNING"]
|
||||
|
||||
lines = [
|
||||
f"**Plugin Audit**: {data.get('plugins_scanned', '?')} plugins scanned — "
|
||||
f"{summary.get('critical', 0)} CRITICAL · {summary.get('warnings', 0)} WARNINGS"
|
||||
]
|
||||
|
||||
if critical_findings:
|
||||
lines += ["", "| Plugin | File | Line | Rule | Message |",
|
||||
"| --- | --- | --- | --- | --- |"]
|
||||
for f in critical_findings[:10]:
|
||||
fname = Path(f.get("file", "")).name
|
||||
lines.append(_md_table_row(
|
||||
f.get("plugin_id", "?"),
|
||||
f"`{fname}`",
|
||||
str(f.get("line", "?")),
|
||||
f.get("rule", "?"),
|
||||
f.get("message", ""),
|
||||
))
|
||||
|
||||
if warning_findings and not critical_findings:
|
||||
lines.append(f"\n_{len(warning_findings)} warning(s) found — see artifact for details_")
|
||||
|
||||
return lines, summary.get("critical", 0)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Main
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate consolidated security audit report",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--artifact-dir", required=True,
|
||||
help="Directory containing downloaded CI artifacts")
|
||||
parser.add_argument("--output", "-o", required=True,
|
||||
help="Output Markdown file path")
|
||||
parser.add_argument("--verbose", "-v", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
artifact_dir = Path(args.artifact_dir)
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
|
||||
bandit_lines, bandit_crit = _summarize_bandit(artifact_dir)
|
||||
pip_audit_lines, pip_audit_crit = _summarize_pip_audit(artifact_dir)
|
||||
gitleaks_lines, gitleaks_crit = _summarize_gitleaks(artifact_dir)
|
||||
proofs_lines, proofs_crit = _summarize_security_proofs(artifact_dir)
|
||||
plugins_lines, plugins_crit = _summarize_plugin_audit(artifact_dir)
|
||||
|
||||
total_critical = bandit_crit + pip_audit_crit + gitleaks_crit + proofs_crit + plugins_crit
|
||||
overall = "ACTION REQUIRED 🚨" if total_critical > 0 else "PASSED ✅"
|
||||
|
||||
def section(title: str, lines: list[str]) -> str:
|
||||
return f"### {title}\n\n" + "\n".join(lines) + "\n"
|
||||
|
||||
report = f"""## 🔒 Security Audit — {overall}
|
||||
|
||||
_Generated: {timestamp}_
|
||||
|
||||
| Critical | High/Warn | Overall |
|
||||
| :---: | :---: | :---: |
|
||||
| {'🚨 ' + str(total_critical) if total_critical else '✅ 0'} | ⚠️ see below | {overall} |
|
||||
|
||||
---
|
||||
|
||||
{section('SAST — Bandit', bandit_lines)}
|
||||
{section('Dependencies — pip-audit', pip_audit_lines)}
|
||||
{section('Secrets — Gitleaks', gitleaks_lines)}
|
||||
{section('LEDMatrix Security Proofs', proofs_lines)}
|
||||
{section('Plugin Security Audit', plugins_lines)}
|
||||
---
|
||||
|
||||
_Total critical findings: **{total_critical}**_
|
||||
"""
|
||||
|
||||
output_path = Path(args.output)
|
||||
output_path.write_text(report, encoding="utf-8")
|
||||
|
||||
if args.verbose:
|
||||
print(f" Report written to: {output_path}")
|
||||
print(f" Status: {overall}")
|
||||
print(f" Critical findings: {total_critical}")
|
||||
print(f" bandit={bandit_crit} pip-audit={pip_audit_crit} "
|
||||
f"gitleaks={gitleaks_crit} proofs={proofs_crit} plugins={plugins_crit}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -33,6 +33,7 @@ POWEROFF_PATH=$(command -v poweroff) || true
|
||||
BASH_PATH=$(command -v bash) || true
|
||||
JOURNALCTL_PATH=$(command -v journalctl) || true
|
||||
SAFE_RM_PATH="$PROJECT_ROOT/scripts/fix_perms/safe_plugin_rm.sh"
|
||||
SAFE_PIP_INSTALL_PATH="$PROJECT_ROOT/scripts/fix_perms/safe_pip_install.sh"
|
||||
|
||||
# Validate required commands (systemctl, bash, python3 are essential)
|
||||
for CMD_NAME in SYSTEMCTL_PATH BASH_PATH PYTHON_PATH; do
|
||||
@@ -48,11 +49,15 @@ if [ ${#MISSING_CMDS[@]} -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Validate helper script exists
|
||||
# Validate helper scripts exist
|
||||
if [ ! -f "$SAFE_RM_PATH" ]; then
|
||||
echo "Error: Safe plugin removal helper not found: $SAFE_RM_PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$SAFE_PIP_INSTALL_PATH" ]; then
|
||||
echo "Error: Safe pip install helper not found: $SAFE_PIP_INSTALL_PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Command paths:"
|
||||
echo " Python: $PYTHON_PATH"
|
||||
@@ -62,6 +67,7 @@ echo " Poweroff: ${POWEROFF_PATH:-(not found, skipping)}"
|
||||
echo " Bash: $BASH_PATH"
|
||||
echo " Journalctl: ${JOURNALCTL_PATH:-(not found, skipping)}"
|
||||
echo " Safe plugin rm: $SAFE_RM_PATH"
|
||||
echo " Safe pip install: $SAFE_PIP_INSTALL_PATH"
|
||||
|
||||
# Create a temporary sudoers file
|
||||
TEMP_SUDOERS="/tmp/ledmatrix_web_sudoers_$$"
|
||||
@@ -101,13 +107,22 @@ TEMP_SUDOERS="/tmp/ledmatrix_web_sudoers_$$"
|
||||
fi
|
||||
|
||||
# Required: python3, bash
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $PYTHON_PATH $PROJECT_DIR/display_controller.py"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $BASH_PATH $PROJECT_DIR/start_display.sh"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $BASH_PATH $PROJECT_DIR/stop_display.sh"
|
||||
# NOTE: display_controller.py/start_display.sh/stop_display.sh live at the
|
||||
# project root, not under scripts/install/ (where this script lives) —
|
||||
# must use PROJECT_ROOT here, not PROJECT_DIR.
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $PYTHON_PATH $PROJECT_ROOT/display_controller.py"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $BASH_PATH $PROJECT_ROOT/start_display.sh"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $BASH_PATH $PROJECT_ROOT/stop_display.sh"
|
||||
echo ""
|
||||
echo "# Allow web user to remove plugin directories via vetted helper script"
|
||||
echo "# The helper validates that the target path resolves inside plugin-repos/ or plugins/"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $BASH_PATH $SAFE_RM_PATH *"
|
||||
echo ""
|
||||
echo "# Allow web user to install a plugin's requirements.txt as root via vetted"
|
||||
echo "# helper script, so packages are visible to root-run ledmatrix.service"
|
||||
echo "# (not just the web interface's own user). The helper validates the target"
|
||||
echo "# is requirements.txt at the project root or under plugin-repos/ or plugins/."
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $BASH_PATH $SAFE_PIP_INSTALL_PATH *"
|
||||
} > "$TEMP_SUDOERS"
|
||||
|
||||
echo ""
|
||||
@@ -126,6 +141,7 @@ echo "- Run display_controller.py directly"
|
||||
echo "- Execute start_display.sh and stop_display.sh"
|
||||
echo "- Reboot and shutdown the system"
|
||||
echo "- Remove plugin directories (for update/uninstall when root-owned files block deletion)"
|
||||
echo "- Install plugin/base requirements.txt as root (so ledmatrix.service can see them)"
|
||||
echo ""
|
||||
|
||||
# Ask for confirmation
|
||||
@@ -147,6 +163,13 @@ fi
|
||||
if ! sudo chmod 755 "$SAFE_RM_PATH"; then
|
||||
echo "Warning: Could not set permissions on $SAFE_RM_PATH"
|
||||
fi
|
||||
echo "Hardening safe_pip_install.sh ownership..."
|
||||
if ! sudo chown root:root "$SAFE_PIP_INSTALL_PATH"; then
|
||||
echo "Warning: Could not set ownership on $SAFE_PIP_INSTALL_PATH"
|
||||
fi
|
||||
if ! sudo chmod 755 "$SAFE_PIP_INSTALL_PATH"; then
|
||||
echo "Warning: Could not set permissions on $SAFE_PIP_INSTALL_PATH"
|
||||
fi
|
||||
|
||||
if sudo cp "$TEMP_SUDOERS" /etc/sudoers.d/ledmatrix_web; then
|
||||
echo "Configuration applied successfully!"
|
||||
@@ -160,7 +183,7 @@ if sudo cp "$TEMP_SUDOERS" /etc/sudoers.d/ledmatrix_web; then
|
||||
echo "✗ systemctl status ledmatrix.service - Failed"
|
||||
fi
|
||||
|
||||
if sudo -n test -f "$PROJECT_DIR/start_display.sh"; then
|
||||
if sudo -n test -f "$PROJECT_ROOT/start_display.sh"; then
|
||||
echo "✓ File access test - OK"
|
||||
else
|
||||
echo "✗ File access test - Failed"
|
||||
|
||||
@@ -340,9 +340,14 @@ main() {
|
||||
echo ""
|
||||
|
||||
# Execute with proper error handling and non-interactive mode
|
||||
# Temporarily disable errexit to capture exit code instead of exiting immediately
|
||||
# Temporarily disable errexit AND the ERR trap to capture exit code instead of
|
||||
# exiting immediately. `set +e` alone does not suppress the ERR trap, so without
|
||||
# `trap '' ERR` a non-zero exit from first_time_install.sh would trigger on_error
|
||||
# here with the generic "Main installation" message instead of the detailed
|
||||
# if/else handling below.
|
||||
set +e
|
||||
|
||||
trap '' ERR
|
||||
|
||||
# Check /tmp permissions - only fix if actually wrong (common in automated scenarios)
|
||||
# When running manually, /tmp usually has correct permissions (1777)
|
||||
TMP_PERMS=$(stat -c '%a' /tmp 2>/dev/null || echo "unknown")
|
||||
@@ -370,6 +375,7 @@ main() {
|
||||
sudo -E env TMPDIR=/tmp LEDMATRIX_ASSUME_YES=1 bash ./first_time_install.sh -y </dev/null
|
||||
fi
|
||||
INSTALL_EXIT_CODE=$?
|
||||
trap 'on_error $LINENO' ERR # Re-enable ERR trap
|
||||
set -e # Re-enable errexit
|
||||
|
||||
if [ $INSTALL_EXIT_CODE -eq 0 ]; then
|
||||
|
||||
@@ -6,82 +6,143 @@ then falls back to pip with --break-system-packages
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import warnings
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
def install_via_apt(package_name):
|
||||
"""Try to install a package via apt."""
|
||||
try:
|
||||
# Map pip package names to apt package names
|
||||
apt_package_map = {
|
||||
'flask': 'python3-flask',
|
||||
'PIL': 'python3-pil',
|
||||
'freetype': 'python3-freetype',
|
||||
'psutil': 'python3-psutil',
|
||||
'werkzeug': 'python3-werkzeug',
|
||||
'numpy': 'python3-numpy',
|
||||
'requests': 'python3-requests',
|
||||
'python-dateutil': 'python3-dateutil',
|
||||
'pytz': 'python3-tz',
|
||||
'geopy': 'python3-geopy',
|
||||
'unidecode': 'python3-unidecode',
|
||||
'websockets': 'python3-websockets',
|
||||
'websocket-client': 'python3-websocket-client'
|
||||
}
|
||||
|
||||
apt_package = apt_package_map.get(package_name, f'python3-{package_name}')
|
||||
|
||||
print(f"Trying to install {apt_package} via apt...")
|
||||
subprocess.check_call([
|
||||
'sudo', 'apt', 'update'
|
||||
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
subprocess.check_call([
|
||||
'sudo', 'apt', 'install', '-y', apt_package
|
||||
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
# How many trailing lines of a failed command's output to keep for the
|
||||
# end-of-run failure summary. Keeps the root cause near the end of the log,
|
||||
# which is where first_time_install.sh's error handler tails from.
|
||||
ERROR_TAIL_LINES = 15
|
||||
|
||||
|
||||
def _run(cmd: List[str]) -> Tuple[bool, str]:
|
||||
"""Run a command, streaming combined stdout/stderr to a temp file.
|
||||
|
||||
Returns (success, output) instead of raising, so callers can report
|
||||
*why* a command failed rather than just that it failed. `output` is
|
||||
bounded to the last ERROR_TAIL_LINES lines so failures from very
|
||||
chatty commands (e.g. pip build logs) don't get buffered in memory.
|
||||
"""
|
||||
with tempfile.TemporaryFile(mode='w+b') as f:
|
||||
result = subprocess.run(cmd, stdout=f, stderr=subprocess.STDOUT) # nosec B603 B607 - hardcoded apt/pip args # nosemgrep
|
||||
f.seek(0)
|
||||
# Stream line-by-line so only the last ERROR_TAIL_LINES are ever held
|
||||
# in memory, regardless of how much output the command produced.
|
||||
tail = deque(
|
||||
(line.decode('utf-8', errors='replace').rstrip('\n') for line in f),
|
||||
maxlen=ERROR_TAIL_LINES,
|
||||
)
|
||||
return result.returncode == 0, '\n'.join(tail)
|
||||
|
||||
|
||||
def install_via_apt(package_name: str) -> Tuple[bool, str]:
|
||||
"""Try to install a package via apt. Returns (success, output)."""
|
||||
# Map pip package names to apt package names
|
||||
apt_package_map = {
|
||||
'flask': 'python3-flask',
|
||||
'PIL': 'python3-pil',
|
||||
'freetype': 'python3-freetype',
|
||||
'psutil': 'python3-psutil',
|
||||
'werkzeug': 'python3-werkzeug',
|
||||
'numpy': 'python3-numpy',
|
||||
'requests': 'python3-requests',
|
||||
'python-dateutil': 'python3-dateutil',
|
||||
'pytz': 'python3-tz',
|
||||
'geopy': 'python3-geopy',
|
||||
'unidecode': 'python3-unidecode',
|
||||
'websockets': 'python3-websockets',
|
||||
'websocket-client': 'python3-websocket-client'
|
||||
}
|
||||
|
||||
apt_package = apt_package_map.get(package_name, f'python3-{package_name}')
|
||||
|
||||
print(f"Trying to install {apt_package} via apt...")
|
||||
success, output = _run(['sudo', 'apt-get', '-o', 'DPkg::Lock::Timeout=180', 'install', '-y', apt_package])
|
||||
if success:
|
||||
print(f"Successfully installed {apt_package} via apt")
|
||||
return True
|
||||
|
||||
except subprocess.CalledProcessError:
|
||||
print(f"Failed to install {package_name} via apt, will try pip")
|
||||
return False
|
||||
return True, ""
|
||||
|
||||
def install_via_pip(package_name):
|
||||
print(f"Failed to install {apt_package} via apt, will try pip")
|
||||
return False, output
|
||||
|
||||
|
||||
def install_via_pip(package_name: str) -> Tuple[bool, str]:
|
||||
"""Install a package via pip with --break-system-packages and --prefer-binary.
|
||||
|
||||
--break-system-packages allows pip to install into the system Python on
|
||||
Debian/Ubuntu-based systems without a virtual environment.
|
||||
--prefer-binary prefers pre-built wheels over source distributions to avoid
|
||||
exhausting /tmp space during compilation.
|
||||
"""
|
||||
try:
|
||||
print(f"Installing {package_name} via pip...")
|
||||
subprocess.check_call([
|
||||
sys.executable, '-m', 'pip', 'install', '--break-system-packages', '--prefer-binary', package_name
|
||||
])
|
||||
print(f"Successfully installed {package_name} via pip")
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Failed to install {package_name} via pip: {e}")
|
||||
return False
|
||||
--ignore-installed stops pip from trying to *uninstall* packages that were
|
||||
installed by apt (e.g. python3-requests). Those Debian packages ship no
|
||||
pip RECORD file, so an uninstall attempt fails with "uninstall-no-record-file"
|
||||
and aborts the whole install. With --ignore-installed, pip lays the new
|
||||
version down in /usr/local where it shadows the apt copy instead of removing
|
||||
it. This matters when a pip dependency (google-api-python-client pulls a
|
||||
newer requests) needs to upgrade an apt-managed package.
|
||||
|
||||
def check_package_installed(package_name):
|
||||
Returns (success, output).
|
||||
"""
|
||||
print(f"Installing {package_name} via pip...")
|
||||
success, output = _run([
|
||||
sys.executable, '-m', 'pip', 'install',
|
||||
'--break-system-packages', '--prefer-binary', '--ignore-installed', package_name
|
||||
])
|
||||
if success:
|
||||
print(f"Successfully installed {package_name} via pip")
|
||||
return True, ""
|
||||
|
||||
print(f"Failed to install {package_name} via pip (see failure summary at end of log)")
|
||||
return False, output
|
||||
|
||||
|
||||
# Distribution (pip/apt) names whose importable module name differs.
|
||||
IMPORT_NAME_MAP = {
|
||||
'python-dateutil': 'dateutil',
|
||||
'websocket-client': 'websocket',
|
||||
}
|
||||
|
||||
|
||||
def check_package_installed(package_name: str) -> bool:
|
||||
"""Check if a package is already installed."""
|
||||
import_name = IMPORT_NAME_MAP.get(package_name, package_name)
|
||||
# Suppress deprecation warnings when checking if packages are installed
|
||||
# (we're just checking, not using them)
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings('ignore', category=DeprecationWarning)
|
||||
try:
|
||||
__import__(package_name)
|
||||
__import__(import_name)
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
def print_failure_summary(failed_packages: List[str], failure_details: dict) -> None:
|
||||
print("\n" + "=" * 60)
|
||||
print("DEPENDENCY INSTALLATION FAILURES - DETAILS")
|
||||
print("=" * 60)
|
||||
for package in failed_packages:
|
||||
print(f"\nPackage: {package}")
|
||||
print("-" * 40)
|
||||
output = failure_details.get(package, "").strip()
|
||||
if not output:
|
||||
print(" (no output captured)")
|
||||
continue
|
||||
for line in output.splitlines()[-ERROR_TAIL_LINES:]:
|
||||
print(f" {line}")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main installation function."""
|
||||
print("Installing dependencies for LED Matrix Web Interface V2...")
|
||||
|
||||
|
||||
print("Refreshing apt package index...")
|
||||
_run(['sudo', 'apt', 'update']) # best-effort; individual installs surface their own errors
|
||||
|
||||
# List of required packages
|
||||
required_packages = [
|
||||
'flask',
|
||||
@@ -98,19 +159,23 @@ def main():
|
||||
'websockets',
|
||||
'websocket-client'
|
||||
]
|
||||
|
||||
|
||||
failed_packages = []
|
||||
|
||||
failure_details = {}
|
||||
|
||||
for package in required_packages:
|
||||
if check_package_installed(package):
|
||||
print(f"{package} is already installed")
|
||||
continue
|
||||
|
||||
|
||||
# Try apt first, then pip
|
||||
if not install_via_apt(package):
|
||||
if not install_via_pip(package):
|
||||
ok, apt_output = install_via_apt(package)
|
||||
if not ok:
|
||||
ok, pip_output = install_via_pip(package)
|
||||
if not ok:
|
||||
failed_packages.append(package)
|
||||
|
||||
failure_details[package] = pip_output or apt_output
|
||||
|
||||
# Install packages that don't have apt equivalents
|
||||
special_packages = [
|
||||
'timezonefinder>=6.5.0,<7.0.0',
|
||||
@@ -122,47 +187,49 @@ def main():
|
||||
'python-socketio>=5.11.0,<6.0.0',
|
||||
'python-engineio>=4.9.0,<5.0.0'
|
||||
]
|
||||
|
||||
|
||||
for package in special_packages:
|
||||
if not install_via_pip(package):
|
||||
ok, pip_output = install_via_pip(package)
|
||||
if not ok:
|
||||
failed_packages.append(package)
|
||||
|
||||
failure_details[package] = pip_output
|
||||
|
||||
# Install rgbmatrix module from local source (optional - may already be installed in Step 6)
|
||||
# Check if already installed first
|
||||
if check_package_installed('rgbmatrix'):
|
||||
print("rgbmatrix module already installed, skipping...")
|
||||
else:
|
||||
print("Installing rgbmatrix module from local source...")
|
||||
try:
|
||||
# Get project root (parent of scripts directory)
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
rgbmatrix_path = PROJECT_ROOT / 'rpi-rgb-led-matrix-master' / 'bindings' / 'python'
|
||||
if rgbmatrix_path.exists():
|
||||
# Check if the module has been built (look for setup.py)
|
||||
setup_py = rgbmatrix_path / 'setup.py'
|
||||
if setup_py.exists():
|
||||
# Try installing - use regular install, not editable mode
|
||||
# This is optional for web interface and should already be installed in Step 6
|
||||
subprocess.check_call([
|
||||
sys.executable, '-m', 'pip', 'install', '--break-system-packages', str(rgbmatrix_path)
|
||||
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
# Get project root (parent of scripts directory)
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
rgbmatrix_path = PROJECT_ROOT / 'rpi-rgb-led-matrix-master' / 'bindings' / 'python'
|
||||
if rgbmatrix_path.exists():
|
||||
# Check if the module has been built (look for setup.py)
|
||||
setup_py = rgbmatrix_path / 'setup.py'
|
||||
if setup_py.exists():
|
||||
# Try installing - use regular install, not editable mode
|
||||
# This is optional for web interface and should already be installed in Step 6
|
||||
ok, output = _run([sys.executable, '-m', 'pip', 'install', '--break-system-packages', str(rgbmatrix_path)])
|
||||
if ok:
|
||||
print("rgbmatrix module installed successfully")
|
||||
else:
|
||||
print("Warning: rgbmatrix setup.py not found, module may need to be built first")
|
||||
print(" This is normal if Step 6 hasn't completed yet.")
|
||||
# Don't fail the whole installation - rgbmatrix is optional for web interface
|
||||
# and should be installed in Step 6 of first_time_install.sh
|
||||
print("Warning: Failed to install rgbmatrix module:")
|
||||
for line in output.strip().splitlines()[-ERROR_TAIL_LINES:]:
|
||||
print(f" {line}")
|
||||
print(" This is normal if rgbmatrix hasn't been built yet (Step 6).")
|
||||
print(" The web interface will work without it.")
|
||||
else:
|
||||
print("Warning: rgbmatrix source not found (this is normal if Step 6 hasn't run yet)")
|
||||
except subprocess.CalledProcessError as e:
|
||||
# Don't fail the whole installation - rgbmatrix is optional for web interface
|
||||
# and should be installed in Step 6 of first_time_install.sh
|
||||
print(f"Warning: Failed to install rgbmatrix module: {e}")
|
||||
print(" This is normal if rgbmatrix hasn't been built yet (Step 6).")
|
||||
print(" The web interface will work without it.")
|
||||
# Don't add to failed_packages since it's optional
|
||||
|
||||
print("Warning: rgbmatrix setup.py not found, module may need to be built first")
|
||||
print(" This is normal if Step 6 hasn't completed yet.")
|
||||
else:
|
||||
print("Warning: rgbmatrix source not found (this is normal if Step 6 hasn't run yet)")
|
||||
|
||||
if failed_packages:
|
||||
print(f"\nFailed to install the following packages: {failed_packages}")
|
||||
print("You may need to install them manually or check your system configuration.")
|
||||
print_failure_summary(failed_packages, failure_details)
|
||||
return False
|
||||
else:
|
||||
print("\nAll dependencies installed successfully!")
|
||||
|
||||
@@ -0,0 +1,504 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
LEDMatrix Security Proof Tests
|
||||
|
||||
Automated proofs that run in CI to verify security properties hold on every
|
||||
commit. Inspired by the Huntarr security review approach of using standard
|
||||
tooling to confirm specific vulnerability classes are absent.
|
||||
|
||||
Usage:
|
||||
python scripts/prove_security.py
|
||||
python scripts/prove_security.py --verbose
|
||||
python scripts/prove_security.py --output results.json
|
||||
|
||||
Exit code: 1 only if CRITICAL findings are detected. Warnings are reported
|
||||
but do not block CI.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, asdict
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Result dataclass
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class TestResult:
|
||||
test_id: str
|
||||
severity: str # PASS | INFO | WARNING | CRITICAL | SKIP
|
||||
message: str
|
||||
details: str = ""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
@property
|
||||
def icon(self) -> str:
|
||||
return {
|
||||
"PASS": "✅",
|
||||
"INFO": "ℹ️ ",
|
||||
"WARNING": "⚠️ ",
|
||||
"CRITICAL": "🚨",
|
||||
"SKIP": "⏭️ ",
|
||||
}.get(self.severity, "❓")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# T1: Plugin Loading / Zip Slip
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_t1a_zip_slip_protection() -> TestResult:
|
||||
"""
|
||||
Verify that zip-slip protection exists in store_manager.py.
|
||||
|
||||
The protection lives at src/plugin_system/store_manager.py and uses
|
||||
Path.is_relative_to() to validate each zip member before extraction.
|
||||
This test confirms the guard is present — it should always pass green.
|
||||
"""
|
||||
store_manager = PROJECT_ROOT / "src" / "plugin_system" / "store_manager.py"
|
||||
if not store_manager.exists():
|
||||
return TestResult("T1a", "CRITICAL",
|
||||
"store_manager.py not found",
|
||||
f"Expected at {store_manager}")
|
||||
|
||||
content = store_manager.read_text(encoding="utf-8")
|
||||
|
||||
has_relative_to = "is_relative_to" in content
|
||||
has_log_message = "Zip-slip detected" in content
|
||||
|
||||
if not has_relative_to:
|
||||
return TestResult("T1a", "CRITICAL",
|
||||
"Zip-slip protection (is_relative_to) NOT FOUND in store_manager.py",
|
||||
"The is_relative_to() guard must be present before zipfile.extractall()")
|
||||
|
||||
if not has_log_message:
|
||||
return TestResult("T1a", "WARNING",
|
||||
"is_relative_to() found but 'Zip-slip detected' log message missing",
|
||||
"Verify the protection block is still active and the log was not removed")
|
||||
|
||||
return TestResult("T1a", "PASS",
|
||||
"Zip-slip protection verified",
|
||||
"is_relative_to() guard + 'Zip-slip detected' log present in store_manager.py")
|
||||
|
||||
|
||||
def test_t1b_dangerous_plugin_calls() -> list[TestResult]:
|
||||
"""
|
||||
Scan plugin directories for dangerous function calls (eval, exec).
|
||||
These represent arbitrary code execution risks in plugin code.
|
||||
"""
|
||||
results = []
|
||||
plugin_dirs = [
|
||||
PROJECT_ROOT / "plugins",
|
||||
PROJECT_ROOT / "plugin-repos",
|
||||
]
|
||||
|
||||
violations: list[str] = []
|
||||
files_scanned = 0
|
||||
|
||||
for base in plugin_dirs:
|
||||
if not base.exists():
|
||||
continue
|
||||
for plugin_dir in sorted(base.iterdir()):
|
||||
if not plugin_dir.is_dir() or plugin_dir.name.startswith(('.', '_')):
|
||||
continue
|
||||
for py_file in plugin_dir.rglob("*.py"):
|
||||
files_scanned += 1
|
||||
try:
|
||||
source = py_file.read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=str(py_file))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
||||
if node.func.id in ("eval", "exec"):
|
||||
rel = py_file.relative_to(PROJECT_ROOT)
|
||||
violations.append(
|
||||
f"{rel}:{node.lineno} — {node.func.id}() call")
|
||||
except (SyntaxError, OSError):
|
||||
pass
|
||||
|
||||
if violations:
|
||||
results.append(TestResult(
|
||||
"T1b", "CRITICAL",
|
||||
f"Dangerous function calls found in plugins ({len(violations)} instance(s))",
|
||||
"; ".join(violations[:10])
|
||||
))
|
||||
else:
|
||||
results.append(TestResult(
|
||||
"T1b", "PASS",
|
||||
f"No eval()/exec() calls found in plugins",
|
||||
f"{files_scanned} plugin Python files scanned"
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# T2: API Surface Inventory
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_t2a_api_surface_inventory() -> TestResult:
|
||||
"""
|
||||
Document the API surface area.
|
||||
|
||||
This app intentionally has no authentication (local-only Raspberry Pi
|
||||
design, documented in web_interface/app.py). This test produces an
|
||||
inventory for audit purposes and warns only if the design-intent comment
|
||||
is removed from app.py (which would indicate someone deleted the rationale
|
||||
without adding auth, rather than a deliberate undocumented change).
|
||||
"""
|
||||
api_file = PROJECT_ROOT / "web_interface" / "blueprints" / "api_v3.py"
|
||||
app_file = PROJECT_ROOT / "web_interface" / "app.py"
|
||||
|
||||
if not api_file.exists():
|
||||
return TestResult("T2a", "WARNING", "api_v3.py not found", str(api_file))
|
||||
|
||||
api_content = api_file.read_text(encoding="utf-8")
|
||||
routes = re.findall(r"@api_v3\.route\('([^']+)'", api_content)
|
||||
|
||||
csrf_documented = False
|
||||
if app_file.exists():
|
||||
app_content = app_file.read_text(encoding="utf-8")
|
||||
csrf_documented = "CSRF protection disabled for local-only" in app_content
|
||||
|
||||
summary = (
|
||||
f"{len(routes)} API routes in api_v3.py. "
|
||||
f"No auth decorators (intentional local-only design). "
|
||||
f"CSRF disabled: {'YES — design intent documented in app.py' if csrf_documented else 'YES — but design intent comment NOT found in app.py'}. "
|
||||
f"Rate limiting: 1000/min."
|
||||
)
|
||||
|
||||
if not csrf_documented:
|
||||
return TestResult(
|
||||
"T2a", "WARNING",
|
||||
"CSRF is disabled but the design-intent comment is missing from app.py",
|
||||
"Add the rationale comment back, or add proper CSRF protection if "
|
||||
"the app is now internet-facing"
|
||||
)
|
||||
|
||||
return TestResult("T2a", "INFO", "API surface documented", summary)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# T3: Secrets & Credential Handling
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Patterns that suggest real credentials (must be >8 chars, not placeholders)
|
||||
_SECRET_PATTERNS = [
|
||||
(r'(?i)password\s*=\s*["\'](?!none|empty|placeholder|example|test|default|""|'')[^"\']{8,}["\']', "WARNING"),
|
||||
(r'(?i)api[_-]?key\s*=\s*["\'](?!none|empty|placeholder|YOUR_|example|test)[^"\']{16,}["\']', "WARNING"),
|
||||
(r'(?i)secret\s*=\s*["\'](?!none|empty|placeholder|YOUR_|example|test)[^"\']{16,}["\']', "WARNING"),
|
||||
# Real GitHub token pattern
|
||||
(r'ghp_[a-zA-Z0-9]{36}', "CRITICAL"),
|
||||
# Generic long bearer tokens
|
||||
(r'Bearer\s+[a-zA-Z0-9\-_\.]{32,}', "WARNING"),
|
||||
]
|
||||
|
||||
_TEMPLATE_SKIP_STRINGS = [
|
||||
"YOUR_", "PLACEHOLDER", "_HERE", "example.com", "config_secrets.template",
|
||||
"prove_security", # this file itself
|
||||
]
|
||||
|
||||
_SCAN_DIRS = ["src", "web_interface", "scripts"]
|
||||
|
||||
|
||||
def test_t3a_hardcoded_secrets() -> TestResult:
|
||||
"""Scan source code for hardcoded credentials."""
|
||||
violations: list[str] = []
|
||||
|
||||
for dir_name in _SCAN_DIRS:
|
||||
scan_dir = PROJECT_ROOT / dir_name
|
||||
if not scan_dir.exists():
|
||||
continue
|
||||
for py_file in scan_dir.rglob("*.py"):
|
||||
# Skip test files and this script
|
||||
if "test" in str(py_file).lower() or "prove_security" in str(py_file):
|
||||
continue
|
||||
try:
|
||||
content = py_file.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
for pattern, severity in _SECRET_PATTERNS:
|
||||
for match in re.finditer(pattern, content):
|
||||
line_content = match.group(0)
|
||||
# Skip lines containing template placeholder strings
|
||||
if any(skip in line_content for skip in _TEMPLATE_SKIP_STRINGS):
|
||||
continue
|
||||
rel = py_file.relative_to(PROJECT_ROOT)
|
||||
line_no = content[: match.start()].count("\n") + 1
|
||||
violations.append(
|
||||
f"[{severity}] {rel}:{line_no} — {line_content[:60]}"
|
||||
)
|
||||
|
||||
critical_violations = [v for v in violations if "[CRITICAL]" in v]
|
||||
if critical_violations:
|
||||
return TestResult(
|
||||
"T3a", "CRITICAL",
|
||||
f"Hardcoded secrets found ({len(critical_violations)} critical)",
|
||||
"; ".join(critical_violations[:5])
|
||||
)
|
||||
if violations:
|
||||
return TestResult(
|
||||
"T3a", "WARNING",
|
||||
f"Potential hardcoded secrets found ({len(violations)} instance(s))",
|
||||
"; ".join(violations[:5])
|
||||
)
|
||||
|
||||
return TestResult("T3a", "PASS", "No hardcoded secrets detected",
|
||||
f"Scanned {', '.join(_SCAN_DIRS)}")
|
||||
|
||||
|
||||
def test_t3b_plaintext_password_storage() -> TestResult:
|
||||
"""
|
||||
Check for user account password storage without hashing.
|
||||
|
||||
The LEDMatrix app has no user account system, so this should produce INFO.
|
||||
It would only CRITICAL if someone added user auth and stored passwords without hashing.
|
||||
|
||||
We require all three of: a password *variable assignment or DB operation*,
|
||||
a clear storage call (INSERT / db commit / ORM save), and no hashing lib present
|
||||
— to avoid false positives from files that contain 'password' for WiFi handling
|
||||
and '.save()' for image/file saving in unrelated functions.
|
||||
"""
|
||||
hashing_libs = ["bcrypt", "argon2", "pbkdf2", "scrypt",
|
||||
"generate_password_hash", "hashpw", "make_password"]
|
||||
# Patterns that indicate password being stored in a database / ORM context.
|
||||
# Must be specific enough to avoid matching set.add(), file.save(), etc.
|
||||
db_storage_patterns = ["INSERT INTO", "db.session", "session.add(", "session.commit(", "orm.save"]
|
||||
|
||||
password_storage_found = False
|
||||
|
||||
for dir_name in _SCAN_DIRS:
|
||||
scan_dir = PROJECT_ROOT / dir_name
|
||||
if not scan_dir.exists():
|
||||
continue
|
||||
for py_file in scan_dir.rglob("*.py"):
|
||||
try:
|
||||
content = py_file.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
# Require DB/ORM context specifically — not just any .save() call
|
||||
if ("password" in content.lower() and
|
||||
any(store in content for store in db_storage_patterns) and
|
||||
not any(h in content for h in hashing_libs)):
|
||||
password_storage_found = True
|
||||
|
||||
if password_storage_found:
|
||||
return TestResult(
|
||||
"T3b", "CRITICAL",
|
||||
"Potential plaintext password storage in database/ORM detected",
|
||||
"Found password + database storage operations without a recognized hashing library"
|
||||
)
|
||||
|
||||
return TestResult("T3b", "INFO",
|
||||
"No plaintext password storage detected",
|
||||
"App has no user account system — expected result")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# T4: Path Traversal
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_t4a_path_traversal() -> TestResult:
|
||||
"""
|
||||
Verify static file serving uses send_from_directory (safe) rather than
|
||||
open() with user-supplied paths. Also checks for extractall() calls that
|
||||
lack the is_relative_to() guard.
|
||||
"""
|
||||
issues: list[str] = []
|
||||
|
||||
app_file = PROJECT_ROOT / "web_interface" / "app.py"
|
||||
if app_file.exists():
|
||||
content = app_file.read_text(encoding="utf-8")
|
||||
# The file-serve route should use send_from_directory or commonpath
|
||||
if "send_from_directory" not in content and "commonpath" not in content:
|
||||
issues.append("app.py: file-serve routes may not use send_from_directory/commonpath")
|
||||
|
||||
# Check all extractall() calls have a preceding is_relative_to guard
|
||||
for py_file in (PROJECT_ROOT / "src").rglob("*.py"):
|
||||
try:
|
||||
content = py_file.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
if "extractall(" in content and "is_relative_to" not in content:
|
||||
rel = py_file.relative_to(PROJECT_ROOT)
|
||||
issues.append(f"{rel}: extractall() without is_relative_to() guard")
|
||||
|
||||
if issues:
|
||||
return TestResult(
|
||||
"T4a", "WARNING",
|
||||
f"Potential path traversal patterns found ({len(issues)})",
|
||||
"; ".join(issues)
|
||||
)
|
||||
|
||||
return TestResult("T4a", "PASS",
|
||||
"Path traversal mitigations verified",
|
||||
"send_from_directory/commonpath used for file serving; "
|
||||
"extractall() calls have is_relative_to() guards")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# T5: Auth Bypass Patterns
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_t5a_auth_bypass_patterns() -> TestResult:
|
||||
"""
|
||||
Look for broken auth bypass patterns — not the intentional no-auth design
|
||||
(T2a covers that), but patterns that suggest auth was INTENDED to exist
|
||||
but has an exploitable bypass: broad substring matching, debug-mode skips,
|
||||
or if-True conditions.
|
||||
"""
|
||||
bypass_signals = [
|
||||
(r'if\s+True\s*:', "if True: bypass"),
|
||||
(r'if\s+debug\s*:', "debug-mode auth skip"),
|
||||
(r'request\.path\s+in\s+', "substring path matching in auth (Huntarr pattern)"),
|
||||
(r'EXEMPT_ROUTES\s*=', "exempt routes list"),
|
||||
]
|
||||
|
||||
findings: list[str] = []
|
||||
|
||||
for dir_name in ["src", "web_interface"]:
|
||||
scan_dir = PROJECT_ROOT / dir_name
|
||||
if not scan_dir.exists():
|
||||
continue
|
||||
for py_file in scan_dir.rglob("*.py"):
|
||||
try:
|
||||
content = py_file.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
for pattern, label in bypass_signals:
|
||||
if re.search(pattern, content):
|
||||
# Only flag if the file also contains auth-related terms
|
||||
if any(auth in content.lower() for auth in
|
||||
["auth", "login", "authenticate", "token", "permission"]):
|
||||
rel = py_file.relative_to(PROJECT_ROOT)
|
||||
findings.append(f"{rel}: {label}")
|
||||
|
||||
if findings:
|
||||
return TestResult(
|
||||
"T5a", "WARNING",
|
||||
f"Potential auth bypass patterns found ({len(findings)})",
|
||||
"; ".join(findings[:5])
|
||||
)
|
||||
|
||||
return TestResult("T5a", "PASS",
|
||||
"No auth bypass patterns detected",
|
||||
"Checked src/ and web_interface/ for bypass signals")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# T6: Docker / Container Hardening
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_t6_docker_hardening() -> TestResult:
|
||||
"""Container security — skipped if no Dockerfile exists."""
|
||||
dockerfile = PROJECT_ROOT / "Dockerfile"
|
||||
if not dockerfile.exists():
|
||||
return TestResult("T6", "SKIP",
|
||||
"No Dockerfile found — container security scan not applicable",
|
||||
"If Docker support is added in future, enable hadolint/trivy scanning "
|
||||
"in .github/workflows/security-audit.yml")
|
||||
|
||||
content = dockerfile.read_text(encoding="utf-8")
|
||||
issues: list[str] = []
|
||||
|
||||
# Check for non-root USER directive
|
||||
user_lines = [l for l in content.splitlines() if l.strip().startswith("USER")]
|
||||
if not user_lines or user_lines[-1].strip() == "USER root":
|
||||
issues.append("Container runs as root — use USER directive to drop privileges")
|
||||
|
||||
# Check for pinned base image tags
|
||||
from_lines = [l for l in content.splitlines() if l.strip().startswith("FROM")]
|
||||
for from_line in from_lines:
|
||||
parts = from_line.split()
|
||||
if len(parts) >= 2:
|
||||
image = parts[1]
|
||||
if ":" not in image or image.endswith(":latest"):
|
||||
issues.append(f"Unpinned base image: {image}")
|
||||
|
||||
if issues:
|
||||
return TestResult("T6", "WARNING",
|
||||
f"Dockerfile hardening issues ({len(issues)})",
|
||||
"; ".join(issues))
|
||||
|
||||
return TestResult("T6", "PASS", "Dockerfile hardening checks passed", "")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Runner
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="LEDMatrix security proof tests",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--output", "-o", default=None,
|
||||
help="Write JSON results to this file")
|
||||
parser.add_argument("--verbose", "-v", action="store_true",
|
||||
help="Show details for each check")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("=" * 60)
|
||||
print("LEDMatrix Security Proof Tests")
|
||||
print(f"Project root: {PROJECT_ROOT}")
|
||||
print("=" * 60)
|
||||
|
||||
all_results: list[TestResult] = []
|
||||
|
||||
# Run all test groups
|
||||
all_results.append(test_t1a_zip_slip_protection())
|
||||
all_results.extend(test_t1b_dangerous_plugin_calls())
|
||||
all_results.append(test_t2a_api_surface_inventory())
|
||||
all_results.append(test_t3a_hardcoded_secrets())
|
||||
all_results.append(test_t3b_plaintext_password_storage())
|
||||
all_results.append(test_t4a_path_traversal())
|
||||
all_results.append(test_t5a_auth_bypass_patterns())
|
||||
all_results.append(test_t6_docker_hardening())
|
||||
|
||||
# Print results
|
||||
print()
|
||||
for r in all_results:
|
||||
line = f" {r.icon} [{r.severity:<8}] {r.test_id}: {r.message}"
|
||||
print(line)
|
||||
if args.verbose and r.details:
|
||||
print(f" {r.details}")
|
||||
|
||||
# Tally
|
||||
critical = [r for r in all_results if r.severity == "CRITICAL"]
|
||||
warnings = [r for r in all_results if r.severity == "WARNING"]
|
||||
passed = [r for r in all_results if r.severity == "PASS"]
|
||||
skipped = [r for r in all_results if r.severity == "SKIP"]
|
||||
|
||||
print()
|
||||
print(f" Results: {len(passed)} PASS {len(warnings)} WARN "
|
||||
f"{len(critical)} CRITICAL {len(skipped)} SKIP")
|
||||
|
||||
# Write JSON output
|
||||
if args.output:
|
||||
output_data = [r.to_dict() for r in all_results]
|
||||
Path(args.output).write_text(
|
||||
json.dumps(output_data, indent=2), encoding="utf-8"
|
||||
)
|
||||
print(f" Results written to: {args.output}")
|
||||
|
||||
if critical:
|
||||
print(f"\n 🚨 {len(critical)} CRITICAL issue(s) found — blocking")
|
||||
return 1
|
||||
|
||||
if warnings:
|
||||
print(f"\n ⚠️ {len(warnings)} warning(s) found — non-blocking")
|
||||
|
||||
print("\n ✅ All checks passed (warnings are non-blocking)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -17,7 +17,6 @@ import os
|
||||
import json
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Sequence, Union
|
||||
|
||||
# Add project root to path
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -28,49 +27,15 @@ os.environ['EMULATOR'] = 'true'
|
||||
|
||||
# Import logger after path setup so src.logging_config is importable
|
||||
from src.logging_config import get_logger # noqa: E402
|
||||
from src.plugin_system.testing.loading import ( # noqa: E402
|
||||
find_plugin_dir, load_manifest, load_config_defaults,
|
||||
)
|
||||
logger = get_logger("[Render Plugin]")
|
||||
|
||||
MIN_DIMENSION = 1
|
||||
MAX_DIMENSION = 512
|
||||
|
||||
|
||||
def find_plugin_dir(plugin_id: str, search_dirs: Sequence[Union[str, Path]]) -> Optional[Path]:
|
||||
"""Find a plugin directory by searching multiple paths."""
|
||||
from src.plugin_system.plugin_loader import PluginLoader
|
||||
loader = PluginLoader()
|
||||
for search_dir in search_dirs:
|
||||
search_path = Path(search_dir)
|
||||
if not search_path.exists():
|
||||
continue
|
||||
result = loader.find_plugin_directory(plugin_id, search_path)
|
||||
if result:
|
||||
return Path(result)
|
||||
return None
|
||||
|
||||
|
||||
def load_manifest(plugin_dir: Path) -> Dict[str, Any]:
|
||||
"""Load and return manifest.json from plugin directory."""
|
||||
manifest_path = plugin_dir / 'manifest.json'
|
||||
if not manifest_path.exists():
|
||||
raise FileNotFoundError(f"No manifest.json in {plugin_dir}")
|
||||
with open(manifest_path, 'r') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def load_config_defaults(plugin_dir: Path) -> Dict[str, Any]:
|
||||
"""Extract default values from config_schema.json."""
|
||||
schema_path = plugin_dir / 'config_schema.json'
|
||||
if not schema_path.exists():
|
||||
return {}
|
||||
with open(schema_path, 'r') as f:
|
||||
schema = json.load(f)
|
||||
defaults: Dict[str, Any] = {}
|
||||
for key, prop in schema.get('properties', {}).items():
|
||||
if 'default' in prop:
|
||||
defaults[key] = prop['default']
|
||||
return defaults
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""Load a plugin, call update() + display(), and save the result as a PNG image."""
|
||||
parser = argparse.ArgumentParser(description='Render a plugin display to a PNG image')
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
"""
|
||||
Background Cache Mixin for Sports Managers
|
||||
|
||||
This mixin provides common caching functionality to eliminate code duplication
|
||||
across all sports managers. It implements the background service cache pattern
|
||||
where Recent/Upcoming managers consume data from the background service cache.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Dict, Optional, Any, Callable
|
||||
|
||||
|
||||
class BackgroundCacheMixin:
|
||||
"""
|
||||
Mixin class that provides background service cache functionality to sports managers.
|
||||
|
||||
This mixin eliminates code duplication by providing a common implementation
|
||||
for the background service cache pattern used across all sports managers.
|
||||
|
||||
Note: For non-sports managers (weather, stocks, news, etc.), use
|
||||
GenericCacheMixin instead. See src/generic_cache_mixin.py for details.
|
||||
"""
|
||||
|
||||
def _fetch_data_with_background_cache(self,
|
||||
sport_key: str,
|
||||
api_fetch_method: Callable,
|
||||
live_manager_class: type = None) -> Optional[Dict]:
|
||||
"""
|
||||
Common logic for fetching data with background service cache support.
|
||||
|
||||
This method implements the background service cache pattern:
|
||||
1. Live managers always fetch fresh data
|
||||
2. Recent/Upcoming managers try background cache first
|
||||
3. Fallback to direct API call if background data unavailable
|
||||
|
||||
Args:
|
||||
sport_key: Sport identifier (e.g., 'nba', 'nfl', 'ncaa_fb')
|
||||
api_fetch_method: Method to call for direct API fetch
|
||||
live_manager_class: Class to check if this is a live manager
|
||||
|
||||
Returns:
|
||||
Cached or fresh data from API
|
||||
"""
|
||||
start_time = time.time()
|
||||
cache_hit = False
|
||||
cache_source = None
|
||||
|
||||
try:
|
||||
# For Live managers, always fetch fresh data
|
||||
if live_manager_class and isinstance(self, live_manager_class):
|
||||
self.logger.info(f"[{sport_key.upper()}] Live manager - fetching fresh data")
|
||||
result = api_fetch_method(use_cache=False)
|
||||
cache_source = "live_fresh"
|
||||
else:
|
||||
# For Recent/Upcoming managers, try background service cache first
|
||||
cache_key = self.cache_manager.generate_sport_cache_key(sport_key)
|
||||
|
||||
# Check if background service has fresh data
|
||||
if self.cache_manager.is_background_data_available(cache_key, sport_key):
|
||||
cached_data = self.cache_manager.get_background_cached_data(cache_key, sport_key)
|
||||
if cached_data:
|
||||
self.logger.info(f"[{sport_key.upper()}] Using background service cache for {cache_key}")
|
||||
result = cached_data
|
||||
cache_hit = True
|
||||
cache_source = "background_cache"
|
||||
else:
|
||||
self.logger.warning(f"[{sport_key.upper()}] Background cache check passed but no data returned for {cache_key}")
|
||||
result = None
|
||||
cache_source = "background_miss"
|
||||
else:
|
||||
self.logger.info(f"[{sport_key.upper()}] Background data not available for {cache_key}")
|
||||
result = None
|
||||
cache_source = "background_unavailable"
|
||||
|
||||
# Fallback to direct API call if background data not available
|
||||
if result is None:
|
||||
self.logger.info(f"[{sport_key.upper()}] Fetching directly from API for {cache_key}")
|
||||
result = api_fetch_method(use_cache=True)
|
||||
cache_source = "api_fallback"
|
||||
|
||||
# Record performance metrics
|
||||
duration = time.time() - start_time
|
||||
self.cache_manager.record_fetch_time(duration)
|
||||
|
||||
# Log performance metrics
|
||||
self._log_fetch_performance(sport_key, duration, cache_hit, cache_source)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
duration = time.time() - start_time
|
||||
self.logger.error(f"[{sport_key.upper()}] Error in background cache fetch after {duration:.2f}s: {e}")
|
||||
self.cache_manager.record_fetch_time(duration)
|
||||
raise
|
||||
|
||||
def _log_fetch_performance(self, sport_key: str, duration: float, cache_hit: bool, cache_source: str):
|
||||
"""
|
||||
Log detailed performance metrics for fetch operations.
|
||||
|
||||
Args:
|
||||
sport_key: Sport identifier
|
||||
duration: Fetch operation duration in seconds
|
||||
cache_hit: Whether this was a cache hit
|
||||
cache_source: Source of the data (background_cache, api_fallback, etc.)
|
||||
"""
|
||||
# Log basic performance info
|
||||
self.logger.info(f"[{sport_key.upper()}] Fetch completed in {duration:.2f}s "
|
||||
f"(cache_hit={cache_hit}, source={cache_source})")
|
||||
|
||||
# Log detailed metrics every 10 operations
|
||||
if hasattr(self, '_fetch_count'):
|
||||
self._fetch_count += 1
|
||||
else:
|
||||
self._fetch_count = 1
|
||||
|
||||
if self._fetch_count % 10 == 0:
|
||||
metrics = self.cache_manager.get_cache_metrics()
|
||||
self.logger.info(f"[{sport_key.upper()}] Cache Performance Summary - "
|
||||
f"Hit Rate: {metrics['cache_hit_rate']:.2%}, "
|
||||
f"Background Hit Rate: {metrics['background_hit_rate']:.2%}, "
|
||||
f"API Calls Saved: {metrics['api_calls_saved']}")
|
||||
|
||||
def get_cache_performance_summary(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get cache performance summary for this manager.
|
||||
|
||||
Returns:
|
||||
Dictionary containing cache performance metrics
|
||||
"""
|
||||
return self.cache_manager.get_cache_metrics()
|
||||
|
||||
def log_cache_performance(self):
|
||||
"""Log current cache performance metrics."""
|
||||
self.cache_manager.log_cache_metrics()
|
||||