mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-10-18 20:59:14 +00:00
Compare commits
39
Commits
2f64dbc48c
...
v3.3.1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
32d637a446 | ||
|
|
bc2dbf3824 | ||
|
|
cb0545ecb3 | ||
|
|
300cdaa250 | ||
|
|
10da2f97fd | ||
|
|
92f9d06af9 | ||
|
|
6e361e05cc | ||
|
|
a686932c7e | ||
|
|
154525beb8 | ||
|
|
9b522d412c | ||
|
|
cbc540a679 | ||
|
|
4aeb0033e0 | ||
|
|
eae063700f | ||
|
|
af96bd5cb0 | ||
|
|
5e5979973e | ||
|
|
bdced206dc | ||
|
|
39e7f8cbe0 | ||
|
|
f90638a9ec | ||
|
|
333fd17d28 | ||
|
|
a4a55a23fc | ||
|
|
085fb93a87 | ||
|
|
c321b94085 | ||
|
|
5a1f121e6b | ||
|
|
6138a3cbef | ||
|
|
568cb6d77f | ||
|
|
6b74506695 | ||
|
|
5f29243e87 | ||
|
|
1fbe244e49 | ||
|
|
863e4a1ecd | ||
|
|
9c0c0dc851 | ||
|
|
71739d85d1 | ||
|
|
cc258aaffd | ||
|
|
fe5a3aa99d | ||
|
|
10e75b977f | ||
|
|
cf0a551f7b | ||
|
|
9018fa23cd | ||
|
|
0c5b9c57d3 | ||
|
|
9083df9f5c | ||
|
|
0901d044d3 |
@@ -72,4 +72,4 @@ jobs:
|
||||
--ignore=test/plugins \
|
||||
--cov=src --cov=web_interface \
|
||||
--cov-report=term \
|
||||
--cov-fail-under=48
|
||||
--cov-fail-under=52
|
||||
|
||||
@@ -17,6 +17,81 @@ release that ships it.
|
||||
accepts both, but the store flags the old spelling as deprecated
|
||||
(`store_manager.py`) and only the new one is in `schema/manifest_schema.json`.
|
||||
|
||||
## 3.3.0
|
||||
|
||||
**The release the sports scoreboards floor on to delete their bundled copies.**
|
||||
3.2.0 shipped the unified sports library and made `ledmatrix_min_version`
|
||||
enforceable; this ships the last three shared modules and completes the store
|
||||
gate, so a scoreboard can now floor here and carry no fallback at all.
|
||||
|
||||
New modules a plugin may import via `src.*` and floor on 3.3.0 for:
|
||||
|
||||
- `src/common/sports_card.py` — settings, colour, font and date helpers for a
|
||||
scoreboard's `game_renderer.py`. Free functions taking `config`/`fonts`
|
||||
explicitly, so nothing about the caller's class is assumed.
|
||||
- `src/common/sports_game_renderer.py` — `SportsGameRendererMixin`: scroll/Vegas
|
||||
card geometry (centre gap, logo slot and cache key, layout offsets, the
|
||||
upcoming-card date and time layout). No `__init__` and no state, so adoption
|
||||
is one line on the class statement.
|
||||
- `src/common/sports_shared.py` — `SportsCoreSharedMixin`,
|
||||
`SportsLiveSharedMixin`, `SportsRecentSharedMixin`: the `sports.py` bodies
|
||||
byte-identical in all eight lineage-sharing scoreboards.
|
||||
|
||||
All three sit under `src/common/` rather than `src/base_classes/sports/`,
|
||||
deliberately: importing that package pulls `core.py` → `DisplayManager` →
|
||||
`rgbmatrix`, and these are pure logic. Plugins importing them must not acquire a
|
||||
hardware dependency.
|
||||
|
||||
Three notes for anyone adopting `sports_shared`:
|
||||
|
||||
- `SportsRecentSharedMixin` defines `__init__`. Its bare `super()` binds to the
|
||||
mixin, so it reaches the host only when the mixin is listed **first** in the
|
||||
bases. Reversing that order silently skips the host constructor.
|
||||
- Methods that resolve the plugin's `config_schema.json` use `_plugin_dir()`,
|
||||
which walks the MRO rather than reading `__file__` — `__file__` is now
|
||||
`src/common/`. It walks because `SportsCore` is an ABC: a subclass built with
|
||||
`type(name, bases, ns)` reports `__module__` as `"abc"`.
|
||||
- `_get_timezone`, `_extract_game_details` and `_fetch_data` are byte-identical
|
||||
across the eight but stay in the plugins. The first binds a per-plugin
|
||||
timezone module whose contents differ; the other two are the abstract stubs
|
||||
that define the sport.
|
||||
|
||||
**The store's compatibility gate is now on every registry-managed route.** 3.2.0
|
||||
gated `install_plugin`. This release gates the git-pull update path and
|
||||
`install_from_url`, so a plugin whose floor the core cannot meet is refused
|
||||
after download with no partial directory left behind.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **ESPN 403s.** `site.api` began rejecting the User-Agent strings this repo
|
||||
sent on 2026-08-04; every shared-data-source scoreboard returned
|
||||
`403 Forbidden`. Requests now send an identifying token with a project URL —
|
||||
browser-style strings and bare custom tokens are both refused.
|
||||
- **Low-memory boards becoming unreachable under load** while still answering
|
||||
pings and serving the web UI. Fetched payloads are released after delivery
|
||||
rather than pinned on the completed request for up to an hour, malloc arenas
|
||||
are capped, and log volume and SD writes are reduced. Available memory is now
|
||||
reported in Tools diagnostics.
|
||||
- **A failed logo download pinning a team to a grey box**: the placeholder was
|
||||
written under the real logo's filename, so later attempts found a file and
|
||||
reported success without retrying.
|
||||
- **A plugin enabled but never loaded is retried** rather than staying absent
|
||||
with `error = null`.
|
||||
- `ttl` now controls cache expiry; abandoned cache writes no longer leave temp
|
||||
files; one cache-cleanup thread per directory rather than per manager.
|
||||
- Odds are fetched for the games displayed, not the whole schedule window, and a
|
||||
stalled ESPN no longer stalls the whole plugin update.
|
||||
- Array-item secrets are no longer wiped or logged.
|
||||
- A restored backup matches the device it was taken from.
|
||||
- The web UI reports the real error instead of "unknown", rejects non-finite
|
||||
JSON numbers, and stops checkbox groups posting back hidden options.
|
||||
|
||||
### Added
|
||||
|
||||
- `display.hardware.orientation` for panels mounted upside down.
|
||||
- Vegas keeps live content in the ticker rather than being preempted by it.
|
||||
- Schemas can label enum dropdown options.
|
||||
|
||||
## 3.2.0
|
||||
|
||||
**The first release shipping the unified sports library.** This is the version
|
||||
|
||||
@@ -161,9 +161,11 @@ The system supports live, recent, and upcoming game information for multiple spo
|
||||
### LED Matrix Panels
|
||||
(2x in a horizontal chain is recommended)
|
||||
- [Adafruit 64×32](https://www.adafruit.com/product/2278) – designed for 128×32 but works with dynamic scaling on many displays (pixel pitch is user preference)
|
||||
**Warning: Lately the Waveshare Panels have had different variations - only some are compatible with this project. I hope to identify what is different to fix it but so far there is a decent chance you get a mis-matched set of panels if you don't buy them all at once! **
|
||||
- [Waveshare 64×32](https://amzn.to/3Kw55jK) - Does not require E addressable pad
|
||||
- [Waveshare 96×48](https://amzn.to/4bydNcv) – higher resolution, requires soldering the **E addressable pad** on the [Adafruit RGB Bonnet](https://www.adafruit.com/product/3211) to “8” **OR** toggling the DIP switch on the Adafruit Triple LED Matrix Bonnet *(no soldering required!)*
|
||||
> Amazon Affiliate Link – ChuckBuilds receives a small commission on purchases
|
||||
- [Waveshare 96×48](https://amzn.to/4bydNcv) – higher resolution, requires soldering the **E addressable pad** on the [Adafruit RGB Bonnet](https://www.adafruit.com/product/3211) to “8” **OR** toggling the DIP switch on the Adafruit Triple LED Matrix Bonnet *(no soldering required!)*
|
||||
- There are some Panels on Aliexpress that have worked fine for me, shop around! I think Adafruit is probably the "safest" but they do have some limitation on resolution and layout.
|
||||
> Amazon Affiliate Links – ChuckBuilds receives a small commission on purchases
|
||||
|
||||
### Power Supply
|
||||
- [5V 4A DC Power Supply](https://www.adafruit.com/product/658) (good for 2 -3 displays, depending on brightness and pixel density, you'll need higher amperage for more)
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 23 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 467 B |
Binary file not shown.
|
Before Width: | Height: | Size: 39 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 105 KiB |
@@ -18,7 +18,7 @@ tooling against it.
|
||||
| `web_display_autostart` | bool, `true` | Whether the web interface service starts with the system | `scripts/utils/start_web_conditionally.py` |
|
||||
| `timezone` | string, `"America/New_York"` | IANA timezone for schedules and displays | `ConfigManager.get_timezone()` |
|
||||
| `target_fps` | int, `100` | Frame-rate ceiling for plugin rendering | `src/plugin_system/base_plugin.py`, `src/common/sports_scroll.py` |
|
||||
| `location` | object | `city` / `state` / `country`, offered to plugins that need a location (weather, etc.) | plugins via merged config |
|
||||
| `location` | object | `city` / `state` / `country`. Supplies the **default** for a plugin's own `location_city` / `location_state` / `location_country` setting, so weather, radar and friends follow this device without being configured twice. A value saved on the plugin itself still overrides it. | `SchemaManager.apply_device_location()`, then plugins via merged config |
|
||||
|
||||
## `schedule` — display on/off hours
|
||||
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
# Running on Low-Memory Boards
|
||||
|
||||
Applies to the Pi Zero 2 W (512 MB), Pi 3 / 3B+ (1 GB), and the 1 GB Pi 4.
|
||||
If your board has 2 GB or more you can skip this document.
|
||||
|
||||
## The failure this prevents
|
||||
|
||||
The display process is the largest thing on the board. On a 1 GB Pi 3B+ with
|
||||
around 20 plugins enabled it settles near **600 MB of 905 MB usable**, leaving
|
||||
under 200 MB of headroom for everything else.
|
||||
|
||||
When that headroom runs out, the board does not crash cleanly. `fork()` starts
|
||||
failing, and because a new process is needed to do almost anything, the
|
||||
symptoms look nothing like "out of memory":
|
||||
|
||||
| What you see | Why |
|
||||
|---|---|
|
||||
| SSH accepts the connection then closes it instantly, before any banner | `sshd` forks a session per connection; the fork fails |
|
||||
| The web UI still responds quickly | Already running, serves from existing threads, forks nothing |
|
||||
| Ping is perfect, 0% loss | Handled entirely in the kernel |
|
||||
| The panel is dark | The display process was killed and cannot be respawned |
|
||||
| The clock is wrong after the next boot | `fake-hwclock`'s periodic save is a scheduled job, and it cannot fork either |
|
||||
|
||||
The board looks healthy from the outside and cannot be logged into. Only a
|
||||
power cycle clears it. If you are here because SSH stopped working, also see
|
||||
[SSH_UNAVAILABLE_AFTER_INSTALL.md](SSH_UNAVAILABLE_AFTER_INSTALL.md), which
|
||||
covers the more common cause (AP mode).
|
||||
|
||||
## Check your headroom
|
||||
|
||||
```bash
|
||||
free -m
|
||||
ps -eo rss,comm --sort=-rss | head -5
|
||||
```
|
||||
|
||||
If `MemAvailable` is under ~150 MB while the display is running, you are close
|
||||
to the edge. To watch it over time:
|
||||
|
||||
```bash
|
||||
watch -n 30 'free -m | head -2'
|
||||
```
|
||||
|
||||
Available memory that falls steadily rather than holding flat means you will
|
||||
reach the wall; it is a question of when.
|
||||
|
||||
## What to do
|
||||
|
||||
**1. Enable the memory cgroup controller.** Without it, the `MemoryMax=85%` in
|
||||
`systemd/ledmatrix.service` is accepted by systemd and silently ignored, so the
|
||||
service has no ceiling and a runaway takes the whole board down instead of just
|
||||
restarting. Raspberry Pi firmware disables this controller by default.
|
||||
|
||||
`first_time_install.sh` does this for you. To check it took effect:
|
||||
|
||||
```bash
|
||||
grep memory /sys/fs/cgroup/cgroup.controllers
|
||||
```
|
||||
|
||||
If that prints nothing, add `cgroup_enable=memory cgroup_memory=1` to the
|
||||
kernel command line and reboot. Edit whichever file your image uses —
|
||||
`/boot/firmware/cmdline.txt` on current Raspberry Pi OS, `/boot/cmdline.txt` on
|
||||
older layouts (the installer checks the first and falls back to the second).
|
||||
Everything must stay on a single line.
|
||||
|
||||
This changes the failure mode from "the board becomes unreachable" to "the
|
||||
display service restarts". It is a safety net, not a fix.
|
||||
|
||||
**2. Run fewer plugins.** This is the actual remedy. Every enabled plugin costs
|
||||
memory permanently — its module, its parsed config, and its cached API
|
||||
responses. On a 512 MB or 1 GB board, keep the enabled set small and prefer
|
||||
plugins that poll infrequently.
|
||||
|
||||
**3. Lower the cache ceiling.** The in-memory cache is sized from total RAM
|
||||
(150 entries at 1 GB and below, up to 1500 at 8 GB). To go lower still:
|
||||
|
||||
```ini
|
||||
# /etc/systemd/system/ledmatrix.service.d/override.conf
|
||||
[Service]
|
||||
Environment=LEDMATRIX_CACHE_MAX_ENTRIES=75
|
||||
```
|
||||
|
||||
Writing the file does not change the running service. Reload systemd and
|
||||
restart it:
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart ledmatrix
|
||||
```
|
||||
|
||||
Fewer entries means more API calls, so lower this only while you are actually
|
||||
short of memory.
|
||||
|
||||
**4. Consider `MemoryHigh`.** `MemoryMax` kills and restarts. `MemoryHigh`
|
||||
throttles and reclaims instead, which is gentler — but on a board where the
|
||||
process genuinely wants more than the limit, sustained reclaim can stall the
|
||||
render loop and show as visible stutter on the panel. Add it only if you prefer
|
||||
degraded output to a restart:
|
||||
|
||||
```ini
|
||||
[Service]
|
||||
MemoryHigh=70%
|
||||
```
|
||||
|
||||
## Keep your logs
|
||||
|
||||
These images default to volatile journald storage, so every reboot destroys the
|
||||
logs — including the ones explaining why the board rebooted. `first_time_install.sh`
|
||||
enables persistent storage capped at 64 MB. To confirm:
|
||||
|
||||
```bash
|
||||
journalctl --list-boots
|
||||
```
|
||||
|
||||
More than one boot listed means logs are surviving reboots. If only one is
|
||||
listed, journald is still writing to `/run` (tmpfs).
|
||||
@@ -14,6 +14,7 @@ the one-shot installer. The pages here go deeper.
|
||||
5. [TROUBLESHOOTING.md](TROUBLESHOOTING.md) — common issues and fixes
|
||||
6. [SSH_UNAVAILABLE_AFTER_INSTALL.md](SSH_UNAVAILABLE_AFTER_INSTALL.md) — recovering SSH after install
|
||||
7. [CONFIG_DEBUGGING.md](CONFIG_DEBUGGING.md) — diagnosing config problems
|
||||
8. [LOW_MEMORY_BOARDS.md](LOW_MEMORY_BOARDS.md) — Pi Zero 2 W / 3B+ / 1GB Pi 4 memory limits
|
||||
|
||||
## I want to write a plugin
|
||||
|
||||
|
||||
+169
-37
@@ -27,7 +27,7 @@ These are independent concerns. Conflating them is what produces god classes.
|
||||
| Plugin loads on a core that predates a module | Guarded import with a bundled fallback (`try: from src.X import Y / except ModuleNotFoundError: from y import Y`) |
|
||||
| Plugin loads on a core that predates a *method* | Capability probing — `hasattr(SportsCore, "_detect_stale_games")` — never a version comparison. The loader's compat check is advisory-only (it logs and continues), so probing is the real protection. |
|
||||
| Core changes never break a plugin's rendering | The **view-model contract**: `_extract_game_details_common` returns a dict whose `GUARANTEED_KEYS` are frozen by `test/test_skin_system.py::TestViewModelContract`. Keys may be added, never renamed or removed. |
|
||||
| A plugin can drop its bundled copy safely | The **sunset rule**: its manifest must floor `ledmatrix_min_version` at the first core release shipping the module (recorded in `CHANGELOG.md`) — *necessary but not sufficient*. Nothing enforces that floor today, so the copy also waits for the B6 gate below. |
|
||||
| A plugin can drop its bundled copy safely | The **sunset rule**: its manifest must floor `ledmatrix_min_version` at the first core release shipping the module (recorded in `CHANGELOG.md`) — *necessary but not sufficient*. The store enforces that floor on every registry-managed install and on both supported update paths (sideloading via `install_from_url` is not gated), but a floor cannot reach a user who never updates, so the copy also waits for the B6 gate below. |
|
||||
|
||||
The core API is **additive-only**. A method the plugins call is never removed or
|
||||
given a new required parameter; new behavior arrives as new methods with
|
||||
@@ -204,7 +204,7 @@ legacy compatibility rather than the mechanism.
|
||||
B0–B3 are merged and shipping in core 3.2.0. Everything that remains is
|
||||
**rollout**, and it splits into three phases with very different risk profiles.
|
||||
The original plan folded the last two together; they are separated here because
|
||||
one of them is safe by construction and the other is not.
|
||||
one of them cannot break a user on an old core and the other can.
|
||||
|
||||
| Phase | Scope | Status | Gate |
|
||||
|---|---|---|---|
|
||||
@@ -212,9 +212,9 @@ one of them is safe by construction and the other is not.
|
||||
| **B1** | Promote the nine universal methods; convert `sports.py` → package | ✅ | Characterization suite green; no behavior change intended |
|
||||
| **B2** | `CelebrationMixin` + rotation strategies as opt-in capabilities | ✅ | Non-adopters have zero new code in their MRO; strategies checked against verbatim plugin transcriptions |
|
||||
| **B3** | Upstream the scroll **orchestration** layer as `src/common/sports_scroll.py`, reading `global_config['target_fps']` natively | ✅ | Content building stays per-sport |
|
||||
| **B4** | Ship 3.2.0 *and* make version reporting trustworthy | ⏳ **next** | Tag, release, and `src.__version__` agree; compatibility gate merged |
|
||||
| **B5** | Adoption — guarded core imports: three pilots, then the remaining six. **Bundled copies stay.** | after B4 | Per plugin: harness + goldens byte-identical, then a device soak |
|
||||
| **B6** | Sunset — delete the bundled copies | **blocked** | B4's gate shipped *and* in users' hands (see below) |
|
||||
| **B4** | Ship 3.2.0 *and* make version reporting trustworthy | ✅ | Released 2026-08-03; tag, release and `src.__version__` agree; compatibility gate merged (#428, #431, #433) |
|
||||
| **B5** | Adoption — guarded core imports, all eight. **Bundled copies stay.** | ✅ | All eight adopted; harness byte-identical; see the B5 retrospective below — four shipped broken and were repaired in plugins #251 |
|
||||
| **B6** | Sunset — delete the bundled copies | ✅ | Ran 2026-09-01, all eight. Floors at 3.2.0; the store refuses on all three routes in (#431/#433, #508, #510). See "B6 — what actually happened" |
|
||||
|
||||
### B4 — what "ship 3.2.0" actually requires
|
||||
|
||||
@@ -234,6 +234,20 @@ a floor can be trusted against, and today it is not:
|
||||
compares the plugin's manifest version against the registry's
|
||||
`latest_version` and nothing else.
|
||||
|
||||
*Fixed, in two parts.* `install_plugin` gained the gate in #431/#433, which
|
||||
covers every registry-managed install and, through `_reinstall_with_rollback`,
|
||||
the update path that re-downloads.
|
||||
`update_plugin`'s git branch pulls in place and re-downloads nothing, so it
|
||||
stayed ungated until `_gate_pulled_commit` closed it — checked after the pull
|
||||
(the registry carries no floor field, so the incoming floor is unknowable
|
||||
before it) and undone with `git reset --hard` to the pre-pull commit. That
|
||||
route is rare in practice, since monorepo plugins install as archives; it was
|
||||
closed because the sunset rule in the plugins repo's
|
||||
`08-shared-sports-code.md` states as **condition 3** that the core enforces
|
||||
the floor "at install/update time", and B6 rests on that being true rather
|
||||
than merely written down. `install_from_url` — sideloading a plugin from a
|
||||
URL — is still ungated.
|
||||
|
||||
So B4 is: tag and release 3.2.0; make the tag, the release, and `__version__`
|
||||
agree, and keep them agreeing; reconsider the `< 2.0.0` skip; migrate manifests
|
||||
from `ledmatrix_min` to `ledmatrix_min_version`; and add the install/update
|
||||
@@ -262,13 +276,21 @@ default is the more restrictive.
|
||||
`compatible_versions`. No manifest still carries it, so there is nothing to
|
||||
migrate there.)
|
||||
|
||||
### B5 — adoption is safe by construction
|
||||
### B5 — the *fallback* is safe by construction; the modern path is not
|
||||
|
||||
The heading matters, because the unqualified version of this claim is false and
|
||||
this document proves it two sections down: four of the eight adopted plugins
|
||||
shipped with scroll mode broken on a 3.2.0 core. What is safe by construction is
|
||||
narrower than "adoption".
|
||||
|
||||
A plugin adopting core imports keeps its bundled copy and reaches it through the
|
||||
guarded import (see the Upgradability table above). On a core that ships the
|
||||
module the plugin uses core code; on one that doesn't it falls back and behaves
|
||||
exactly as it does today. There is no version of this step that breaks a user,
|
||||
which is why it does not wait for B6's gate.
|
||||
guarded import (see the Upgradability table above). On a core that doesn't ship
|
||||
the module the plugin falls back and behaves exactly as it does today. That
|
||||
fallback compatibility — and only that — is safe by construction. On a core that
|
||||
*does* ship the module, correctness is not automatic — object-level and scroll-mode validation
|
||||
(building both classes and comparing, per the retrospective below) is required
|
||||
to prove full behavior. There is no version of this step that breaks a user *on
|
||||
an old core*, which is why it does not wait for B6's gate.
|
||||
|
||||
The hockey scroll-display pilot is **already validated**: adopted against a core
|
||||
carrying 3.2.0, `scroll_display.py` went from 691 to 289 lines and all 16 harness
|
||||
@@ -297,7 +319,8 @@ been in users' hands long enough that the population running a core without it
|
||||
is small.** The bundled copies cost disk space; deleting them early costs
|
||||
scoreboards, silently. That trade is not close.
|
||||
|
||||
Before the first sunset, add a **compatibility regression test**. It has to
|
||||
Before the first sunset, add a **compatibility regression test**. **Built:**
|
||||
core `test/test_sports_sunset_matrix.py` (#505). It has to
|
||||
cover four cases, not one — B5's safety claim and B6's failure mode are
|
||||
different propositions and only the second is obvious:
|
||||
|
||||
@@ -321,35 +344,144 @@ gate rather than trusting the failure to be noticed.
|
||||
The same suite should exercise the install/update gate, since it is the other
|
||||
half of the guarantee.
|
||||
|
||||
### B6 — what actually happened
|
||||
|
||||
**Ran 2026-09-01, across all eight scoreboards.** Held from 2026-08-05 to
|
||||
2026-09-01 on the argument below, which is kept because the reasoning applies to
|
||||
the next module, not because it is still in force.
|
||||
|
||||
**The hold, and why it lifted.** The stated gate was evidence of 3.2.0 uptake —
|
||||
"a few months of it being the default download, or store-side install data".
|
||||
That evidence never arrived and could not: the core updates by
|
||||
`git pull --rebase`, so release-asset counts cannot measure uptake, and no
|
||||
store-side telemetry exists. What changed instead is that the *risk* the gate
|
||||
protected against was closed directly. The store now refuses a plugin whose
|
||||
floor exceeds the running core on **all three** routes in:
|
||||
|
||||
| route | gated by |
|
||||
|---|---|
|
||||
| `install_plugin` — every path that re-downloads, `_reinstall_with_rollback` included | #431, #433 |
|
||||
| `update_plugin`'s git branch — pulls in place, re-downloads nothing | #508 |
|
||||
| `install_from_url` — sideloading | #510 |
|
||||
|
||||
With all three closed a pre-3.2.0 user cannot receive a sunset plugin at all;
|
||||
they keep the version they already run. The population the hold existed to
|
||||
protect is protected by refusal rather than by a bundled copy — which is what
|
||||
the copy was standing in for.
|
||||
|
||||
**What shipped.** Eight plugins, ~5,800 lines of frozen fallback deleted. Each:
|
||||
copy removed, guarded import collapsed to a plain one, floor raised to 3.2.0,
|
||||
`test_core_fallback.py` rewritten as `test_core_scroll.py` asserting the sunset
|
||||
rather than the fallback. `scripts/check_scroll_adoption.py` gained
|
||||
`sunset_violations` and a `SUNSET_PLUGINS` set naming all eight, so a
|
||||
resurrected copy or a returned guard fails CI.
|
||||
|
||||
Delivered as plugins #346 (hockey, later folded into #351), #349 (football),
|
||||
#350 (baseball), #351 (the remaining six).
|
||||
|
||||
**Two things found by doing it, both worth carrying forward:**
|
||||
|
||||
- **Only one fallback held orchestration logic the core lacked.** baseball's
|
||||
`_configure_scroll_helper` reinterpreted `scroll_speed` as pixels-per-*frame*
|
||||
when `speed × delay` fell outside the 0.1–5.0 window — measured, 10–20×
|
||||
faster than configured for a speed between 1.0 and 5.0. Standardised onto the
|
||||
core's behaviour (honour the documented px/sec, clamp) rather than preserved.
|
||||
Every other difference across the eight was a docstring, an unreachable
|
||||
`scroll_helper is None` guard, or an equivalent diagnostic.
|
||||
- **Two tests had been leaning on the guard without anyone knowing.**
|
||||
`soccer/test_live_screens.py` stubbed `src` in a way that shadowed the core,
|
||||
so its guarded import fell back and the test had been exercising the *frozen
|
||||
copy* rather than the shipping class since B5. Before sunsetting anything else
|
||||
that carries a guarded core import, grep for tests that stub `src`.
|
||||
|
||||
**The floor-raising traps still apply** to any future sunset: four plugins
|
||||
declare their floor top-level, where editing `versions[0]` is a silent no-op,
|
||||
and the floor has three live spellings (`min_ledmatrix_version`,
|
||||
`requires.min_ledmatrix_version`, `versions[].ledmatrix_min_version`, plus
|
||||
deprecated `ledmatrix_min`). See
|
||||
`src/plugin_system/compatibility.py:declared_min_version` for the resolution
|
||||
order any floor-raising tool must reproduce — and note the name is **inverted**
|
||||
between the top level and `versions[]`.
|
||||
|
||||
**Still not adopted, deliberately:** `data_sources.py`, `game_renderer.py` and
|
||||
`base_odds_manager.py`. The standing decision held them until B6 closed; it now
|
||||
has, so they can be reconsidered — with B5's lesson applied, which is to build
|
||||
the object and diff rendered output rather than trust a static check.
|
||||
|
||||
### B5 retrospective — what the adoption actually cost
|
||||
|
||||
Recorded because it is the evidence behind the two decisions above, and because
|
||||
"the adoption went fine" is not what happened.
|
||||
|
||||
**Four of the eight shipped with scroll mode broken** on a 3.2.0 core, and were
|
||||
repaired in plugins-repo #251. The restructure lifted the content methods
|
||||
verbatim but left the state they read off `self` behind: separator-icon
|
||||
constants (hockey, basketball, lacrosse) and the game-renderer cache (afl).
|
||||
hockey/basketball/lacrosse could not construct the scroll display at all; afl
|
||||
raised inside `prepare_scroll_content`, which the core base *catches*, so its
|
||||
only symptom was scroll mode silently drawing nothing.
|
||||
|
||||
Three things are worth carrying forward:
|
||||
|
||||
- **The bundled fallback did not protect anyone from this.** The break was on
|
||||
the modern path, which the fallback never touches. Carrying the second copy
|
||||
bought nothing against the actual defect while creating the divergence that
|
||||
produced it. That is an argument *for* B6, not against it.
|
||||
- **Every gate was green.** The safety harness renders the scoreboard screens,
|
||||
not scroll mode; `test_core_fallback.py` checked that methods existed and that
|
||||
their *globals* resolved, and `self.NHL_SEPARATOR_ICON` is an attribute read,
|
||||
invisible to an AST scan for `Name` loads. The fix was to stop reasoning about
|
||||
source and **build the object**: construct both classes on both paths, compare
|
||||
the separator icons they end up with, and assert the adopted class ends up
|
||||
with every instance attribute the bundled one sets.
|
||||
- **Test what the change touches, not what is convenient to render.** Scroll
|
||||
mode had no coverage because the harness could not reach it. A comparison
|
||||
harness that renders the same games through both paths and diffs the pixels
|
||||
needs no per-sport knowledge of the right answer, only that adopting core code
|
||||
did not change it.
|
||||
|
||||
**The ledger.** Before adoption, eight duplicated copies totalled 5,685 lines.
|
||||
After adoption plus the frozen legacy copies it was 10,610; removing the dead
|
||||
inline duplication (plugins #252) brought it to roughly 8,620. B6 would take it
|
||||
to about 3,300 including the shared core module — some 2,400 fewer than before
|
||||
this project started. **Until B6 runs, the adoption is net negative on disk**,
|
||||
and its one delivered user-visible gain is that adopted plugins honour the
|
||||
global `target_fps` instead of hardcoding ~100 FPS.
|
||||
|
||||
### Decision: stop adopting further modules until B6 closes
|
||||
|
||||
`data_sources.py` (9 copies), `game_renderer.py` (8) and `base_odds_manager.py`
|
||||
are the obvious next candidates. **Do not adopt them yet.** Each adoption adds
|
||||
carrying cost — a second copy to keep in step — against a payoff that is
|
||||
contingent on B6, and B6 is gated on an installed base we cannot currently
|
||||
measure. Consolidate what is already committed; revisit when B6 does.
|
||||
|
||||
## What's next
|
||||
|
||||
In order. Each step is independently useful and independently revertible.
|
||||
Steps 1–5 of the original plan are **done**: 3.2.0 is tagged and published with
|
||||
a version number CI now asserts (#428), the compatibility gate is in
|
||||
`install_plugin` and reads `compatible_versions` as well as the floor
|
||||
(#431, #433), the newest manifest entry is required to use `ledmatrix_min_version`
|
||||
(plugins #244), and all eight plugins have adopted the scroll orchestration
|
||||
(plugins #245–#249, repaired in #251, tidied in #252).
|
||||
|
||||
1. **Tag and publish v3.2.0.** The code is already on `main` (`21825cbf`).
|
||||
Nothing else blocks this, and it is what makes `ledmatrix_min_version:
|
||||
"3.2.0"` refer to something real.
|
||||
2. **Make the version number honest.** Have the release process assert that the
|
||||
tag, the GitHub release, and `src.__version__` agree — a check in CI is
|
||||
cheaper than the confusion of the last two releases. Then revisit the
|
||||
`< 2.0.0` skip in `_warn_if_incompatible`, which currently silences the
|
||||
warning for the users who most need it.
|
||||
3. **Add the compatibility gate** to `StoreManager.install_plugin` and
|
||||
`.update_plugin`: refuse a plugin whose declared floor exceeds
|
||||
`src.__version__`, and surface the reason in the store UI rather than only
|
||||
the log. This is the single change that turns the floor from documentation
|
||||
into a guarantee, and B6 depends on it.
|
||||
4. **Migrate the manifests** to `ledmatrix_min_version`, and reconcile them with
|
||||
`compatible_versions` (see above — that field is the required, canonical one,
|
||||
and the gate does not read it yet). Currently 28 plugins spell the floor both
|
||||
ways across their `versions[]` entries, 12 use only the old spelling, and 2
|
||||
only the new. Scope the sweep to the nine sports plugins if a 42-plugin
|
||||
version-bump wave isn't worth it — but the `compatible_versions` half has to
|
||||
cover every manifest the gate can refuse, or define explicit legacy handling,
|
||||
before the gate is allowed to block anything.
|
||||
5. **Run B5 adoption** — hockey, soccer, football, then the remaining six.
|
||||
Bundled copies stay. Byte-identical harness output per plugin, then a soak.
|
||||
6. **Only then plan B6**, with the compatibility regression test described above
|
||||
in CI first.
|
||||
What actually remains, smallest first:
|
||||
|
||||
1. **Soak the adoptions on hardware.** football and hockey have been run on a
|
||||
live rig through real games; baseball was watched through one earlier. The
|
||||
rest are proven by harness, unit tests and pixel comparison. Out-of-season
|
||||
sports cannot be soaked until their season starts. When you do, **check the
|
||||
rig's `*_display_mode` first** — a board in `switch` mode will happily load a
|
||||
sunset plugin and tell you nothing about the scroll code the sunset changed.
|
||||
2. **Cut 3.3.0.** Not required by B6 — its floors are 3.2.0, which is released —
|
||||
but `calendar` 1.2.3 floors at 3.3.0 for the device-authorization endpoints
|
||||
that landed after 3.2.0, so it is un-installable until the release exists.
|
||||
3. **Reconsider the held modules** (`data_sources.py`, `game_renderer.py`,
|
||||
`base_odds_manager.py`) now that the sunset has closed. `game_renderer.py` is
|
||||
the largest single duplication left: ~11,500 lines across eight plugins, with
|
||||
~36,500 more in the eight `sports.py`. Note that core already ships
|
||||
`src/base_classes/sports/` (~143KB, promoted in B1/B2) that **no plugin
|
||||
imports** — check whether it has drifted before treating it as the target.
|
||||
|
||||
## How to keep this project healthy
|
||||
|
||||
|
||||
@@ -20,7 +20,22 @@ The installation script:
|
||||
- Installs and configures `dnsmasq` (DHCP server for AP mode)
|
||||
- These services can interfere with normal WiFi client mode
|
||||
|
||||
### 3. Reboot After Installation
|
||||
### 3. The Board Ran Out of Memory
|
||||
|
||||
On a 512MB or 1GB board, memory exhaustion stops `sshd` being able to fork a
|
||||
session process. The connection is accepted and then closed immediately, before
|
||||
any banner:
|
||||
|
||||
```text
|
||||
kex_exchange_identification: Connection closed by remote host
|
||||
```
|
||||
|
||||
The giveaway is that the board is otherwise healthy — ping is clean and the web
|
||||
UI still responds — but nothing that needs to start a new process works, and
|
||||
the panel is usually dark. Only a power cycle clears it. See
|
||||
[LOW_MEMORY_BOARDS.md](LOW_MEMORY_BOARDS.md).
|
||||
|
||||
### 4. Reboot After Installation
|
||||
|
||||
If the script reboots the Pi (which it recommends), network services may restart in a different state, potentially triggering AP mode.
|
||||
|
||||
@@ -190,11 +205,23 @@ The web interface allows you to:
|
||||
|
||||
## Summary
|
||||
|
||||
**SSH becomes unavailable because**:
|
||||
**SSH becomes unavailable because** — two unrelated causes, and they need
|
||||
different responses:
|
||||
|
||||
*AP mode (most common):*
|
||||
- WiFi monitor service enables AP mode when WiFi disconnects
|
||||
- AP mode switches WiFi from client to access point mode
|
||||
- Pi loses connection to your original network
|
||||
|
||||
*Memory exhaustion (low-memory boards):*
|
||||
- The board runs out of memory, so `sshd` cannot fork a session process
|
||||
- The connection is accepted and closed before any banner
|
||||
- Ping still answers and the web UI still responds, so it looks healthy
|
||||
- The panel is usually dark and the service cannot restart
|
||||
- **Only a power cycle clears this** — there is no remote recovery, because
|
||||
every remote route needs a new process
|
||||
- Prevention and tuning: [LOW_MEMORY_BOARDS.md](LOW_MEMORY_BOARDS.md)
|
||||
|
||||
**To regain SSH**:
|
||||
1. Connect to **LEDMatrix-Setup** AP network (password: `ledmatrix123`)
|
||||
2. SSH to `192.168.4.1`
|
||||
|
||||
Regular → Executable
+101
-3
@@ -1419,9 +1419,16 @@ $ACTUAL_USER ALL=(ALL) NOPASSWD: $BASH_PATH $PROJECT_ROOT_DIR/scripts/fix_perms/
|
||||
EOF
|
||||
if [ -n "$JOURNALCTL_PATH" ]; then
|
||||
cat >> /tmp/ledmatrix_web_sudoers << EOF
|
||||
$ACTUAL_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -u ledmatrix.service *
|
||||
$ACTUAL_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -u ledmatrix *
|
||||
$ACTUAL_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -t ledmatrix *
|
||||
# NOEXEC, because these rules end in a wildcard and journalctl starts a pager
|
||||
# when its output is a terminal. From that pager (less) a "!sh" is a root
|
||||
# shell -- the standard journalctl escalation. The web interface always passes
|
||||
# --no-pager, so nothing here needs it, but the rule cannot require a flag that
|
||||
# sits in the middle of the command line. NOEXEC stops the command executing
|
||||
# another program at all, which closes the hole without depending on wildcard
|
||||
# matching subtleties.
|
||||
$ACTUAL_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -u ledmatrix.service *
|
||||
$ACTUAL_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -u ledmatrix *
|
||||
$ACTUAL_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -t ledmatrix *
|
||||
EOF
|
||||
fi
|
||||
|
||||
@@ -1688,6 +1695,97 @@ else
|
||||
echo "✗ $CMDLINE_FILE not found; skipping isolcpus optimization"
|
||||
fi
|
||||
|
||||
# Enable the memory cgroup controller (idempotent).
|
||||
# The Pi firmware boots with cgroup_disable=memory, so systemd's MemoryMax= is
|
||||
# accepted and silently ignored — the display service then has no ceiling, and
|
||||
# a runaway takes the whole board down (sshd can no longer fork, the panel goes
|
||||
# dark) rather than just restarting the one service.
|
||||
if [ "$SKIP_PERF" != "1" ] && [ -f "$CMDLINE_FILE" ]; then
|
||||
# Both parameters are required for the memory controller, and they can get
|
||||
# separated -- an image, another tool or a half-applied earlier run can
|
||||
# leave one without the other. Checking only cgroup_enable=memory would
|
||||
# report success while MemoryMax= silently does nothing, so each is checked
|
||||
# and appended independently.
|
||||
cgroup_missing=""
|
||||
for cgroup_param in cgroup_enable=memory cgroup_memory=1; do
|
||||
if ! grep -qw "$cgroup_param" "$CMDLINE_FILE"; then
|
||||
cgroup_missing="$cgroup_missing $cgroup_param"
|
||||
fi
|
||||
done
|
||||
if [ -z "$cgroup_missing" ]; then
|
||||
echo "cgroup memory parameters already present in $CMDLINE_FILE"
|
||||
else
|
||||
echo "Adding${cgroup_missing} to $CMDLINE_FILE..."
|
||||
cp "$CMDLINE_FILE" "$CMDLINE_FILE.bak" 2>/dev/null || true
|
||||
# The kernel command line must stay on one line.
|
||||
sed -i "1 s|\$|${cgroup_missing}|" "$CMDLINE_FILE"
|
||||
echo " Takes effect after reboot. Verify with:"
|
||||
echo " grep memory /sys/fs/cgroup/cgroup.controllers"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Persist the journal (idempotent).
|
||||
# These images default to volatile storage: journald keeps everything in /run
|
||||
# (tmpfs), so every reboot destroys the logs — including the ones that would
|
||||
# explain why the board rebooted. Capped so an SD card is not worn out by logs.
|
||||
# A non-empty /var/log/journal does not prove journald is configured the way
|
||||
# this needs: the directory survives a switch back to volatile storage, and it
|
||||
# says nothing about whether a size cap is set. Read the effective
|
||||
# configuration instead, and only write the keys the user has not set
|
||||
# themselves so an explicit local limit is preserved.
|
||||
journald_effective() {
|
||||
# systemd-analyze merges journald.conf with every drop-in; grep is the
|
||||
# fallback for images that ship without it.
|
||||
if command -v systemd-analyze >/dev/null 2>&1 &&
|
||||
systemd-analyze cat-config systemd/journald.conf >/dev/null 2>&1; then
|
||||
systemd-analyze cat-config systemd/journald.conf 2>/dev/null
|
||||
else
|
||||
cat /etc/systemd/journald.conf /etc/systemd/journald.conf.d/*.conf 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
journald_conf="$(journald_effective)"
|
||||
# Both settings are optional, and stock images ship them commented out. grep
|
||||
# exits 1 on no match, which pipefail turns fatal under set -e — an absent
|
||||
# setting must read as empty, not abort the install.
|
||||
journald_storage="$(printf '%s\n' "$journald_conf" | grep -E '^[[:space:]]*Storage=' | tail -n1 | cut -d= -f2 | tr -d '[:space:]' || true)"
|
||||
journald_cap="$(printf '%s\n' "$journald_conf" | grep -E '^[[:space:]]*SystemMaxUse=' | tail -n1 | cut -d= -f2 | tr -d '[:space:]' || true)"
|
||||
|
||||
if [ "$journald_storage" = "persistent" ] && [ -n "$journald_cap" ]; then
|
||||
echo "Persistent journald storage already configured (SystemMaxUse=$journald_cap)"
|
||||
else
|
||||
echo "Enabling persistent journald storage..."
|
||||
mkdir -p /etc/systemd/journald.conf.d
|
||||
{
|
||||
echo "# Installed by LEDMatrix first_time_install.sh"
|
||||
echo "[Journal]"
|
||||
echo "Storage=persistent"
|
||||
if [ -n "$journald_cap" ]; then
|
||||
echo "# SystemMaxUse left to your existing setting ($journald_cap)"
|
||||
else
|
||||
# Capped so logs cannot wear out or fill an SD card.
|
||||
echo "SystemMaxUse=64M"
|
||||
fi
|
||||
} > /etc/systemd/journald.conf.d/ledmatrix-persistent.conf
|
||||
mkdir -p /var/log/journal
|
||||
systemd-tmpfiles --create --prefix /var/log/journal >/dev/null 2>&1 || true
|
||||
systemctl restart systemd-journald >/dev/null 2>&1 || true
|
||||
|
||||
# Drop-ins are applied in lexical order, so a locally added file that sorts
|
||||
# after ledmatrix-persistent.conf (zz-local.conf and friends) still wins.
|
||||
# Writing the file is not evidence it took effect -- re-read and say so
|
||||
# plainly rather than reporting success we cannot confirm.
|
||||
journald_now="$(journald_effective | grep -E '^[[:space:]]*Storage=' | tail -n1 | cut -d= -f2 | tr -d '[:space:]' || true)"
|
||||
if [ "$journald_now" = "persistent" ]; then
|
||||
echo " Persistent journald storage active"
|
||||
else
|
||||
echo " WARNING: journald storage is still '${journald_now:-unset}' after"
|
||||
echo " writing /etc/systemd/journald.conf.d/ledmatrix-persistent.conf."
|
||||
echo " Another drop-in that sorts later is overriding it. Check:"
|
||||
echo " systemd-analyze cat-config systemd/journald.conf | grep -n Storage="
|
||||
echo " Logs will not survive a reboot until that is resolved."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Ensure dtparam=audio=off in config.txt (idempotent)
|
||||
if [ "$SKIP_PERF" = "1" ]; then
|
||||
: # skipped
|
||||
|
||||
@@ -24,9 +24,29 @@ echo "========================================"
|
||||
# Auto-detect latest version if needed
|
||||
if [ "$PIXLET_VERSION" = "latest" ]; then
|
||||
echo "Detecting latest version..."
|
||||
PIXLET_VERSION=$(curl -s "https://api.github.com/repos/${REPO}/releases/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/')
|
||||
if [ -z "$PIXLET_VERSION" ]; then
|
||||
echo "Failed to detect latest version, using fallback"
|
||||
# When this response arrives on a single line -- as it did on the device
|
||||
# where Starlark apps were failing -- `grep '"tag_name"'` matches the whole
|
||||
# document and a greedy `sed 's/.*"([^"]+)".*/\1/'` captures the LAST
|
||||
# quoted token in it rather than the tag. That resolved to
|
||||
# "mentions_count", which built a download URL for a release that does not
|
||||
# exist. (The API is pretty-printed by default, which is why the old
|
||||
# command looks correct when you try it by hand -- but the formatting is
|
||||
# not something to depend on.) Match the field itself and take the value
|
||||
# after it, which is right for either shape.
|
||||
PIXLET_VERSION=$(curl -fsSL "https://api.github.com/repos/${REPO}/releases/latest" \
|
||||
| grep -o '"tag_name"[[:space:]]*:[[:space:]]*"[^"]*"' \
|
||||
| head -n1 \
|
||||
| sed -E 's/.*:[[:space:]]*"([^"]*)".*/\1/')
|
||||
|
||||
# A wrong-but-non-empty value is what made the old bug silent, so check the
|
||||
# shape rather than just that something came back. Anchored at both ends: a
|
||||
# partial match would accept "v0.53garbage" or "0.53" and build a URL for a
|
||||
# release that cannot exist, which is the failure this check is here to
|
||||
# stop. Every tronbyt/pixlet release to date is vX.Y.Z; the optional suffix
|
||||
# leaves room for a future -rc.1 or +build tag.
|
||||
if ! printf '%s' "$PIXLET_VERSION" \
|
||||
| grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$'; then
|
||||
echo "Could not detect the latest version (got: '${PIXLET_VERSION:-<empty>}'), using fallback"
|
||||
PIXLET_VERSION="v0.50.2"
|
||||
fi
|
||||
fi
|
||||
@@ -67,8 +87,26 @@ download_binary() {
|
||||
temp_dir=$(mktemp -d -p "$PROJECT_ROOT" -t pixlet_download.XXXXXXXXXX)
|
||||
local temp_file="$temp_dir/$archive_name"
|
||||
|
||||
if ! curl -L -o "$temp_file" "$url" 2>/dev/null; then
|
||||
echo "✗ Failed to download $arch"
|
||||
# -f so an HTTP error is a failure. Without it curl writes the 404 body
|
||||
# to the file and exits 0, and the first sign of trouble is tar saying
|
||||
# "not in gzip format" about what is actually a page of HTML.
|
||||
if ! curl -fL -o "$temp_file" "$url" 2>/dev/null; then
|
||||
echo "✗ Failed to download $arch from $url"
|
||||
rm -rf "$temp_dir"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Belt and braces: a mirror or proxy can return 200 with an error page.
|
||||
if ! gzip -t "$temp_file" 2>/dev/null; then
|
||||
echo "✗ Downloaded file is not a gzip archive: $url"
|
||||
# These bytes come from whatever answered the request, so strip
|
||||
# everything non-printable before echoing them: an error page carrying
|
||||
# terminal escapes would otherwise be able to rewrite this output or
|
||||
# bury it in a CI log. Printable characters are kept rather than
|
||||
# hex-encoding the lot, because "<!DOCTYPE html>" is the diagnostic.
|
||||
local first_bytes
|
||||
first_bytes=$(head -c 60 "$temp_file" | tr -cd '[:print:]')
|
||||
printf ' (first bytes: %s)\n' "$first_bytes"
|
||||
rm -rf "$temp_dir"
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -100,10 +100,15 @@ TEMP_SUDOERS="/tmp/ledmatrix_web_sudoers_$$"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH restart ledmatrix-web.service"
|
||||
|
||||
# Optional: journalctl (non-critical — skip if not found)
|
||||
#
|
||||
# NOEXEC, matching first_time_install.sh. These rules end in a wildcard and
|
||||
# journalctl starts a pager, so without it the caller can reach a shell:
|
||||
# less runs "!command" as the user the pager belongs to, which here is
|
||||
# root. NOEXEC stops the granted command executing anything of its own.
|
||||
if [ -n "$JOURNALCTL_PATH" ]; then
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -u ledmatrix.service *"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -u ledmatrix *"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD: $JOURNALCTL_PATH -t ledmatrix *"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -u ledmatrix.service *"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -u ledmatrix *"
|
||||
echo "$WEB_USER ALL=(ALL) NOPASSWD:NOEXEC: $JOURNALCTL_PATH -t ledmatrix *"
|
||||
fi
|
||||
|
||||
# Required: python3, bash
|
||||
|
||||
@@ -25,9 +25,44 @@ if [ "$EUID" -eq 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Resolve command paths against a fixed PATH, and check what we resolved.
|
||||
#
|
||||
# Every path found here is written into a sudoers file as a NOPASSWD grant, so
|
||||
# whoever controls the binary at that path controls root. first_time_install.sh
|
||||
# re-execs itself with `sudo -E`, which preserves the invoking user's
|
||||
# environment -- PATH included -- so without pinning it, `which nmcli` can
|
||||
# resolve to anything on that PATH: a writable directory early in it turns a
|
||||
# compromise of the low-privilege web user into permanent root.
|
||||
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
||||
export PATH
|
||||
|
||||
# A binary named in a sudoers rule must be root-owned and writable by nobody
|
||||
# else, or the grant hands root to whoever can rewrite it.
|
||||
require_trusted_binary() {
|
||||
local label="$1" path="$2"
|
||||
if [ ! -x "$path" ]; then
|
||||
echo "✗ $label: $path is not an executable file"
|
||||
exit 1
|
||||
fi
|
||||
local owner perms
|
||||
owner=$(stat -c '%u' "$path") || exit 1
|
||||
perms=$(stat -c '%a' "$path") || exit 1
|
||||
if [ "$owner" != "0" ]; then
|
||||
echo "✗ $label: $path is not owned by root (uid $owner); refusing to"
|
||||
echo " grant it NOPASSWD sudo."
|
||||
exit 1
|
||||
fi
|
||||
# Group- or world-writable means someone other than root can replace it.
|
||||
case "$perms" in
|
||||
*[2367]) echo "✗ $label: $path is writable by group or other ($perms);"
|
||||
echo " refusing to grant it NOPASSWD sudo."
|
||||
exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Get the full paths to commands
|
||||
NMCLI_PATH=$(which nmcli || echo "/usr/bin/nmcli")
|
||||
SYSTEMCTL_PATH=$(which systemctl)
|
||||
NMCLI_PATH=$(command -v nmcli || echo "/usr/bin/nmcli")
|
||||
SYSTEMCTL_PATH=$(command -v systemctl)
|
||||
|
||||
echo "Command paths:"
|
||||
echo " nmcli: $NMCLI_PATH"
|
||||
@@ -37,6 +72,18 @@ echo " systemctl: $SYSTEMCTL_PATH"
|
||||
echo ""
|
||||
echo "Step 1: Configuring sudo permissions for nmcli..."
|
||||
SUDOERS_FILE="/etc/sudoers.d/ledmatrix_wifi"
|
||||
SYSCTL_PATH=$(command -v sysctl || echo /usr/sbin/sysctl)
|
||||
NFT_PATH=$(command -v nft || echo /usr/sbin/nft)
|
||||
RFKILL_PATH=$(command -v rfkill || echo /usr/sbin/rfkill)
|
||||
MKDIR_PATH=$(command -v mkdir || echo /usr/bin/mkdir)
|
||||
|
||||
# Checked before any of them reaches the sudoers file.
|
||||
require_trusted_binary "nmcli" "$NMCLI_PATH"
|
||||
require_trusted_binary "systemctl" "$SYSTEMCTL_PATH"
|
||||
require_trusted_binary "sysctl" "$SYSCTL_PATH"
|
||||
require_trusted_binary "nft" "$NFT_PATH"
|
||||
require_trusted_binary "rfkill" "$RFKILL_PATH"
|
||||
require_trusted_binary "mkdir" "$MKDIR_PATH"
|
||||
|
||||
# Create a temporary sudoers file using mktemp (handles permissions better)
|
||||
TEMP_SUDOERS=$(mktemp) || {
|
||||
@@ -62,6 +109,36 @@ $WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH start dnsmasq
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH stop dnsmasq
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH restart dnsmasq
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: $SYSTEMCTL_PATH restart NetworkManager
|
||||
# The captive portal turns IP forwarding on while the access point is up and
|
||||
# restores the previous value when it comes down (wifi_manager._setup_iptables_
|
||||
# redirect / _teardown_iptables_redirect). Without this rule that sudo call
|
||||
# needs a password, so forwarding stays off and clients associate to the AP but
|
||||
# cannot route. It goes unnoticed on a stock Raspberry Pi image, where
|
||||
# /etc/sudoers.d/010_pi-nopasswd grants the default user blanket NOPASSWD and
|
||||
# masks every gap in this file -- it only bites once that blanket rule is
|
||||
# removed.
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: $SYSCTL_PATH -w net.ipv4.ip_forward=0
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: $SYSCTL_PATH -w net.ipv4.ip_forward=1
|
||||
# The portal's redirect lives in its own nftables table, created when the AP
|
||||
# comes up and deleted when it goes down, and the radio has to be unblocked
|
||||
# before the AP can start at all. Same story as the sysctl rules above: called
|
||||
# with sudo, never granted here, and invisible on a stock Pi image.
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: $NFT_PATH add table ip ledmatrix
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: $NFT_PATH delete table ip ledmatrix
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: $RFKILL_PATH unblock wifi
|
||||
# NetworkManager's dnsmasq drop-in directory, exact path.
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: $MKDIR_PATH -p /etc/NetworkManager/dnsmasq-shared.d
|
||||
#
|
||||
# iptables is deliberately NOT granted here. Its rules are built from the live
|
||||
# interface name and port, so a rule covering them needs a trailing wildcard --
|
||||
# and `iptables --modprobe=/path/to/anything` runs that path as root, so
|
||||
# `NOPASSWD: iptables *` is a root shell for the web user by another name. That
|
||||
# is a worse outcome than the gap it would close, which today is masked anyway
|
||||
# by the blanket NOPASSWD rule on stock Pi images.
|
||||
#
|
||||
# Closing it safely means a wrapper script that builds the rules itself and
|
||||
# takes only an interface and a port, granted the way safe_plugin_rm.sh already
|
||||
# is. That belongs in its own change rather than being smuggled into this one.
|
||||
|
||||
# Allow copying hostapd and dnsmasq config files into place
|
||||
$WEB_USER ALL=(ALL) NOPASSWD: /usr/bin/cp /tmp/hostapd.conf /etc/hostapd/hostapd.conf
|
||||
|
||||
Regular → Executable
Regular → Executable
+1
-1
@@ -4,5 +4,5 @@ LEDMatrix Display System
|
||||
Core source package for the LED Matrix Display project.
|
||||
"""
|
||||
|
||||
__version__ = "3.2.0"
|
||||
__version__ = "3.3.0"
|
||||
|
||||
|
||||
+210
-11
@@ -14,11 +14,12 @@ Key Features:
|
||||
- Memory-efficient data storage
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import time
|
||||
import logging
|
||||
import threading
|
||||
import requests
|
||||
from typing import Dict, Any, Optional, Callable
|
||||
from typing import Dict, Any, Optional, Callable, List
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
import queue
|
||||
@@ -50,14 +51,33 @@ class FetchRequest:
|
||||
max_retries: int = 3
|
||||
priority: int = 1 # Higher number = higher priority
|
||||
callback: Optional[Callable] = None
|
||||
# Callbacks from submitters that JOINED this fetch instead of starting a
|
||||
# duplicate one. The primary `callback` above belongs to whoever created
|
||||
# the request; these belong to everyone who asked for the same cache_key
|
||||
# while it was still in flight.
|
||||
extra_callbacks: List[Callable] = field(default_factory=list)
|
||||
created_at: float = field(default_factory=time.time)
|
||||
status: FetchStatus = FetchStatus.PENDING
|
||||
# Set once the worker has decided this response will be cached, while it
|
||||
# still holds the lock. From that point cancelling is refused: the write
|
||||
# is already authorised, and abandoning it here would put the payload in
|
||||
# the cache with the callbacks suppressed -- joiners waiting forever for a
|
||||
# fetch that did, in fact, succeed.
|
||||
commit_claimed: bool = False
|
||||
result: Optional[Any] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
@dataclass
|
||||
class FetchResult:
|
||||
"""Result of a background fetch operation."""
|
||||
"""Result of a background fetch operation.
|
||||
|
||||
``data`` survives on the stored result only for requests submitted without
|
||||
a ``callback``, where polling ``get_result()`` is the sole way to collect
|
||||
it. When a callback was given, the payload has already been delivered and
|
||||
the service releases it -- see :meth:`BackgroundDataService._release_payload`.
|
||||
Either way the data remains in the cache under the request's ``cache_key``,
|
||||
which is where consumers read it from.
|
||||
"""
|
||||
request_id: str
|
||||
success: bool
|
||||
data: Optional[Any] = None
|
||||
@@ -66,6 +86,10 @@ class FetchResult:
|
||||
fetch_time: float = 0.0
|
||||
retry_count: int = 0
|
||||
completed_at: float = field(default_factory=time.time) # Timestamp when request completed
|
||||
# The request's final status, recorded so a finished request can still be
|
||||
# reported accurately. Without it a caller can only be told COMPLETED or
|
||||
# FAILED, which turns "you cancelled this" into "this errored".
|
||||
final_status: Optional[FetchStatus] = None
|
||||
|
||||
class BackgroundDataService:
|
||||
"""
|
||||
@@ -90,6 +114,20 @@ class BackgroundDataService:
|
||||
|
||||
# Thread management
|
||||
self.executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="BackgroundData")
|
||||
# cache_key -> request_id for fetches currently in flight. Submitting
|
||||
# the same key twice used to start two identical fetches: request_id
|
||||
# carries a millisecond timestamp, so every submit looked new, and
|
||||
# active_requests is keyed by it rather than by what is being fetched.
|
||||
# On a real board the season-schedule key is requested by both the
|
||||
# Recent and the Upcoming manager, which miss the cache in the same
|
||||
# millisecond and each download and parse the same payload.
|
||||
self._inflight_by_cache_key: Dict[str, str] = {}
|
||||
# request_id was sport_year_milliseconds, which is not unique: two
|
||||
# submits inside the same millisecond produced the SAME id, so one
|
||||
# silently replaced the other in active_requests and completed_requests.
|
||||
# Rare before, but dedupe hands this id back to every joiner as their
|
||||
# handle for get_result(), so it has to be unique. A counter is enough.
|
||||
self._request_seq = itertools.count()
|
||||
self.active_requests: Dict[str, FetchRequest] = {}
|
||||
self.completed_requests: Dict[str, FetchResult] = {}
|
||||
self.request_queue = queue.PriorityQueue()
|
||||
@@ -177,7 +215,9 @@ class BackgroundDataService:
|
||||
if cache_key is None:
|
||||
cache_key = self.get_sport_cache_key(sport)
|
||||
|
||||
request_id = f"{sport}_{year}_{int(time.time() * 1000)}"
|
||||
with self._lock:
|
||||
request_id = (f"{sport}_{year}_{int(time.time() * 1000)}"
|
||||
f"_{next(self._request_seq)}")
|
||||
|
||||
# Check cache first
|
||||
cached_data = self.cache_manager.get(cache_key)
|
||||
@@ -191,14 +231,19 @@ class BackgroundDataService:
|
||||
cached=True,
|
||||
fetch_time=0.0
|
||||
)
|
||||
# Filed before the callback runs, as it always was: a callback
|
||||
# that queries get_result()/is_request_complete() for its own
|
||||
# request must still find it. Releasing afterwards mutates the
|
||||
# same object the dict holds.
|
||||
self.completed_requests[request_id] = result
|
||||
|
||||
|
||||
if callback:
|
||||
try:
|
||||
callback(result)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in callback for request {request_id}: {e}")
|
||||
|
||||
self._release_payload(result)
|
||||
|
||||
logger.debug(f"Cache hit for {sport} {year} data")
|
||||
return request_id
|
||||
|
||||
@@ -218,7 +263,29 @@ class BackgroundDataService:
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
existing_id = self._inflight_by_cache_key.get(cache_key)
|
||||
existing = self.active_requests.get(existing_id) if existing_id else None
|
||||
if existing_id and existing is None:
|
||||
# Stranded index entry: the request it names is gone. Drop it and
|
||||
# fetch normally. Looking the request up rather than trusting the
|
||||
# id is what stops a stale entry wedging a key forever.
|
||||
del self._inflight_by_cache_key[cache_key]
|
||||
if existing is not None:
|
||||
# Someone is already fetching this key. Ride along rather than
|
||||
# duplicating the download, the parse and the resident copy.
|
||||
if callback:
|
||||
existing.extra_callbacks.append(callback)
|
||||
self.stats['deduplicated_requests'] = (
|
||||
self.stats.get('deduplicated_requests', 0) + 1
|
||||
)
|
||||
logger.info(
|
||||
"Joined in-flight fetch %s for %s (cache_key=%s) instead of "
|
||||
"starting a duplicate", existing_id, sport, cache_key
|
||||
)
|
||||
return existing_id
|
||||
|
||||
self.active_requests[request_id] = request
|
||||
self._inflight_by_cache_key[cache_key] = request_id
|
||||
self.stats['total_requests'] += 1
|
||||
self.stats['cache_misses'] += 1
|
||||
|
||||
@@ -243,7 +310,31 @@ class BackgroundDataService:
|
||||
|
||||
try:
|
||||
with self._lock:
|
||||
request.status = FetchStatus.IN_PROGRESS
|
||||
# A request cancelled while it sat in the executor queue must
|
||||
# stay cancelled. Overwriting the status here undid the cancel
|
||||
# outright: the worker went on to download, cache and call back
|
||||
# for work the caller had already withdrawn.
|
||||
if request.status == FetchStatus.CANCELLED:
|
||||
cancelled_before_start = True
|
||||
else:
|
||||
cancelled_before_start = False
|
||||
request.status = FetchStatus.IN_PROGRESS
|
||||
if cancelled_before_start:
|
||||
logger.info(
|
||||
"Request %s was cancelled before its worker started; "
|
||||
"skipping the fetch entirely", request.id
|
||||
)
|
||||
# Assign before returning: the finally block stores `result`
|
||||
# in completed_requests, so building a fresh one here would
|
||||
# file the untouched placeholder instead of this outcome.
|
||||
result = FetchResult(
|
||||
request_id=request.id,
|
||||
success=False,
|
||||
error="cancelled",
|
||||
fetch_time=time.time() - start_time,
|
||||
retry_count=request.retry_count
|
||||
)
|
||||
return result
|
||||
|
||||
logger.info(f"Starting background fetch for {request.sport} {request.year}")
|
||||
|
||||
@@ -269,6 +360,37 @@ class BackgroundDataService:
|
||||
# Log data validation
|
||||
logger.debug(f"Validated {len(events)} events for {request.sport} {request.year}")
|
||||
|
||||
# A cancelled request must not commit anything. Cancelling
|
||||
# releases the cache_key, so a replacement fetch for the same key
|
||||
# may already be in flight or finished -- writing this response to
|
||||
# the cache now would overwrite fresher data with the response
|
||||
# nobody wanted. The worker has no way to abort the HTTP call, so
|
||||
# this is where the work gets discarded.
|
||||
with self._lock:
|
||||
cancelled = request.status == FetchStatus.CANCELLED
|
||||
if not cancelled:
|
||||
# Claim the commit in the same critical section that read
|
||||
# the status, so a cancel cannot slip in between the check
|
||||
# and the cache write below. The write itself stays outside
|
||||
# the lock: it serialises a multi-megabyte payload to the
|
||||
# SD card, and holding the service lock across that would
|
||||
# stall every submit, status query and cancel behind it.
|
||||
request.commit_claimed = True
|
||||
if cancelled:
|
||||
logger.info(
|
||||
"Discarding response for cancelled request %s; %s may "
|
||||
"already belong to a replacement fetch",
|
||||
request.id, request.cache_key
|
||||
)
|
||||
result = FetchResult(
|
||||
request_id=request.id,
|
||||
success=False,
|
||||
error="cancelled",
|
||||
fetch_time=time.time() - start_time,
|
||||
retry_count=request.retry_count
|
||||
)
|
||||
return result
|
||||
|
||||
# Cache the data
|
||||
self.cache_manager.set(request.cache_key, data)
|
||||
|
||||
@@ -294,7 +416,12 @@ class BackgroundDataService:
|
||||
logger.error(f"Failed to fetch {request.sport} {request.year} data: {error_msg}")
|
||||
|
||||
with self._lock:
|
||||
request.status = FetchStatus.FAILED
|
||||
# Don't relabel a cancelled request. The callback gate in the
|
||||
# finally block only suppresses CANCELLED, so promoting it to
|
||||
# FAILED here delivered an error callback for a fetch nobody
|
||||
# was waiting on any more.
|
||||
if request.status != FetchStatus.CANCELLED:
|
||||
request.status = FetchStatus.FAILED
|
||||
request.error = error_msg
|
||||
|
||||
result = FetchResult(
|
||||
@@ -308,9 +435,26 @@ class BackgroundDataService:
|
||||
finally:
|
||||
# Store result and clean up
|
||||
with self._lock:
|
||||
result.final_status = request.status
|
||||
self.completed_requests[request.id] = result
|
||||
if request.id in self.active_requests:
|
||||
del self.active_requests[request.id]
|
||||
# Stop accepting joiners and take the callback list in the same
|
||||
# critical section. A submitter that arrives after this point
|
||||
# finds no in-flight entry and either hits the cache (written
|
||||
# above, before the result was built) or starts a fresh fetch --
|
||||
# what it must never do is join a fetch whose callbacks have
|
||||
# already run and then never be called.
|
||||
if self._inflight_by_cache_key.get(request.cache_key) == request.id:
|
||||
del self._inflight_by_cache_key[request.cache_key]
|
||||
# A cancelled request delivers nothing: its joiners were told
|
||||
# about a fetch that has been abandoned, and a replacement will
|
||||
# call them via its own request.
|
||||
if request.status == FetchStatus.CANCELLED:
|
||||
callbacks = []
|
||||
else:
|
||||
callbacks = ([request.callback] if request.callback else [])
|
||||
callbacks.extend(request.extra_callbacks)
|
||||
|
||||
# Update statistics
|
||||
if result.success:
|
||||
@@ -327,14 +471,53 @@ class BackgroundDataService:
|
||||
# Periodic cleanup after storing result
|
||||
self._cleanup_completed_requests()
|
||||
|
||||
# Call callback if provided
|
||||
if request.callback:
|
||||
# Call every callback: the original submitter's and any that joined
|
||||
# this fetch. One raising must not stop the others being delivered.
|
||||
for cb in callbacks:
|
||||
try:
|
||||
request.callback(result)
|
||||
cb(result)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in callback for request {request.id}: {e}")
|
||||
|
||||
|
||||
# Released AFTER the loop, not inside it. Every callback here holds
|
||||
# the same FetchResult, so releasing per-delivery handed the first
|
||||
# one the data and every joiner `result.data is None` -- which is
|
||||
# not a quiet degradation: they read `result.data.get('events')` and
|
||||
# raise AttributeError, which this very loop catches and logs, so
|
||||
# the symptom was one ERROR line and a manager that silently never
|
||||
# got its schedule. Deduplication is the normal case, not a corner:
|
||||
# a sport's recent, upcoming and live managers all ride one season
|
||||
# fetch.
|
||||
#
|
||||
# Guarded on `callbacks`, because a request submitted without one
|
||||
# has no other way to collect its payload than polling get_result().
|
||||
# The old per-delivery release got that right by accident: an empty
|
||||
# list never entered the loop body.
|
||||
if callbacks:
|
||||
self._release_payload(result)
|
||||
request.result = None
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _release_payload(result: FetchResult) -> None:
|
||||
"""Drop a delivered payload, keeping the result's status and timings.
|
||||
|
||||
Only called once EVERY callback has been handed the data -- callers
|
||||
that joined an in-flight fetch share this object, so releasing between
|
||||
deliveries strips the payload out from under the ones still queued.
|
||||
Consumers read fetched data back from the cache under ``cache_key``;
|
||||
the copy carried
|
||||
here was pinning a parsed season schedule -- 946 games for NCAA
|
||||
football, roughly a tenth of total RAM on a 1GB Pi -- in memory until
|
||||
the hourly sweep.
|
||||
|
||||
The cache-hit path matters most: it runs once per update interval per
|
||||
sport, mints a fresh request_id each time, and a memory-tier miss
|
||||
re-parses the payload from disk. Those were genuinely separate copies
|
||||
accumulating toward the 500-entry cap, not shared references.
|
||||
"""
|
||||
result.data = None
|
||||
|
||||
def _make_request_with_retry(self, request: FetchRequest) -> requests.Response:
|
||||
"""
|
||||
@@ -422,6 +605,8 @@ class BackgroundDataService:
|
||||
return self.active_requests[request_id].status
|
||||
elif request_id in self.completed_requests:
|
||||
result = self.completed_requests[request_id]
|
||||
if result.final_status is not None:
|
||||
return result.final_status
|
||||
return FetchStatus.COMPLETED if result.success else FetchStatus.FAILED
|
||||
return None
|
||||
|
||||
@@ -438,8 +623,22 @@ class BackgroundDataService:
|
||||
with self._lock:
|
||||
if request_id in self.active_requests:
|
||||
request = self.active_requests[request_id]
|
||||
if request.commit_claimed:
|
||||
# Too late: the worker holds an authorised commit. Report
|
||||
# the failure rather than half-cancelling a request whose
|
||||
# data is about to land in the cache.
|
||||
logger.debug(
|
||||
"Not cancelling %s: its response is already being "
|
||||
"committed", request_id
|
||||
)
|
||||
return False
|
||||
request.status = FetchStatus.CANCELLED
|
||||
del self.active_requests[request_id]
|
||||
# Cancelling is the other way a request leaves active_requests,
|
||||
# so the in-flight index has to be released here too or the key
|
||||
# stays pointed at a request that no longer exists.
|
||||
if self._inflight_by_cache_key.get(request.cache_key) == request_id:
|
||||
del self._inflight_by_cache_key[request.cache_key]
|
||||
logger.info(f"Cancelled request {request_id}")
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -145,8 +145,7 @@ class SportsUpcoming(SportsCore):
|
||||
if (game['home_abbr'] in self.favorite_teams or
|
||||
game['away_abbr'] in self.favorite_teams):
|
||||
favorite_games_found += 1
|
||||
if self.show_odds:
|
||||
self._fetch_odds(game)
|
||||
# Odds are NOT fetched here -- see after selection below.
|
||||
|
||||
# Enhanced logging for debugging
|
||||
self.logger.info(f"Found {all_upcoming_games} total upcoming games in data")
|
||||
@@ -190,6 +189,20 @@ class SportsUpcoming(SportsCore):
|
||||
# Limit to the specified number of upcoming games
|
||||
team_games = team_games[:self.upcoming_games_to_show]
|
||||
|
||||
# Odds are fetched here, for the games that survived selection,
|
||||
# rather than in the loop that collects them. That loop walks every
|
||||
# upcoming game in the schedule window, and for a college league
|
||||
# the window is enormous -- a live rig logged 946 upcoming games in
|
||||
# one cycle and displayed 1 of them. The comment up there claimed
|
||||
# odds were fetched "only for games that will be displayed", but
|
||||
# the only narrowing it applied was show_favorite_teams_only, which
|
||||
# is not the default; in the usual case nothing narrowed it at all
|
||||
# and every game cost a separate ESPN request on a Pi that is also
|
||||
# driving the panel.
|
||||
if self.show_odds:
|
||||
for game in team_games:
|
||||
self._fetch_odds(game)
|
||||
|
||||
# Log changes or periodically
|
||||
should_log = (
|
||||
current_time - self.last_log_time >= self.log_interval or
|
||||
|
||||
Vendored
+75
-16
@@ -4,11 +4,58 @@ Memory Cache
|
||||
Handles in-memory caching with TTL support, size limits, and automatic cleanup.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
import logging
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
# Historical fixed ceiling, kept as the fallback when RAM cannot be read.
|
||||
DEFAULT_MAX_SIZE = 1000
|
||||
|
||||
|
||||
def _total_memory_mb() -> Optional[float]:
|
||||
"""Physical RAM in MB, or None where /proc/meminfo is unavailable."""
|
||||
try:
|
||||
with open('/proc/meminfo', 'r', encoding='utf-8') as fh:
|
||||
for line in fh:
|
||||
if line.startswith('MemTotal:'):
|
||||
return int(line.split()[1]) / 1024
|
||||
except (OSError, ValueError, IndexError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def default_max_size() -> int:
|
||||
"""Entry ceiling scaled to this machine's RAM.
|
||||
|
||||
One fixed ceiling cannot serve both a 512 MB Pi Zero 2 W and an 8 GB Pi 5.
|
||||
Entries here are parsed API payloads that routinely run tens of kilobytes
|
||||
each, so a thousand of them is a comfortable cache on a large board and a
|
||||
substantial fraction of total RAM on a small one — where the process
|
||||
competing for that RAM is also driving the panel. Set
|
||||
LEDMATRIX_CACHE_MAX_ENTRIES to override.
|
||||
"""
|
||||
override = os.environ.get('LEDMATRIX_CACHE_MAX_ENTRIES')
|
||||
if override:
|
||||
try:
|
||||
value = int(override)
|
||||
if value > 0:
|
||||
return value
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
total_mb = _total_memory_mb()
|
||||
if total_mb is None:
|
||||
return DEFAULT_MAX_SIZE
|
||||
if total_mb < 1536: # 512 MB and 1 GB boards
|
||||
return 150
|
||||
if total_mb < 3072: # 2 GB
|
||||
return 400
|
||||
if total_mb < 6144: # 4 GB
|
||||
return 800
|
||||
return 1500 # 8 GB and up
|
||||
|
||||
|
||||
class MemoryCache:
|
||||
"""Manages in-memory cache with TTL and size limits."""
|
||||
@@ -87,6 +134,32 @@ class MemoryCache:
|
||||
with self._lock:
|
||||
self._cache[key] = value
|
||||
self._timestamps[key] = time.time()
|
||||
# Enforce the ceiling here rather than leaving it to the periodic
|
||||
# cleanup, which only runs every cleanup_interval seconds (300 by
|
||||
# default). A burst of inserts between two sweeps could otherwise
|
||||
# take the cache far past _max_size, which is the memory growth this
|
||||
# limit exists to prevent -- and on a 1GB board that is the
|
||||
# difference between a bounded cache and an unreachable Pi.
|
||||
self._evict_over_limit_locked()
|
||||
|
||||
def _evict_over_limit_locked(self) -> int:
|
||||
"""Drop oldest entries until the cache is within _max_size.
|
||||
|
||||
Caller must hold self._lock. Returns the number of entries removed.
|
||||
"""
|
||||
excess = len(self._cache) - self._max_size
|
||||
if excess <= 0:
|
||||
return 0
|
||||
oldest = sorted(
|
||||
self._timestamps.items(),
|
||||
key=lambda item: float(item[1]) if isinstance(item[1], (int, float)) else 0.0
|
||||
)
|
||||
removed = 0
|
||||
for key, _ in oldest[:excess]:
|
||||
self._cache.pop(key, None)
|
||||
self._timestamps.pop(key, None)
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
def clear(self, key: Optional[str] = None) -> None:
|
||||
"""
|
||||
@@ -143,22 +216,8 @@ class MemoryCache:
|
||||
self._timestamps.pop(key, None)
|
||||
removed_count += 1
|
||||
|
||||
# Enforce size limit by removing oldest entries if cache is too large
|
||||
if len(self._cache) > self._max_size:
|
||||
# Sort by timestamp (oldest first)
|
||||
sorted_entries = sorted(
|
||||
self._timestamps.items(),
|
||||
key=lambda x: float(x[1]) if isinstance(x[1], (int, float)) else 0
|
||||
)
|
||||
|
||||
# Remove oldest entries until we're under the limit
|
||||
excess_count = len(self._cache) - self._max_size
|
||||
for i in range(excess_count):
|
||||
if i < len(sorted_entries):
|
||||
key = sorted_entries[i][0]
|
||||
self._cache.pop(key, None)
|
||||
self._timestamps.pop(key, None)
|
||||
removed_count += 1
|
||||
# Same ceiling enforcement set() uses, so the two cannot drift.
|
||||
removed_count += self._evict_over_limit_locked()
|
||||
|
||||
self._last_cleanup = current_time
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ import logging
|
||||
import threading
|
||||
import tempfile
|
||||
from src.exceptions import CacheError
|
||||
from src.cache.memory_cache import MemoryCache
|
||||
from src.cache.memory_cache import MemoryCache, default_max_size
|
||||
from src.cache.disk_cache import DiskCache
|
||||
from src.cache.cache_strategy import CacheStrategy
|
||||
from src.cache.cache_metrics import CacheMetrics
|
||||
@@ -84,7 +84,9 @@ class CacheManager:
|
||||
self.logger.warning("ConfigManager not available, using default cache intervals")
|
||||
|
||||
# Initialize cache components using composition
|
||||
self._memory_cache_component = MemoryCache(max_size=1000, cleanup_interval=300.0)
|
||||
self._memory_cache_component = MemoryCache(
|
||||
max_size=default_max_size(), cleanup_interval=300.0
|
||||
)
|
||||
self._disk_cache_component = DiskCache(cache_dir=self.cache_dir, logger=self.logger)
|
||||
self._strategy_component = CacheStrategy(config_manager=self.config_manager, logger=self.logger)
|
||||
self._metrics_component = CacheMetrics(logger=self.logger)
|
||||
|
||||
+116
-14
@@ -6,6 +6,8 @@ Extracted from LEDMatrix core to provide reusable functionality for plugins.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
@@ -19,6 +21,10 @@ from src.common.permission_utils import (
|
||||
)
|
||||
|
||||
|
||||
# Well above any real team logo; bounds what a remote URL can write to disk.
|
||||
MAX_LOGO_BYTES = 10 * 1024 * 1024
|
||||
|
||||
|
||||
class LogoHelper:
|
||||
"""
|
||||
Helper class for logo loading, caching, and resizing.
|
||||
@@ -137,8 +143,10 @@ class LogoHelper:
|
||||
"""
|
||||
logo_path = Path(logo_path)
|
||||
|
||||
# Try to load existing logo first
|
||||
if logo_path.exists():
|
||||
# Try to load existing logo first. A placeholder written by a previous
|
||||
# failed download does not count: it wears the real logo's filename, so
|
||||
# trusting the file's existence is what left teams as grey boxes.
|
||||
if logo_path.exists() and not self._is_stale_placeholder(logo_path):
|
||||
return self.load_logo(team_abbr, logo_path, max_width, max_height)
|
||||
|
||||
# Download if URL provided and file doesn't exist
|
||||
@@ -146,13 +154,61 @@ class LogoHelper:
|
||||
try:
|
||||
self.logger.info(f"Downloading logo for {team_abbr} from {logo_url}")
|
||||
self._download_logo(logo_url, logo_path)
|
||||
# The file on disk just changed. Any cached image for it is the
|
||||
# placeholder we came here to replace, and load_logo() answers
|
||||
# from the cache before touching the disk -- so without this the
|
||||
# real logo would not appear until the process restarted.
|
||||
self._invalidate_cached_logo(team_abbr, logo_path)
|
||||
return self.load_logo(team_abbr, logo_path, max_width, max_height)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Failed to download logo for {team_abbr}: {e}")
|
||||
# The retry failed, so restart the back-off. The stale
|
||||
# placeholder is still on disk with its old timestamp, and
|
||||
# leaving it there means the next call retries immediately --
|
||||
# a download attempt per call, which is what the back-off
|
||||
# exists to prevent.
|
||||
self._refresh_stale_placeholder(logo_path)
|
||||
|
||||
# Create placeholder if all else fails
|
||||
return self._create_placeholder_logo(team_abbr, max_width, max_height)
|
||||
|
||||
def _invalidate_cached_logo(self, team_abbr: str, logo_path: Path) -> None:
|
||||
"""Drop every cached size of one logo after its file changed on disk."""
|
||||
prefix = f"{team_abbr}_{logo_path}_"
|
||||
for key in [k for k in self._logo_cache if k.startswith(prefix)]:
|
||||
self._logo_cache.pop(key, None)
|
||||
if key in self._cache_order:
|
||||
self._cache_order.remove(key)
|
||||
|
||||
@staticmethod
|
||||
def _refresh_stale_placeholder(logo_path: Path) -> None:
|
||||
"""Restart the retry back-off after a failed download attempt."""
|
||||
try:
|
||||
from src.logo_downloader import refresh_placeholder_timestamp
|
||||
except ImportError:
|
||||
return
|
||||
refresh_placeholder_timestamp(logo_path)
|
||||
|
||||
@staticmethod
|
||||
def _is_stale_placeholder(logo_path: Path) -> bool:
|
||||
"""True if the file is a placeholder old enough to be worth retrying.
|
||||
|
||||
Imported lazily so this module keeps working against a core build whose
|
||||
logo_downloader predates placeholder marking.
|
||||
"""
|
||||
try:
|
||||
from src.logo_downloader import (
|
||||
PLACEHOLDER_RETRY_SECONDS,
|
||||
is_placeholder_logo,
|
||||
placeholder_age_seconds,
|
||||
)
|
||||
except ImportError:
|
||||
return False
|
||||
if not is_placeholder_logo(logo_path):
|
||||
return False
|
||||
age = placeholder_age_seconds(logo_path)
|
||||
return age is None or age >= PLACEHOLDER_RETRY_SECONDS
|
||||
|
||||
def get_logo_variations(self, team_abbr: str) -> List[str]:
|
||||
"""
|
||||
Get possible filename variations for a team abbreviation.
|
||||
@@ -226,7 +282,10 @@ class LogoHelper:
|
||||
return {
|
||||
'cached_logos': len(self._logo_cache),
|
||||
'cache_size_limit': self.cache_size,
|
||||
'cache_usage_percent': (len(self._logo_cache) / self.cache_size) * 100
|
||||
'cache_usage_percent': (
|
||||
(len(self._logo_cache) / self.cache_size) * 100
|
||||
if self.cache_size else 0
|
||||
),
|
||||
}
|
||||
|
||||
def _resize_logo(self, logo: Image.Image, max_width: Optional[int] = None,
|
||||
@@ -258,21 +317,64 @@ class LogoHelper:
|
||||
self._cache_order.append(cache_key)
|
||||
|
||||
def _download_logo(self, url: str, file_path: Path) -> None:
|
||||
"""Download logo from URL."""
|
||||
"""Download logo from URL.
|
||||
|
||||
The response size is capped and the saved file is verified as a
|
||||
decodable image before it is left on disk: a logo URL is remote
|
||||
input, and without this an oversized or malformed response would
|
||||
be cached for every later load_logo() call to trip over.
|
||||
|
||||
The body is streamed and counted as it arrives rather than read
|
||||
through response.content, which buffers the whole thing first —
|
||||
a server that omits Content-Length and never stops sending would
|
||||
exhaust memory before any size check could run. Nothing lands at
|
||||
file_path until the download completes and decodes, so a failed
|
||||
download cannot leave a truncated logo behind either.
|
||||
"""
|
||||
# Ensure directory exists with proper permissions
|
||||
ensure_directory_permissions(file_path.parent, get_assets_dir_mode())
|
||||
|
||||
# Download with timeout
|
||||
response = self.session.get(url, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
# Save to file
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(response.content)
|
||||
|
||||
|
||||
# A unique temp name, not a fixed "<name>.part": two plugins can
|
||||
# ask for the same logo at once, and a shared name would let them
|
||||
# interleave writes into one file, publish the mixture, or delete
|
||||
# each other's partial. Same directory, so os.replace stays atomic.
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
dir=str(file_path.parent), prefix=file_path.name + '.', suffix='.part')
|
||||
tmp_path = Path(tmp_name)
|
||||
try:
|
||||
# fdopen outermost so the descriptor mkstemp handed back is
|
||||
# always adopted and closed, including when the request itself
|
||||
# raises — load_logo_with_download swallows that, so a leak
|
||||
# here would accumulate quietly on a URL that keeps failing.
|
||||
with os.fdopen(fd, 'wb') as f:
|
||||
with self.session.get(url, timeout=30, stream=True) as response:
|
||||
response.raise_for_status()
|
||||
downloaded = 0
|
||||
for chunk in response.iter_content(chunk_size=64 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
downloaded += len(chunk)
|
||||
if downloaded > MAX_LOGO_BYTES:
|
||||
raise ValueError(
|
||||
f"Logo at {url} exceeds the "
|
||||
f"{MAX_LOGO_BYTES}-byte limit; not saved")
|
||||
f.write(chunk)
|
||||
|
||||
# Verify it decodes before it becomes the cached logo. PIL
|
||||
# raises DecompressionBombError past its own pixel limit; a
|
||||
# partial or non-image response raises UnidentifiedImageError
|
||||
# (an OSError subclass).
|
||||
with Image.open(tmp_path) as probe:
|
||||
probe.load()
|
||||
|
||||
os.replace(tmp_path, file_path)
|
||||
except BaseException:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
# Set proper file permissions after saving
|
||||
ensure_file_permissions(file_path, get_assets_file_mode())
|
||||
|
||||
|
||||
self.logger.debug(f"Downloaded logo to {file_path}")
|
||||
|
||||
def _create_placeholder_logo(self, team_abbr: str,
|
||||
|
||||
@@ -328,7 +328,11 @@ class ScrollHelper:
|
||||
elapsed_time = current_time - (self.scroll_start_time or current_time)
|
||||
# The image already includes display_width padding, so we only need total_scroll_width
|
||||
required_total_distance = self.total_scroll_width
|
||||
self.logger.info(
|
||||
# Progress telemetry, emitted every few seconds for the whole of
|
||||
# every scroll. It says how far along a marquee is, which is what
|
||||
# you turn debug on to watch and not something an operator needs
|
||||
# in the journal on a device that scrolls all day.
|
||||
self.logger.debug(
|
||||
"Scroll progress: elapsed=%.2fs, target=%.2fs, total_scrolled=%.0f/%d px (%.1f%%)",
|
||||
elapsed_time,
|
||||
self.calculated_duration,
|
||||
|
||||
@@ -0,0 +1,460 @@
|
||||
"""Card-drawing helpers shared by every sports scoreboard plugin.
|
||||
|
||||
The eight scoreboards each carried byte-identical copies of the functions
|
||||
below: the colour pickers, the settings lookup, the date and time formatting,
|
||||
the favourite-team rules and the font-size grid snapping. One fix had to be
|
||||
made eight times, and a new scoreboard started by copying them a ninth.
|
||||
|
||||
Everything here is a **free function taking explicit arguments**, not a base
|
||||
class. Adoption is therefore per-function and reversible: a plugin keeps its
|
||||
method and delegates the body, so the call sites and the override points are
|
||||
untouched. That is also why `config`, `logger` and `fonts` are parameters
|
||||
rather than attributes -- the helper never reaches back into the caller.
|
||||
|
||||
The bodies are the plugins' own code, moved rather than rewritten. The one
|
||||
deliberate difference is `crisp_size`, which takes the seven-plugin guard
|
||||
(`not desired`) instead of football's: they agree on every real input, and
|
||||
the extra guard only stops a None size raising TypeError.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
__all__ = [
|
||||
"ELEMENT_FOR_FONT", "FAVORITE_RESULT_COLOR_DEFAULTS", "FONT_NAME_ALIASES",
|
||||
"FONT_PIXEL_GRID", "MONTH_ABBR", "WEEKDAY_ABBR",
|
||||
"scroll_card_option", "element_color", "font_color", "coerce_rgb",
|
||||
"score_color_for", "recent_score_color", "favorite_teams_for",
|
||||
"side_is_favorite", "side_score", "favorite_result",
|
||||
"card_tzinfo", "weekday_for", "format_game_date", "format_game_time",
|
||||
"vs_text", "upcoming_center_mode", "crisp_size", "schema_font_size",
|
||||
"resolve_font_size", "unshare_element_fonts",
|
||||
]
|
||||
|
||||
#: Which customization element owns each font key, for colour resolution.
|
||||
ELEMENT_FOR_FONT: Dict[str, str] = {
|
||||
"score": "score_text",
|
||||
"time": "period_text",
|
||||
"team": "team_name",
|
||||
"status": "status_text",
|
||||
"detail": "detail_text",
|
||||
"rank": "rank_text",
|
||||
}
|
||||
|
||||
#: Fallback colours when favourite_result_colors is on but a slot is unset.
|
||||
FAVORITE_RESULT_COLOR_DEFAULTS: Dict[str, Tuple[int, int, int]] = {
|
||||
"win": (0, 255, 0),
|
||||
"loss": (255, 0, 0),
|
||||
"tie": (255, 200, 0),
|
||||
}
|
||||
|
||||
#: Family aliases the web UI may write, mapped to the shipped filename.
|
||||
FONT_NAME_ALIASES: Dict[str, str] = {
|
||||
"press_start": "PressStart2P-Regular.ttf",
|
||||
"four_by_six": "4x6-font.ttf",
|
||||
}
|
||||
|
||||
#: Pixel grid each face renders crisply on. Off-grid sizes anti-alias, which
|
||||
#: on an LED matrix is a dim lamp rather than a soft edge.
|
||||
FONT_PIXEL_GRID: Dict[str, int] = {
|
||||
"PressStart2P-Regular.ttf": 8,
|
||||
"4x6-font.ttf": 7,
|
||||
}
|
||||
|
||||
MONTH_ABBR = ("Jan", "Feb", "Mar", "Apr", "May", "Jun",
|
||||
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
|
||||
WEEKDAY_ABBR = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Settings lookup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def scroll_card_option(config: Optional[Dict[str, Any]], key: str,
|
||||
default: Any = None) -> Any:
|
||||
"""Read one key from the scroll_card config block."""
|
||||
block = (config or {}).get("scroll_card")
|
||||
if isinstance(block, dict) and block.get(key) is not None:
|
||||
return block.get(key)
|
||||
return default
|
||||
|
||||
|
||||
def vs_text(config: Optional[Dict[str, Any]]) -> str:
|
||||
"""Separator drawn between the teams -- "VS", "@", "at", anything."""
|
||||
return str(scroll_card_option(config, "vs_text", "VS"))
|
||||
|
||||
|
||||
def upcoming_center_mode(config: Optional[Dict[str, Any]]) -> str:
|
||||
"""Middle of an upcoming card: 'vs', 'date_time' or 'none'."""
|
||||
mode = str(scroll_card_option(config, "upcoming_center", "vs") or "vs").lower()
|
||||
return mode if mode in ("vs", "date_time", "none") else "vs"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Colour
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def element_color(config: Optional[Dict[str, Any]], element: str,
|
||||
default: Tuple[int, int, int] = (255, 255, 255)):
|
||||
"""Per-element text colour from customization.<element>.text_color."""
|
||||
try:
|
||||
cfg = (config or {}).get("customization", {}).get(element, {})
|
||||
value = cfg.get("text_color")
|
||||
if isinstance(value, (list, tuple)) and len(value) == 3:
|
||||
return tuple(max(0, min(255, int(c))) for c in value)
|
||||
if isinstance(value, str) and value.startswith("#") and len(value) == 7:
|
||||
return tuple(int(value[i:i + 2], 16) for i in (1, 3, 5))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return default
|
||||
|
||||
|
||||
def font_color(config: Optional[Dict[str, Any]], fonts: Optional[Dict[str, Any]],
|
||||
font, default: Tuple[int, int, int] = (255, 255, 255)):
|
||||
"""Colour for whichever element owns this face.
|
||||
|
||||
Matched on identity, and deliberately gives up when one object is
|
||||
shared: the last-resort font path can hand the same face to several
|
||||
keys, and there is no right answer for which element's colour that is.
|
||||
White is what those draws used before, so ambiguity costs nothing.
|
||||
"""
|
||||
try:
|
||||
fonts = fonts or {}
|
||||
matches = [element for key, element in ELEMENT_FOR_FONT.items()
|
||||
if fonts.get(key) is font]
|
||||
if len(matches) == 1:
|
||||
return element_color(config, matches[0], default)
|
||||
except (AttributeError, TypeError):
|
||||
pass
|
||||
return default
|
||||
|
||||
|
||||
def coerce_rgb(value, fallback):
|
||||
"""Turn a configured [R, G, B] list into a clamped (r, g, b) tuple."""
|
||||
# Checked before unpacking: a 3-character string ("123") would otherwise
|
||||
# iterate into three digits and yield a colour rather than the fallback.
|
||||
if not isinstance(value, (list, tuple)) or len(value) != 3:
|
||||
return fallback
|
||||
try:
|
||||
r, g, b = (max(0, min(255, int(channel))) for channel in value)
|
||||
except (TypeError, ValueError):
|
||||
return fallback
|
||||
return (r, g, b)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Favourite teams
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def favorite_teams_for(config: Dict[str, Any], game: Dict[str, Any]) -> list:
|
||||
"""Favorite teams that apply to this game.
|
||||
|
||||
Both sources are used. Games carry the league manager's *resolved*
|
||||
favorites, which is the only place dynamic groups such as AP_TOP_25
|
||||
appear expanded; the config is read as well so an edit takes effect on
|
||||
already-fetched games, and so hand-built game dicts (tests, other
|
||||
callers) still work.
|
||||
"""
|
||||
favorites = list(game.get("favorite_teams") or [])
|
||||
league_config = config.get(str(game.get("league", "") or ""))
|
||||
if isinstance(league_config, dict):
|
||||
favorites += list(league_config.get("favorite_teams") or [])
|
||||
else:
|
||||
favorites += list(config.get("favorite_teams") or [])
|
||||
return favorites
|
||||
|
||||
|
||||
def side_is_favorite(game: Dict[str, Any], side: str, favorites: set) -> bool:
|
||||
"""Is the home/away side of this game a favorite team?
|
||||
|
||||
Reads both the flat (``home_abbr``) and nested (``home_team.abbrev``)
|
||||
payload shapes, and matches on the ESPN id too, because a couple of
|
||||
leagues (NRL) key favorites by id where abbreviations collide.
|
||||
"""
|
||||
candidates = [game.get(f"{side}_abbr"), game.get(f"{side}_id")]
|
||||
team = game.get(f"{side}_team")
|
||||
if isinstance(team, dict):
|
||||
candidates += [team.get("abbrev"), team.get("abbreviation"), team.get("id")]
|
||||
for value in candidates:
|
||||
if value is not None and str(value).strip().upper() in favorites:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def side_score(game: Dict[str, Any], side: str) -> Optional[int]:
|
||||
"""Numeric score for one side, from either payload shape."""
|
||||
raw = None
|
||||
team = game.get(f"{side}_team")
|
||||
if isinstance(team, dict) and team.get("score") is not None:
|
||||
raw = team.get("score")
|
||||
if raw is None:
|
||||
raw = game.get(f"{side}_score")
|
||||
try:
|
||||
return int(float(str(raw).strip()))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def favorite_result(config: Dict[str, Any], game: Dict[str, Any]) -> Optional[str]:
|
||||
"""Say how the favorite team did in a finished game.
|
||||
|
||||
Returns 'win', 'loss' or 'tie', or None when there is no single team
|
||||
to root for: no favorites configured, neither side is a favorite, or
|
||||
*both* are -- a favorite-vs-favorite game has no losing side worth
|
||||
flagging in red. Also None when the scores are not usable numbers.
|
||||
"""
|
||||
favorites = {
|
||||
str(team).strip().upper()
|
||||
for team in favorite_teams_for(config, game)
|
||||
if str(team).strip()
|
||||
}
|
||||
if not favorites:
|
||||
return None
|
||||
|
||||
home_fav = side_is_favorite(game, "home", favorites)
|
||||
away_fav = side_is_favorite(game, "away", favorites)
|
||||
if home_fav == away_fav:
|
||||
return None
|
||||
|
||||
home_score = side_score(game, "home")
|
||||
away_score = side_score(game, "away")
|
||||
if home_score is None or away_score is None:
|
||||
return None
|
||||
|
||||
if home_score == away_score:
|
||||
return "tie"
|
||||
favorite_score, other_score = (
|
||||
(home_score, away_score) if home_fav else (away_score, home_score)
|
||||
)
|
||||
return "win" if favorite_score > other_score else "loss"
|
||||
|
||||
|
||||
def recent_score_color(config: Dict[str, Any], logger, game: Dict[str, Any], default):
|
||||
"""Fill color for a finished game's score, per favorite_result_colors."""
|
||||
try:
|
||||
settings = (config.get("customization") or {}).get(
|
||||
"favorite_result_colors"
|
||||
) or {}
|
||||
if not settings.get("enabled", False):
|
||||
return default
|
||||
result = favorite_result(config, game)
|
||||
if result is None:
|
||||
return default
|
||||
return coerce_rgb(
|
||||
settings.get(f"{result}_color"),
|
||||
FAVORITE_RESULT_COLOR_DEFAULTS[result],
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Could not resolve favorite result color", exc_info=True)
|
||||
return default
|
||||
|
||||
|
||||
def score_color_for(config: Dict[str, Any], logger, game: Dict[str, Any],
|
||||
game_type: str, default=None):
|
||||
"""Fill color for a game card's score. Only finished games are tinted.
|
||||
|
||||
The default is the configured score colour rather than a flat white,
|
||||
so customization.score_text.text_color shows on games the favourite
|
||||
tint does not apply to. The tint still wins where it applies.
|
||||
"""
|
||||
if default is None:
|
||||
default = element_color(config, 'score_text')
|
||||
if game_type != "recent":
|
||||
return default
|
||||
return recent_score_color(config, logger, game, default)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Date and time
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def card_tzinfo(config: Optional[Dict[str, Any]], logger):
|
||||
"""Timezone for weekday/24h conversions; falls back to UTC."""
|
||||
configured = (config or {}).get("timezone")
|
||||
if configured:
|
||||
try:
|
||||
return ZoneInfo(configured)
|
||||
except (KeyError, ValueError, TypeError, OSError) as exc:
|
||||
# KeyError covers ZoneInfoNotFoundError. A bad zone name in
|
||||
# config should fall back to UTC, not blank the card.
|
||||
logger.debug("Unusable timezone %r: %s", configured, exc)
|
||||
return timezone.utc
|
||||
|
||||
|
||||
def weekday_for(config: Optional[Dict[str, Any]], logger,
|
||||
game: Optional[Dict]) -> str:
|
||||
"""Weekday abbreviation from the game's start time, or ''."""
|
||||
if not game:
|
||||
return ""
|
||||
raw = game.get("start_time_utc") or game.get("start_time")
|
||||
if not raw:
|
||||
return ""
|
||||
try:
|
||||
start = raw if isinstance(raw, datetime) else datetime.fromisoformat(
|
||||
str(raw).replace("Z", "+00:00"))
|
||||
return WEEKDAY_ABBR[start.astimezone(card_tzinfo(config, logger)).weekday()]
|
||||
except (ValueError, TypeError):
|
||||
return ""
|
||||
|
||||
|
||||
def format_game_date(config: Optional[Dict[str, Any]], logger, date_text: str,
|
||||
game: Optional[Dict] = None) -> str:
|
||||
"""Format an upcoming card's date per scroll_card.date_format."""
|
||||
raw = str(date_text or "").strip()
|
||||
if not raw:
|
||||
return ""
|
||||
fmt = str(scroll_card_option(config, "date_format", "abbrev") or "abbrev")
|
||||
if fmt == "numeric":
|
||||
return raw
|
||||
parts = raw.replace("-", "/").split("/")
|
||||
if not (len(parts) >= 2 and parts[0].strip().isdigit() and parts[1].strip().isdigit()):
|
||||
return raw
|
||||
month, day = int(parts[0]), int(parts[1])
|
||||
if not 1 <= month <= 12:
|
||||
return raw
|
||||
name = MONTH_ABBR[month - 1]
|
||||
if fmt == "numeric_day_first":
|
||||
return f"{day}/{month}"
|
||||
if fmt == "day_first":
|
||||
return f"{day} {name}"
|
||||
if fmt == "weekday":
|
||||
weekday = weekday_for(config, logger, game)
|
||||
return f"{weekday} {name} {day}" if weekday else f"{name} {day}"
|
||||
return f"{name} {day}"
|
||||
|
||||
|
||||
def format_game_time(config: Optional[Dict[str, Any]], time_text: str) -> str:
|
||||
"""Return the time as-is (12h) or converted to 24h."""
|
||||
raw = str(time_text or "").strip()
|
||||
if not raw or str(scroll_card_option(config, "time_format", "12h")) != "24h":
|
||||
return raw
|
||||
cleaned = raw.upper().replace(" ", "")
|
||||
meridiem = "AM" if cleaned.endswith("AM") else "PM" if cleaned.endswith("PM") else ""
|
||||
if not meridiem:
|
||||
return raw
|
||||
try:
|
||||
hh, _, mm = cleaned[:-2].partition(":")
|
||||
hour, minute = int(hh), int(mm or 0)
|
||||
except ValueError:
|
||||
return raw
|
||||
if not (0 <= hour <= 12 and 0 <= minute <= 59):
|
||||
return raw
|
||||
hour = hour % 12 + (12 if meridiem == "PM" else 0)
|
||||
return f"{hour:02d}:{minute:02d}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Font sizing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Per-schema caches, keyed by the schema's absolute path. Keyed rather than
|
||||
#: global because each plugin declares its own defaults; keyed rather than
|
||||
#: per-class because the helper has no class to hang it on.
|
||||
_SCHEMA_FONT_SIZE_CACHE: Dict[str, Dict[str, int]] = {}
|
||||
|
||||
|
||||
def crisp_size(font_file, desired, aliases=None, grid_table=None):
|
||||
"""Snap *desired* to the nearest size *font_file* renders crisply at.
|
||||
|
||||
A face with no known grid is returned unchanged, so a user-supplied
|
||||
font is never second-guessed.
|
||||
|
||||
``aliases`` and ``grid_table`` default to the shared tables; a plugin
|
||||
that ships an extra face can pass its own without forking this.
|
||||
"""
|
||||
aliases = FONT_NAME_ALIASES if aliases is None else aliases
|
||||
grid_table = FONT_PIXEL_GRID if grid_table is None else grid_table
|
||||
font_file = aliases.get(font_file, font_file)
|
||||
grid = grid_table.get(font_file)
|
||||
if not grid or not desired or desired <= 0:
|
||||
return desired
|
||||
return max(grid, int(round(float(desired) / grid)) * grid)
|
||||
|
||||
|
||||
def schema_font_size(schema_path: str, element_key) -> Optional[int]:
|
||||
"""The font_size this plugin's config_schema.json declares, or None.
|
||||
|
||||
Cached per schema path. The plugins cached this on their own class; the
|
||||
path is the same distinction expressed without one, so two plugins never
|
||||
share an entry.
|
||||
"""
|
||||
if not element_key:
|
||||
return None
|
||||
cache = _SCHEMA_FONT_SIZE_CACHE.get(schema_path)
|
||||
if cache is None:
|
||||
cache = {}
|
||||
try:
|
||||
import json
|
||||
with open(schema_path) as fh:
|
||||
schema = json.load(fh)
|
||||
props = (schema.get('properties', {})
|
||||
.get('customization', {})
|
||||
.get('properties', {}))
|
||||
for key, spec in props.items():
|
||||
size = spec.get('properties', {}).get('font_size', {}).get('default')
|
||||
if size is not None:
|
||||
cache[key] = int(size)
|
||||
except Exception:
|
||||
cache = {}
|
||||
_SCHEMA_FONT_SIZE_CACHE[schema_path] = cache
|
||||
return cache.get(element_key)
|
||||
|
||||
|
||||
def resolve_font_size(schema_path: str, element_config, element_key,
|
||||
default_size, font_name, aliases=None, grid_table=None):
|
||||
"""Size to render at: the user's choice, or a grid-snapped default.
|
||||
|
||||
A configured size counts as a real choice only when it differs from
|
||||
the schema default. The web UI writes the whole schema default block
|
||||
on every save, so "font_size == schema default" carries no intent and
|
||||
would otherwise pin every install to an anti-aliased size forever.
|
||||
"""
|
||||
configured = (element_config or {}).get('font_size')
|
||||
if configured is not None:
|
||||
try:
|
||||
configured = int(configured)
|
||||
if configured != schema_font_size(schema_path, element_key):
|
||||
return configured
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return crisp_size(font_name, default_size, aliases, grid_table)
|
||||
|
||||
|
||||
def unshare_element_fonts(logger, fonts):
|
||||
"""Give each colourable element its own face object.
|
||||
|
||||
The colour a draw gets is resolved from the face it was handed, and
|
||||
several of these loaders legitimately hand one object to more than one
|
||||
element -- a size resolver that lands two elements on the same face, a
|
||||
fallback that fills every key from one default, football's narrowing
|
||||
step that deliberately shrinks the clock along with the score. Sharing
|
||||
the object makes the element ambiguous and the colour unresolvable.
|
||||
|
||||
Re-instantiating from the same path and size gives a distinct object
|
||||
with identical metrics, so nothing about the rendering changes; only
|
||||
the ability to tell two elements apart does. Faces that cannot be
|
||||
rebuilt (a BDF loaded through freetype.Face, anything without a usable
|
||||
path) are left shared, and their draws stay white as before.
|
||||
"""
|
||||
try:
|
||||
from PIL import ImageFont as _IF
|
||||
except ImportError: # pragma: no cover
|
||||
return fonts
|
||||
seen = {}
|
||||
for key in ELEMENT_FOR_FONT:
|
||||
font = fonts.get(key)
|
||||
if font is None:
|
||||
continue
|
||||
if id(font) not in seen:
|
||||
seen[id(font)] = key
|
||||
continue
|
||||
path, size = getattr(font, "path", None), getattr(font, "size", None)
|
||||
if not path or not size:
|
||||
continue
|
||||
try:
|
||||
fonts[key] = _IF.truetype(path, size)
|
||||
except (OSError, ValueError, TypeError):
|
||||
logger.debug(
|
||||
"Could not un-share the %s face; it keeps the default colour", key)
|
||||
return fonts
|
||||
@@ -0,0 +1,261 @@
|
||||
"""The scroll/Vegas card geometry the sports scoreboards all share.
|
||||
|
||||
Eight scoreboards -- afl, baseball, basketball, football, hockey, lacrosse,
|
||||
nrl and soccer -- each carried their own ``game_renderer.py``. After the card
|
||||
helpers moved to ``src/common/sports_card.py`` the settings lookups were
|
||||
shared, but the *geometry* was not: nine methods that decide how wide the
|
||||
centre gap is, how much room a logo gets, and where an upcoming card's date
|
||||
and time land were still eight separate copies. Five were byte-identical
|
||||
across all eight plugins; the other four were identical in seven, with a
|
||||
different single outlier each time.
|
||||
|
||||
That last detail is why this is a mixin rather than free functions. There is
|
||||
no per-sport branching to write -- baseball needs its own
|
||||
``_draw_upcoming_game_status`` and ``_logo_slot_width``, hockey its own
|
||||
``_upcoming_date_and_time``, football its own ``_score_reserve_width``, and
|
||||
every one of those is an ordinary override. The seven that agree inherit and
|
||||
say nothing.
|
||||
|
||||
It is deliberately *only* a mixin: no ``__init__``, no state of its own. The
|
||||
plugins' constructors differ in six ways and none of that difference is worth
|
||||
unifying, so adoption is one line on the class statement plus deleting the
|
||||
methods that now come from here.
|
||||
|
||||
What a host class must provide
|
||||
------------------------------
|
||||
Attributes: ``display_width``, ``display_height``, ``config``, ``fonts``,
|
||||
``logger``, ``_team_rankings_cache``.
|
||||
|
||||
Methods: ``_draw_text_with_outline(draw, text, position, font, fill=None,
|
||||
outline_color=(0, 0, 0))`` -- the one hook whose body genuinely varies -- plus
|
||||
the ``sports_card`` delegations ``_scroll_card_option``,
|
||||
``_upcoming_center_mode``, ``_vs_text``, ``_element_color``,
|
||||
``_format_game_date`` and ``_format_game_time``.
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import ClassVar, Dict, Tuple
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
class SportsGameRendererMixin:
|
||||
"""Shared card geometry for the sports scoreboards. See module docstring."""
|
||||
|
||||
#: Centre strip as a fraction of card width, before clamping.
|
||||
CENTER_GAP_RATIO: ClassVar[float] = 0.28
|
||||
#: Clamp floor for the derived centre gap.
|
||||
CENTER_GAP_MIN_PX: ClassVar[int] = 22
|
||||
#: Clamp ceiling for the derived centre gap.
|
||||
CENTER_GAP_MAX_PX: ClassVar[int] = 40
|
||||
#: Breathing room kept between the score and each logo.
|
||||
_SCORE_LOGO_GUTTER_PX: ClassVar[int] = 4
|
||||
#: Widest score the centre strip must fit. Leagues that can reach three
|
||||
#: digits a side override this with "000-000".
|
||||
_SCORE_PROBE: ClassVar[str] = "00-00"
|
||||
|
||||
# ---- geometry ------------------------------------------------------
|
||||
|
||||
# Non-finite settings are rejected before any int()/round(): "inf" reaches
|
||||
# these from config as a float or a string, passes an `isinstance` plus
|
||||
# `>= 0` check unharmed, and then raises OverflowError out of int() --
|
||||
# which the old `except (TypeError, ValueError)` did not catch, so it
|
||||
# aborted the whole card render. Present in all eight plugins before this
|
||||
# moved to the core; fixing it here fixes it in all eight.
|
||||
|
||||
def _score_reserve_width(self) -> int:
|
||||
"""Centre strip the score actually needs, measured rather than assumed.
|
||||
|
||||
The gap was derived from the card width alone (width x
|
||||
CENTER_GAP_RATIO, clamped to CENTER_GAP_MAX_PX) while the score's size
|
||||
comes from config and the element-style resolver. Nothing compared the
|
||||
two, so any score wider than the clamp was drawn over the logos.
|
||||
Measuring it keeps the strip wide enough for whatever font is in play.
|
||||
"""
|
||||
try:
|
||||
probe = ImageDraw.Draw(Image.new("RGB", (4, 4)))
|
||||
width = probe.textlength(self._SCORE_PROBE, font=self.fonts['score'])
|
||||
return int(width) + 2 * self._SCORE_LOGO_GUTTER_PX
|
||||
except Exception:
|
||||
self.logger.debug("Score reserve measurement failed", exc_info=True)
|
||||
return 0
|
||||
|
||||
def _center_gap_width(self) -> int:
|
||||
"""Width of the middle strip kept clear of logos.
|
||||
|
||||
``scroll_card.center_gap`` pins it outright; otherwise it scales with
|
||||
the card width between the configurable min and max. 0 restores
|
||||
edge-to-edge logos.
|
||||
"""
|
||||
configured = self._scroll_card_option("center_gap")
|
||||
if (isinstance(configured, (int, float))
|
||||
and math.isfinite(configured) and configured >= 0):
|
||||
return int(configured)
|
||||
ratio = self._scroll_card_option("center_gap_ratio", self.CENTER_GAP_RATIO)
|
||||
low = self._scroll_card_option("center_gap_min", self.CENTER_GAP_MIN_PX)
|
||||
high = self._scroll_card_option("center_gap_max", self.CENTER_GAP_MAX_PX)
|
||||
try:
|
||||
ratio, low, high = float(ratio), float(low), float(high)
|
||||
if not (math.isfinite(ratio) and math.isfinite(low)
|
||||
and math.isfinite(high)):
|
||||
return self.CENTER_GAP_MIN_PX
|
||||
scaled = round(self.display_width * ratio)
|
||||
derived = int(max(int(low), min(int(high), scaled)))
|
||||
# A strip narrower than the score is the bug, not a style choice.
|
||||
# An explicit ``center_gap`` is still honoured above, including 0.
|
||||
return max(derived, self._score_reserve_width())
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return self.CENTER_GAP_MIN_PX
|
||||
|
||||
def _logo_slot_width(self) -> int:
|
||||
"""Per-side logo slot, leaving the center gap clear.
|
||||
|
||||
No longer capped at display_height: the card is sized as two
|
||||
full-height logos plus the measured gap, so what is left after the gap
|
||||
is exactly the logo's share. The cap was what froze the logos at 46px
|
||||
on the old flat 128px card.
|
||||
"""
|
||||
available = (self.display_width - self._center_gap_width()) // 2
|
||||
return max(8, available)
|
||||
|
||||
def _logo_cache_key(self, name: str) -> str:
|
||||
"""Cache key scoped to the logo slot.
|
||||
|
||||
One cache dict is shared by renderers built for different card widths,
|
||||
so a logo sized for a wide slot must not be handed to a narrow one.
|
||||
"""
|
||||
return f"{name}@{self._logo_slot_width()}x{self.display_height}"
|
||||
|
||||
def _layout_offset(self, element: str, axis: str, default: int = 0) -> int:
|
||||
"""X/Y nudge for one element, from customization.layout.
|
||||
|
||||
Same block the full-screen scorebug reads (sports.py
|
||||
_get_layout_offset), so a nudge configured in the web UI now moves
|
||||
the element on the scroll/Vegas card too -- previously the schema
|
||||
advertised these offsets but this renderer ignored them.
|
||||
"""
|
||||
try:
|
||||
layout = (self.config or {}).get("customization", {}).get("layout", {})
|
||||
value = (layout.get(element) or {}).get(axis, default)
|
||||
if isinstance(value, bool):
|
||||
return default
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value) if math.isfinite(value) else default
|
||||
if isinstance(value, str):
|
||||
parsed = float(value)
|
||||
return int(parsed) if math.isfinite(parsed) else default
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
pass
|
||||
return default
|
||||
|
||||
# ---- upcoming cards ------------------------------------------------
|
||||
|
||||
def _upcoming_date_and_time(self, game: Dict) -> Tuple[str, str]:
|
||||
"""(date, time) for an upcoming card, from the extractor's flat keys."""
|
||||
return (
|
||||
str(game.get("game_date", "") or ""),
|
||||
str(game.get("game_time", "") or ""),
|
||||
)
|
||||
|
||||
def _draw_upcoming_center(self, draw: "ImageDraw.ImageDraw", game: Dict) -> None:
|
||||
"""Draw the middle of an upcoming card.
|
||||
|
||||
Never a score: an upcoming game has not started, so the extractor's
|
||||
0-0 is noise. Either the VS text (default), the date and time stacked,
|
||||
or nothing at all.
|
||||
"""
|
||||
mode = self._upcoming_center_mode()
|
||||
if mode == "none":
|
||||
return
|
||||
|
||||
if mode == "vs":
|
||||
vs_text = self._vs_text()
|
||||
if not vs_text:
|
||||
return
|
||||
vs_width = draw.textlength(vs_text, font=self.fonts['score'])
|
||||
vs_x = (self.display_width - vs_width) // 2 + self._layout_offset('score', 'x_offset')
|
||||
vs_y = (self.display_height // 2) - 3 + self._layout_offset('score', 'y_offset')
|
||||
self._draw_text_with_outline(
|
||||
draw, vs_text, (vs_x, vs_y), self.fonts['score'],
|
||||
fill=self._element_color('score_text')
|
||||
)
|
||||
return
|
||||
|
||||
date_text, time_text = self._upcoming_date_and_time(game)
|
||||
lines = []
|
||||
if self._scroll_card_option("show_date", True):
|
||||
lines.append(self._format_game_date(date_text, game))
|
||||
if self._scroll_card_option("show_time", True):
|
||||
lines.append(self._format_game_time(time_text))
|
||||
lines = [t for t in lines if t]
|
||||
if not lines:
|
||||
return
|
||||
font = self.fonts.get('detail') or self.fonts['time']
|
||||
line_h = 7
|
||||
top = (self.display_height // 2) - (len(lines) * line_h) // 2
|
||||
top += self._layout_offset('score', 'y_offset')
|
||||
for i, line in enumerate(lines):
|
||||
width = draw.textlength(line, font=font)
|
||||
x = (self.display_width - width) // 2 + self._layout_offset('score', 'x_offset')
|
||||
self._draw_text_with_outline(
|
||||
draw, line, (x, top + i * line_h), font,
|
||||
fill=self._element_color('detail_text')
|
||||
)
|
||||
|
||||
def _draw_upcoming_game_status(self, draw: "ImageDraw.ImageDraw", game: Dict) -> None:
|
||||
"""Draw the date and time around an upcoming card.
|
||||
|
||||
Time top and date bottom by default; scroll_card.swap_date_time puts
|
||||
the date on top instead. Skipped when the pair is stacked in the
|
||||
middle, which would otherwise print them twice.
|
||||
"""
|
||||
if self._upcoming_center_mode() == "date_time":
|
||||
return
|
||||
|
||||
date_raw, time_raw = self._upcoming_date_and_time(game)
|
||||
date_text = (self._format_game_date(date_raw, game)
|
||||
if self._scroll_card_option("show_date", True) else "")
|
||||
time_text = (self._format_game_time(time_raw)
|
||||
if self._scroll_card_option("show_time", True) else "")
|
||||
|
||||
if self._scroll_card_option("swap_date_time", False):
|
||||
top_text, top_el, bottom_text, bottom_el = (
|
||||
date_text, 'date', time_text, 'time')
|
||||
top_font = self.fonts.get('detail') or self.fonts['time']
|
||||
bottom_font = self.fonts['time']
|
||||
top_color, bottom_color = 'detail_text', 'period_text'
|
||||
else:
|
||||
top_text, top_el, bottom_text, bottom_el = (
|
||||
time_text, 'time', date_text, 'date')
|
||||
top_font = self.fonts['time']
|
||||
bottom_font = self.fonts.get('detail') or self.fonts['time']
|
||||
top_color, bottom_color = 'period_text', 'detail_text'
|
||||
|
||||
if top_text:
|
||||
top_width = draw.textlength(top_text, font=top_font)
|
||||
top_x = (self.display_width - top_width) // 2 + self._layout_offset(top_el, 'x_offset')
|
||||
top_y = 1 + self._layout_offset(top_el, 'y_offset')
|
||||
self._draw_text_with_outline(
|
||||
draw, top_text, (top_x, top_y), top_font,
|
||||
fill=self._element_color(top_color)
|
||||
)
|
||||
|
||||
if bottom_text:
|
||||
bottom_width = draw.textlength(bottom_text, font=bottom_font)
|
||||
bottom_x = ((self.display_width - bottom_width) // 2
|
||||
+ self._layout_offset(bottom_el, 'x_offset'))
|
||||
# Measured, not a fixed -7: the detail font is 6px in most plugins
|
||||
# but 10px in soccer and nrl, where "Sep 19" ran past the card.
|
||||
ink_bottom = draw.textbbox((0, 0), bottom_text, font=bottom_font)[3]
|
||||
bottom_y = (max(0, self.display_height - ink_bottom - 1)
|
||||
+ self._layout_offset(bottom_el, 'y_offset'))
|
||||
self._draw_text_with_outline(
|
||||
draw, bottom_text, (bottom_x, bottom_y), bottom_font,
|
||||
fill=self._element_color(bottom_color)
|
||||
)
|
||||
|
||||
# ---- rankings ------------------------------------------------------
|
||||
|
||||
def set_rankings_cache(self, rankings: Dict[str, int]) -> None:
|
||||
"""Set the team rankings cache for display."""
|
||||
self._team_rankings_cache = rankings
|
||||
File diff suppressed because it is too large
Load Diff
+94
-36
@@ -19,6 +19,7 @@ Port default: 5765 (UDP). Open this port on both Pis if ufw is active:
|
||||
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
@@ -37,6 +38,13 @@ _RAW_MAGIC = b'SYNC_RAW'
|
||||
_RAW_HEADER = struct.Struct('<HH') # width, height (uint16 LE)
|
||||
|
||||
|
||||
# Upper bound on a decoded frame/scroll image. Generous for any real scroll
|
||||
# image (a leader's full cycle is long but only panel-height tall), and low
|
||||
# enough that a crafted image from any host on the LAN cannot force a large
|
||||
# allocation on the render thread. Applied on both receive paths — the TCP
|
||||
# image server and the follower's legacy-PNG UDP fallback.
|
||||
_MAX_FRAME_W, _MAX_FRAME_H = 100_000, 256
|
||||
|
||||
SYNC_PORT = 5765
|
||||
HELLO_INTERVAL = 5.0 # follower broadcasts hello every 5 s
|
||||
HEARTBEAT_INTERVAL = 2.0 # follower sends heartbeat every 2 s
|
||||
@@ -101,6 +109,7 @@ class DisplaySyncManager:
|
||||
self._peer_chain: int = 0
|
||||
self._last_heartbeat_time: float = 0.0
|
||||
self._leader_width: int = 0 # set by display_controller after init
|
||||
self._oversized_frame_warned: bool = False
|
||||
|
||||
# Follower state
|
||||
self._follower_state = FollowerState.STANDALONE
|
||||
@@ -174,6 +183,10 @@ class DisplaySyncManager:
|
||||
continue
|
||||
except Exception as exc:
|
||||
self.logger.debug("Sync leader recv error: %s", exc)
|
||||
# Brief backoff: a socket left in a bad state raises
|
||||
# immediately, which would otherwise spin this thread at
|
||||
# 100% CPU logging the same error.
|
||||
time.sleep(0.1)
|
||||
|
||||
def _handle_hello(self, msg: dict, sender_ip: str) -> None:
|
||||
hw = self._hw_config
|
||||
@@ -273,11 +286,10 @@ class DisplaySyncManager:
|
||||
break
|
||||
data.extend(chunk)
|
||||
img = Image.open(io.BytesIO(data))
|
||||
_MAX_W, _MAX_H = 100_000, 256 # generous for any real scroll image
|
||||
if img.width > _MAX_W or img.height > _MAX_H:
|
||||
if img.width > _MAX_FRAME_W or img.height > _MAX_FRAME_H:
|
||||
self.logger.warning(
|
||||
"Sync: rejected oversized scroll image %dx%d (max %dx%d) from %s",
|
||||
img.width, img.height, _MAX_W, _MAX_H, addr,
|
||||
img.width, img.height, _MAX_FRAME_W, _MAX_FRAME_H, addr,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
@@ -396,7 +408,7 @@ class DisplaySyncManager:
|
||||
data = header + arr.tobytes()
|
||||
if len(data) <= 65000:
|
||||
self._send_sock.sendto(data, (self._peer_ip, self.port))
|
||||
elif not getattr(self, '_oversized_frame_warned', False):
|
||||
elif not self._oversized_frame_warned:
|
||||
self._oversized_frame_warned = True
|
||||
self.logger.warning(
|
||||
"Sync: frame too large for UDP (%d bytes, max 65000) — "
|
||||
@@ -451,43 +463,76 @@ class DisplaySyncManager:
|
||||
)
|
||||
self.write_status_file()
|
||||
|
||||
def _handle_received_frame(self, img: Image.Image, sender_ip: str) -> None:
|
||||
"""Record a decoded leader frame and enter follower mode if needed."""
|
||||
with self._frame_lock:
|
||||
self._latest_frame = img
|
||||
self._last_leader_frame_time = time.time()
|
||||
self._leader_ip = sender_ip
|
||||
|
||||
if self._follower_state == FollowerState.STANDALONE:
|
||||
self._follower_state = FollowerState.FOLLOWER
|
||||
self.logger.info(
|
||||
"Sync: leader active at %s — switching to follower mode",
|
||||
sender_ip,
|
||||
)
|
||||
self.write_status_file()
|
||||
|
||||
def _follower_recv_loop(self) -> None:
|
||||
while self._running:
|
||||
try:
|
||||
data, addr = self._recv_sock.recvfrom(65535)
|
||||
sender_ip = addr[0]
|
||||
|
||||
if data[:8] == _RAW_MAGIC or len(data) > 512:
|
||||
# Frame data: prefer magic-tagged raw RGB; fall back to legacy PNG
|
||||
if data[:8] == _RAW_MAGIC:
|
||||
# Magic-tagged raw RGB frame — self-describing, no guessing.
|
||||
try:
|
||||
if data[:8] == _RAW_MAGIC:
|
||||
w, h = _RAW_HEADER.unpack(data[8:12])
|
||||
raw = data[12:]
|
||||
img = Image.frombuffer(
|
||||
"RGB", (w, h), raw, "raw", "RGB", 0, 1
|
||||
)
|
||||
else:
|
||||
# Fallback: try legacy PNG
|
||||
img = Image.open(io.BytesIO(data))
|
||||
img.load()
|
||||
with self._frame_lock:
|
||||
self._latest_frame = img
|
||||
self._last_leader_frame_time = time.time()
|
||||
self._leader_ip = sender_ip
|
||||
|
||||
if self._follower_state == FollowerState.STANDALONE:
|
||||
self._follower_state = FollowerState.FOLLOWER
|
||||
self.logger.info(
|
||||
"Sync: leader active at %s — switching to follower mode",
|
||||
sender_ip,
|
||||
)
|
||||
self.write_status_file()
|
||||
w, h = _RAW_HEADER.unpack(data[8:12])
|
||||
raw = data[12:]
|
||||
img = Image.frombuffer(
|
||||
"RGB", (w, h), raw, "raw", "RGB", 0, 1
|
||||
)
|
||||
self._handle_received_frame(img, sender_ip)
|
||||
except Exception as exc:
|
||||
self.logger.debug("Sync: frame decode error: %s", exc)
|
||||
else:
|
||||
# Control message
|
||||
# No magic prefix. Whether the payload parses as JSON
|
||||
# decides between a control message and a legacy
|
||||
# (pre-magic) PNG frame — both wire formats are
|
||||
# self-describing, so no size heuristic is needed. A
|
||||
# >512-byte control message used to be misrouted into
|
||||
# image decode and silently dropped.
|
||||
try:
|
||||
msg = json.loads(data.decode("utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
# Not JSON — try a legacy PNG frame.
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
if img.width > _MAX_FRAME_W or img.height > _MAX_FRAME_H:
|
||||
# Same cap the TCP image path applies: decode
|
||||
# is deferred until load(), so check first.
|
||||
self.logger.debug(
|
||||
"Sync: rejected oversized legacy frame %dx%d from %s",
|
||||
img.width, img.height, sender_ip,
|
||||
)
|
||||
continue
|
||||
img.load()
|
||||
self._handle_received_frame(img, sender_ip)
|
||||
except Exception as exc:
|
||||
self.logger.debug("Sync: frame decode error: %s", exc)
|
||||
continue
|
||||
|
||||
# It parsed, so it is a control message and never a
|
||||
# frame. Read and validate its fields under a guard —
|
||||
# a UDP payload is attacker-shaped, so a non-object
|
||||
# body makes .get() raise AttributeError and an "sx"
|
||||
# carrying a non-numeric x raises ValueError/TypeError
|
||||
# — but dispatch the callback *outside* it. Running
|
||||
# the callback in here would let a fault in someone
|
||||
# else's code read as a malformed packet and be
|
||||
# logged as one.
|
||||
fire_new_cycle = False
|
||||
try:
|
||||
t = msg.get("t")
|
||||
if t == "hello_ack":
|
||||
self._leader_ip = sender_ip
|
||||
@@ -501,7 +546,17 @@ class DisplaySyncManager:
|
||||
self.write_status_file()
|
||||
elif t == "sx":
|
||||
# Vegas scroll-position sync — tiny message, renders locally
|
||||
self._latest_scroll_x = float(msg["x"])
|
||||
scroll_x = float(msg["x"])
|
||||
if not math.isfinite(scroll_x):
|
||||
# json.loads accepts the NaN/Infinity literals,
|
||||
# and float("nan") accepts the strings, so a
|
||||
# non-finite x reaches here intact. Left alone
|
||||
# it poisons every offset computed from it —
|
||||
# NaN comparisons are all false, so the
|
||||
# follower renders a frame it can never scroll
|
||||
# back from. Treat it as malformed.
|
||||
raise ValueError(f"non-finite scroll x: {msg['x']!r}")
|
||||
self._latest_scroll_x = scroll_x
|
||||
self._last_leader_frame_time = time.time()
|
||||
self._leader_ip = sender_ip
|
||||
if self._follower_state == FollowerState.STANDALONE:
|
||||
@@ -511,19 +566,22 @@ class DisplaySyncManager:
|
||||
sender_ip,
|
||||
)
|
||||
self.write_status_file()
|
||||
if self._on_new_cycle:
|
||||
self._on_new_cycle() # build initial scroll image
|
||||
fire_new_cycle = True # build initial scroll image
|
||||
elif t == "nc":
|
||||
# Leader started a new scroll cycle — rebuild local image
|
||||
if self._on_new_cycle:
|
||||
self._on_new_cycle()
|
||||
except (json.JSONDecodeError, UnicodeDecodeError, KeyError):
|
||||
pass
|
||||
fire_new_cycle = True
|
||||
except (KeyError, AttributeError, TypeError, ValueError) as exc:
|
||||
self.logger.debug("Sync: malformed control message: %s", exc)
|
||||
continue
|
||||
|
||||
if fire_new_cycle and self._on_new_cycle:
|
||||
self._on_new_cycle()
|
||||
|
||||
except socket.timeout:
|
||||
continue
|
||||
except Exception as exc:
|
||||
self.logger.debug("Sync follower recv error: %s", exc)
|
||||
time.sleep(0.1)
|
||||
|
||||
def _follower_announce_loop(self) -> None:
|
||||
hw = self._hw_config
|
||||
|
||||
@@ -181,6 +181,16 @@ class DisplayController:
|
||||
self.plugin_modes = {} # mode -> plugin_instance mapping for plugin-first dispatch
|
||||
self.mode_to_plugin_id: Dict[str, str] = {}
|
||||
self.plugin_display_modes: Dict[str, List[str]] = {}
|
||||
# plugin_display_modes is mutated only by _register_loaded_plugin /
|
||||
# _unregister_plugin on the render thread, but the config-watcher
|
||||
# thread reads it in _enabled_plugin_not_running. Both mutation sites
|
||||
# run during reconcile (rare), so this lock never touches the per-frame
|
||||
# path -- the hot-path reads are same-thread as the writes.
|
||||
self._plugin_modes_lock = threading.Lock()
|
||||
# Guards the consume-and-clear of _pending_plugin_reconcile. Only taken
|
||||
# when a reconcile is actually pending or a config change arrives, both
|
||||
# rare -- the per-frame path just reads the bool.
|
||||
self._reconcile_flag_lock = threading.Lock()
|
||||
# Per-plugin config-change callbacks, kept so we can unsubscribe a
|
||||
# plugin when it is disabled live.
|
||||
self._plugin_config_callbacks: Dict[str, Callable] = {}
|
||||
@@ -463,8 +473,10 @@ class DisplayController:
|
||||
self._refresh_config_cache(new_config)
|
||||
# If a plugin was enabled/disabled, flag a reconcile for the main
|
||||
# loop to apply (loading/unloading off the watcher thread is unsafe).
|
||||
if self._enabled_set_changed(old_config, new_config):
|
||||
self._pending_plugin_reconcile = True
|
||||
if (self._enabled_set_changed(old_config, new_config)
|
||||
or self._enabled_plugin_not_running(new_config)):
|
||||
with self._reconcile_flag_lock:
|
||||
self._pending_plugin_reconcile = True
|
||||
|
||||
self.config_service.subscribe(_controller_config_change)
|
||||
|
||||
@@ -1749,11 +1761,12 @@ class DisplayController:
|
||||
# rebuilding available_modes happens here on the render thread so
|
||||
# it can't race with rendering. Deferred while on-demand is active
|
||||
# (the flag stays set) so we don't fight its temporary-enable.
|
||||
# The lock-free read is a fast path only; it can be a false
|
||||
# negative (the watcher setting the flag just after it is read
|
||||
# is seen next iteration), never a false positive that loses a
|
||||
# request.
|
||||
if self._pending_plugin_reconcile and not self.on_demand_active:
|
||||
# Only clear the flag on success -- a retryable failure
|
||||
# (e.g. discovery) leaves it set so the request isn't lost.
|
||||
if self._reconcile_enabled_plugins():
|
||||
self._pending_plugin_reconcile = False
|
||||
self._service_pending_reconcile()
|
||||
|
||||
if not self.available_modes:
|
||||
# Nothing to render yet. Re-check _pending_plugin_reconcile
|
||||
@@ -2813,7 +2826,8 @@ class DisplayController:
|
||||
logger.debug("Using manifest display_modes for %s: %s", plugin_id, display_modes)
|
||||
if not (isinstance(display_modes, list) and display_modes):
|
||||
display_modes = [plugin_id]
|
||||
self.plugin_display_modes[plugin_id] = list(display_modes)
|
||||
with self._plugin_modes_lock:
|
||||
self.plugin_display_modes[plugin_id] = list(display_modes)
|
||||
|
||||
# Subscribe to config changes for per-plugin hot-reload. Bind plugin_id
|
||||
# and instance as defaults so each plugin's callback targets its own
|
||||
@@ -2847,7 +2861,8 @@ class DisplayController:
|
||||
def _unregister_plugin(self, plugin_id: str) -> None:
|
||||
"""Remove a plugin's modes, config subscription and instance, then
|
||||
unload it. Used by live disable hot-reload."""
|
||||
modes = self.plugin_display_modes.pop(plugin_id, [])
|
||||
with self._plugin_modes_lock:
|
||||
modes = self.plugin_display_modes.pop(plugin_id, [])
|
||||
for mode in modes:
|
||||
if mode in self.available_modes:
|
||||
self.available_modes.remove(mode)
|
||||
@@ -2892,6 +2907,67 @@ class DisplayController:
|
||||
}
|
||||
return enabled_map(old_config) != enabled_map(new_config)
|
||||
|
||||
def _service_pending_reconcile(self) -> None:
|
||||
"""Consume a pending reconcile request and run it.
|
||||
|
||||
The request is consumed BEFORE reconciling, not cleared after. Clearing
|
||||
after would drop any config change that lands while reconcile is
|
||||
running: reconcile has already read its config by then, so the clear
|
||||
erases a request it never served and the newest config never
|
||||
reconciles -- the same "your save did nothing" failure this whole path
|
||||
exists to prevent. Consuming first means such a request stays set and
|
||||
is picked up on the next pass.
|
||||
|
||||
A retryable failure (e.g. discovery) re-arms the flag.
|
||||
"""
|
||||
with self._reconcile_flag_lock:
|
||||
pending = self._pending_plugin_reconcile
|
||||
self._pending_plugin_reconcile = False
|
||||
if pending and not self._reconcile_enabled_plugins():
|
||||
with self._reconcile_flag_lock:
|
||||
self._pending_plugin_reconcile = True
|
||||
|
||||
def _enabled_plugin_not_running(self, new_config: Dict[str, Any]) -> bool:
|
||||
"""True when a discovered plugin is enabled in config but not running.
|
||||
|
||||
``_enabled_set_changed`` compares only top-level ``enabled`` flags, which
|
||||
misses the case that strands a plugin: one whose ``validate_config()``
|
||||
returned False is absent from the running set, and the edit that fixes it
|
||||
(enabling a league, filling in an API key) lives *nested* inside that
|
||||
plugin's own section. No top-level flag changes, so no reconcile is
|
||||
queued, and the save that should have fixed it appears to do nothing --
|
||||
only toggling some unrelated plugin recovers it. hockey-scoreboard sat
|
||||
enabled-but-absent on a live rig for four days this way.
|
||||
|
||||
Deliberately narrow: it fires only for ids the plugin manager has
|
||||
actually discovered, so non-plugin sections that carry their own
|
||||
``enabled`` flag (``schedule``, ``display``, ...) don't queue a reconcile
|
||||
on every save. In the steady state -- everything enabled is loaded --
|
||||
this is False and costs nothing. That matters because reconcile runs
|
||||
``discover_plugins()`` on the render thread, where a needless
|
||||
filesystem scan per config save would show up as a frame hitch.
|
||||
|
||||
Runs on the config-watcher thread, so both mappings it reads are
|
||||
snapshotted under the lock that guards their writes.
|
||||
"""
|
||||
if self.plugin_manager is None:
|
||||
return False
|
||||
# Two snapshots, each taken under its own lock and never nested, so a
|
||||
# half-written mapping is never observed and this can't deadlock
|
||||
# against discovery (which holds the discovery lock while rebuilding).
|
||||
try:
|
||||
known = self.plugin_manager.discovered_plugin_ids()
|
||||
except AttributeError:
|
||||
# Older manager without the accessor: fall back to a plain read.
|
||||
known = set(getattr(self.plugin_manager, 'plugin_manifests', ()) or ())
|
||||
with self._plugin_modes_lock:
|
||||
running = set(self.plugin_display_modes)
|
||||
for key, value in new_config.items():
|
||||
if (key in known and isinstance(value, dict)
|
||||
and value.get('enabled', False) and key not in running):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _reconcile_enabled_plugins(self) -> bool:
|
||||
"""Load/unload plugins so the running set matches the enabled set in
|
||||
config. Runs on the main display thread (never the config-watcher
|
||||
|
||||
+84
-1
@@ -130,7 +130,12 @@ def setup_logging(
|
||||
# Console handler (always add)
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setLevel(level)
|
||||
console_handler.setFormatter(formatter)
|
||||
# Under systemd, tag each line so the journal records the real severity
|
||||
# rather than filing everything as informational. The file handler below
|
||||
# keeps the plain formatter: the prefix is meaningful to journald and noise
|
||||
# anywhere else.
|
||||
console_handler.setFormatter(
|
||||
JournalPriorityFormatter(formatter) if _under_systemd() else formatter)
|
||||
root_logger.addHandler(console_handler)
|
||||
|
||||
# File handler (if specified)
|
||||
@@ -145,6 +150,84 @@ def setup_logging(
|
||||
sys.stderr.write(f"Warning: Could not set up file logging to {log_file}: {e}\n")
|
||||
|
||||
|
||||
#: syslog priorities, which is what systemd parses from a "<N>" prefix on
|
||||
#: stdout. Mapped from Python's levels.
|
||||
_SYSLOG_PRIORITY = {
|
||||
logging.CRITICAL: 2, # LOG_CRIT
|
||||
logging.ERROR: 3, # LOG_ERR
|
||||
logging.WARNING: 4, # LOG_WARNING
|
||||
logging.INFO: 6, # LOG_INFO
|
||||
logging.DEBUG: 7, # LOG_DEBUG
|
||||
}
|
||||
|
||||
|
||||
class JournalPriorityFormatter(logging.Formatter):
|
||||
"""Wraps a formatter, prefixing each line with its syslog priority.
|
||||
|
||||
Under systemd everything this process writes to stdout lands in the journal
|
||||
as PRIORITY=6, whatever the Python level was. Measured on a live rig: 55
|
||||
ERROR lines and 13 WARNING lines in a day, every one of them recorded as
|
||||
informational, so `journalctl -p err -u ledmatrix` returned nothing at all
|
||||
while errors were being logged. Anyone triaging has to grep the message
|
||||
text instead, which is both slower and wrong -- a search for "oom" matches
|
||||
the radar logging "zoom=9".
|
||||
|
||||
systemd reads a leading "<N>" on each line and uses it as the priority
|
||||
(sd-daemon(3)), so this needs no extra dependency. Multi-line records get
|
||||
the prefix on every line, since the journal splits them and an unprefixed
|
||||
continuation would fall back to the default.
|
||||
"""
|
||||
|
||||
def __init__(self, inner: logging.Formatter):
|
||||
super().__init__()
|
||||
self._inner = inner
|
||||
|
||||
@property
|
||||
def inner(self) -> logging.Formatter:
|
||||
"""The formatter doing the actual work.
|
||||
|
||||
Whether journald tagging is applied depends on JOURNAL_STREAM, so it is
|
||||
on under systemd and off in a terminal -- and anything asserting which
|
||||
formatter setup_logging() selected would otherwise get a different
|
||||
answer in CI than on a developer's machine. Exposing the inner one lets
|
||||
those checks stay about format_type, which is what they mean.
|
||||
"""
|
||||
return self._inner
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
text = self._inner.format(record)
|
||||
prefix = f"<{_SYSLOG_PRIORITY.get(record.levelno, 6)}>"
|
||||
return "\n".join(prefix + line for line in text.split("\n"))
|
||||
|
||||
|
||||
def _under_systemd() -> bool:
|
||||
"""True when stdout really is the journal.
|
||||
|
||||
systemd sets JOURNAL_STREAM to "dev:ino" for services whose output it
|
||||
captures. Presence alone is not enough to act on: the variable is
|
||||
inherited by child processes and survives redirection, so a subprocess
|
||||
whose stdout is a pipe or a file still sees it and would emit the "<N>"
|
||||
priority prefixes as literal noise into that output. systemd's own
|
||||
guidance is to fstat the descriptor and compare st_dev/st_ino, which is
|
||||
what distinguishes "the journal is somewhere in my ancestry" from "my
|
||||
stdout is the journal".
|
||||
"""
|
||||
declared = os.environ.get("JOURNAL_STREAM")
|
||||
if not declared:
|
||||
return False
|
||||
try:
|
||||
dev_text, ino_text = declared.split(":", 1)
|
||||
declared_ids = (int(dev_text), int(ino_text))
|
||||
except (ValueError, AttributeError):
|
||||
return False
|
||||
try:
|
||||
stat_result = os.fstat(sys.stdout.fileno())
|
||||
except (OSError, ValueError, AttributeError):
|
||||
# No usable stdout: captured by pytest, detached, or already closed.
|
||||
return False
|
||||
return (stat_result.st_dev, stat_result.st_ino) == declared_ids
|
||||
|
||||
|
||||
class PluginLoggerAdapter(logging.LoggerAdapter):
|
||||
"""LoggerAdapter that stamps every record with its plugin_id.
|
||||
|
||||
|
||||
+116
-8
@@ -14,6 +14,7 @@ import json
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
from src.common.permission_utils import (
|
||||
@@ -25,6 +26,96 @@ from src.common.permission_utils import (
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: PNG text key stamped into a generated placeholder so a later run can tell it
|
||||
#: apart from a real logo that happens to be small.
|
||||
PLACEHOLDER_MARKER = "ledmatrix_placeholder"
|
||||
|
||||
#: Geometry of a generated placeholder, used to recognise ones written before
|
||||
#: the marker existed. Those are already on users' disks and would otherwise
|
||||
#: never be retried.
|
||||
PLACEHOLDER_SIZE = (64, 64)
|
||||
PLACEHOLDER_BG = (100, 100, 100, 255)
|
||||
|
||||
#: How long a placeholder is trusted before the real logo is attempted again.
|
||||
#: A placeholder means the download failed, and download failures are usually
|
||||
#: transient (no network at boot, ESPN blipping). Retrying every frame would
|
||||
#: hammer the API from a Pi that is also driving a panel; never retrying leaves
|
||||
#: the team a grey box forever, which is the bug this exists to avoid.
|
||||
PLACEHOLDER_RETRY_SECONDS = 6 * 60 * 60
|
||||
|
||||
|
||||
def is_placeholder_logo(filepath: Path) -> bool:
|
||||
"""True if the file at ``filepath`` is a generated placeholder, not a logo.
|
||||
|
||||
Checks the marker first, then falls back to matching the placeholder's
|
||||
exact geometry and background colour so files written before the marker was
|
||||
introduced are still recognised.
|
||||
"""
|
||||
try:
|
||||
with Image.open(filepath) as img:
|
||||
if img.info.get(PLACEHOLDER_MARKER):
|
||||
return True
|
||||
if img.size != PLACEHOLDER_SIZE:
|
||||
return False
|
||||
return img.convert("RGBA").getpixel((0, 0)) == PLACEHOLDER_BG
|
||||
except Exception:
|
||||
# Unreadable file: not provably a placeholder, and the caller's own
|
||||
# error handling is better placed to deal with it.
|
||||
return False
|
||||
|
||||
|
||||
def should_attempt_download(filepath: Path, force_download: bool = False) -> bool:
|
||||
"""Whether a real logo is worth (re)fetching for ``filepath``.
|
||||
|
||||
True when nothing is there, when the caller forced it, or when what is
|
||||
there is a placeholder old enough to retry. A *fresh* placeholder says a
|
||||
download just failed, so retrying it immediately would hammer the API for
|
||||
a result that is very unlikely to have changed.
|
||||
"""
|
||||
if force_download or not filepath.exists():
|
||||
return True
|
||||
if not is_placeholder_logo(filepath):
|
||||
return False
|
||||
age = placeholder_age_seconds(filepath)
|
||||
return age is None or age >= PLACEHOLDER_RETRY_SECONDS
|
||||
|
||||
|
||||
def refresh_placeholder_timestamp(filepath: Path) -> bool:
|
||||
"""Restamp a placeholder so a failed retry restarts the back-off clock.
|
||||
|
||||
Without this a stale placeholder stays stale: every later call sees an
|
||||
expired timestamp, retries, fails, and leaves the timestamp untouched --
|
||||
which is a download attempt per call, the opposite of what the back-off is
|
||||
for.
|
||||
"""
|
||||
try:
|
||||
if not is_placeholder_logo(filepath):
|
||||
return False
|
||||
metadata = PngInfo()
|
||||
metadata.add_text(PLACEHOLDER_MARKER, str(time.time()))
|
||||
with Image.open(filepath) as img:
|
||||
img.copy().save(filepath, "PNG", pnginfo=metadata)
|
||||
return True
|
||||
except Exception:
|
||||
logger.debug("Could not refresh placeholder timestamp for %s", filepath,
|
||||
exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def placeholder_age_seconds(filepath: Path) -> Optional[float]:
|
||||
"""Seconds since a placeholder was written, or None if unknown."""
|
||||
try:
|
||||
with Image.open(filepath) as img:
|
||||
stamped = img.info.get(PLACEHOLDER_MARKER)
|
||||
if stamped and stamped != "1":
|
||||
return max(0.0, time.time() - float(stamped))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
return max(0.0, time.time() - filepath.stat().st_mtime)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
class LogoDownloader:
|
||||
"""Centralized logo downloader for team logos from ESPN API."""
|
||||
|
||||
@@ -499,8 +590,10 @@ class LogoDownloader:
|
||||
filename = f"{self.normalize_abbreviation(abbreviation)}.png"
|
||||
filepath = Path(logo_dir) / filename
|
||||
|
||||
# Skip if already exists and not forcing download
|
||||
if filepath.exists() and not force_download:
|
||||
# A placeholder does not count as existing -- it is a previous
|
||||
# failure, and a bulk pass is exactly where it should get another
|
||||
# chance, subject to the same back-off as everywhere else.
|
||||
if not should_attempt_download(filepath, force_download):
|
||||
logger.debug(f"Skipping {display_name}: {filename} already exists")
|
||||
continue
|
||||
|
||||
@@ -559,8 +652,9 @@ class LogoDownloader:
|
||||
filename = f"{self.normalize_abbreviation(abbreviation)}.png"
|
||||
filepath = Path(logo_dir) / filename
|
||||
|
||||
# Skip if already exists and not forcing download
|
||||
if filepath.exists() and not force_download:
|
||||
# Same eligibility rule as every other download site: a stale
|
||||
# placeholder is a failed download, not a logo.
|
||||
if not should_attempt_download(filepath, force_download):
|
||||
logger.debug(f"Skipping {display_name} ({category}, {conference}): {filename} already exists")
|
||||
continue
|
||||
|
||||
@@ -674,11 +768,16 @@ class LogoDownloader:
|
||||
# Fallback without font
|
||||
draw.text((16, 24), text, fill=(255, 255, 255, 255))
|
||||
|
||||
logo.save(filepath)
|
||||
|
||||
# Stamp it so a later run can tell this apart from a real logo and
|
||||
# retry the download, instead of treating the file's existence as
|
||||
# proof the logo was fetched.
|
||||
metadata = PngInfo()
|
||||
metadata.add_text(PLACEHOLDER_MARKER, str(time.time()))
|
||||
logo.save(filepath, "PNG", pnginfo=metadata)
|
||||
|
||||
# Set proper file permissions after saving
|
||||
ensure_file_permissions(filepath, get_assets_file_mode())
|
||||
|
||||
|
||||
logger.info(f"Created placeholder logo for {team_abbreviation} at {filepath}")
|
||||
return True
|
||||
|
||||
@@ -770,9 +869,18 @@ def download_missing_logo(league: str, team_id: str, team_abbreviation: str, log
|
||||
# Use the exact filepath that was passed in (respects config settings)
|
||||
filepath = logo_path
|
||||
|
||||
if filepath.exists():
|
||||
if filepath.exists() and not should_attempt_download(filepath):
|
||||
# Either a real logo, or a placeholder too fresh to be worth retrying.
|
||||
logger.debug(f"Logo already exists for {team_abbreviation} ({league})")
|
||||
return True
|
||||
if filepath.exists():
|
||||
# A placeholder is a *failed* download wearing the real logo's
|
||||
# filename. Treating it as "already exists" is what pinned a team to a
|
||||
# grey box permanently after one transient failure.
|
||||
logger.info(
|
||||
"Logo for %s (%s) is a placeholder from a failed download; "
|
||||
"retrying the real logo", team_abbreviation, league,
|
||||
)
|
||||
|
||||
# Try to download the real logo first
|
||||
logger.info(f"Attempting to download logo for {team_abbreviation} from {league}")
|
||||
|
||||
@@ -28,6 +28,7 @@ fixes their version string. See `docs/SPORTS_UNIFICATION.md`, phase B4.
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
# Below this, the core's self-reported version is not evidence of anything.
|
||||
@@ -35,6 +36,45 @@ from typing import Any, Dict, Optional, Tuple
|
||||
TRUSTWORTHY_FLOOR: Tuple[int, int, int] = (2, 0, 0)
|
||||
|
||||
|
||||
# ``src/__init__.py`` relative to this file: src/plugin_system/ -> src/
|
||||
_VERSION_FILE = Path(__file__).resolve().parent.parent / "__init__.py"
|
||||
_VERSION_RE = re.compile(r'^__version__\s*=\s*["\']([^"\']+)["\']', re.M)
|
||||
|
||||
|
||||
def current_core_version() -> str:
|
||||
"""The core version as it is on disk right now, not as it was at import.
|
||||
|
||||
``from src import __version__`` binds whatever the process loaded at start.
|
||||
The web UI runs as its own long-lived service (``ledmatrix-web.service``),
|
||||
and updating the core replaces files on disk without restarting it -- the
|
||||
update route says so explicitly and asks the user to restart. Its prompt
|
||||
names the *display* service, so a user who follows it leaves the web
|
||||
process holding the old number.
|
||||
|
||||
The plugin store's gate lives in that web process. Stale by one release is
|
||||
exactly the case that matters: every plugin flooring on the release you
|
||||
just installed gets refused, with a message blaming a core version that is
|
||||
already correct on disk. 3.3.0 is the first release where that hits a whole
|
||||
plugin family at once -- all eight sports scoreboards floor there.
|
||||
|
||||
Reading the file costs one stat and a small read per call, and only on the
|
||||
install/update path. Any failure falls back to the imported value, so this
|
||||
can only ever be as wrong as before, never worse.
|
||||
"""
|
||||
try:
|
||||
text = _VERSION_FILE.read_text(encoding="utf-8")
|
||||
match = _VERSION_RE.search(text)
|
||||
if match:
|
||||
return match.group(1)
|
||||
except (OSError, UnicodeDecodeError):
|
||||
pass
|
||||
try:
|
||||
from src import __version__ as imported
|
||||
return imported
|
||||
except Exception: # noqa: BLE001 - never let this raise
|
||||
return "0.0.0"
|
||||
|
||||
|
||||
def parse_semver(value: Any) -> Optional[Tuple[int, int, int]]:
|
||||
"""Parse ``X.Y.Z`` (extra parts and suffixes ignored) into a comparable
|
||||
3-tuple, or ``None`` when unparseable. A leading ``v`` is tolerated."""
|
||||
|
||||
@@ -7,7 +7,7 @@ and circuit breaker state. Provides automatic recovery mechanisms.
|
||||
|
||||
import time
|
||||
import logging
|
||||
from typing import Dict, Optional, Any
|
||||
from typing import Dict, Optional, Any, Tuple
|
||||
from enum import Enum
|
||||
|
||||
|
||||
@@ -64,10 +64,48 @@ class PluginHealthTracker:
|
||||
cache_key, max_age=None, memory_ttl=0 if force_reload else None
|
||||
)
|
||||
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
# Default state
|
||||
if isinstance(cached, dict) and cached:
|
||||
# Complete it rather than trusting it: a persisted record can be
|
||||
# missing fields the callers index directly (a partial write, a
|
||||
# restored backup, an older schema), and returning it verbatim makes
|
||||
# record_success / record_failure raise KeyError, which takes the
|
||||
# display down in a restart loop that survives reboots because the
|
||||
# bad entry is on disk.
|
||||
state, repaired = self._repair_health_state(cached)
|
||||
if repaired:
|
||||
self.logger.warning(
|
||||
f"Repaired health state for {plugin_id}: "
|
||||
f"{sorted(repaired)} missing or invalid, using defaults for those."
|
||||
)
|
||||
return state
|
||||
|
||||
# Not a dict at all: written by something other than
|
||||
# _save_health_state (a key collision, a corrupted entry). Nothing to
|
||||
# salvage.
|
||||
if cached is not None and not isinstance(cached, dict):
|
||||
self.logger.warning(
|
||||
f"Discarding malformed health state for {plugin_id}: expected "
|
||||
f"dict, got {type(cached).__name__}. Falling back to defaults."
|
||||
)
|
||||
|
||||
return self._default_health_state()
|
||||
|
||||
def _save_health_state(self, plugin_id: str, state: Dict[str, Any]) -> None:
|
||||
"""Save health state to cache."""
|
||||
cache_key = self._get_health_key(plugin_id)
|
||||
self.cache_manager.set(cache_key, state) # Persist indefinitely
|
||||
self._health_state[plugin_id] = state
|
||||
|
||||
# The fields callers index directly (state['circuit_state'] and friends).
|
||||
# A cached dict missing any of them raises KeyError deep in record_success /
|
||||
# record_failure, so the value is completed before it is handed out.
|
||||
_COUNTER_FIELDS = ('consecutive_failures', 'total_failures', 'total_successes')
|
||||
_TIMESTAMP_FIELDS = ('last_success_time', 'last_failure_time',
|
||||
'circuit_opened_time', 'half_open_start_time')
|
||||
|
||||
@staticmethod
|
||||
def _default_health_state() -> Dict[str, Any]:
|
||||
"""A fresh state with every field the callers expect."""
|
||||
return {
|
||||
'consecutive_failures': 0,
|
||||
'total_failures': 0,
|
||||
@@ -77,15 +115,56 @@ class PluginHealthTracker:
|
||||
'circuit_state': CircuitState.CLOSED.value,
|
||||
'circuit_opened_time': None,
|
||||
'half_open_start_time': None,
|
||||
'last_error': None
|
||||
'last_error': None,
|
||||
}
|
||||
|
||||
def _save_health_state(self, plugin_id: str, state: Dict[str, Any]) -> None:
|
||||
"""Save health state to cache."""
|
||||
cache_key = self._get_health_key(plugin_id)
|
||||
self.cache_manager.set(cache_key, state) # Persist indefinitely
|
||||
self._health_state[plugin_id] = state
|
||||
|
||||
|
||||
@classmethod
|
||||
def _repair_health_state(cls, cached: Dict[str, Any]) -> Tuple[Dict[str, Any], list]:
|
||||
"""Return `cached` completed against the defaults, plus what was repaired.
|
||||
|
||||
Per-field rather than all-or-nothing: a record that has real failure
|
||||
counts but is missing `last_error` should keep the counts, not be reset
|
||||
to healthy. Only values that are absent or the wrong type fall back to
|
||||
the default, so a partial or older-schema record survives with whatever
|
||||
it does carry, while every field the callers index is guaranteed present
|
||||
and of a usable type.
|
||||
"""
|
||||
state = cls._default_health_state()
|
||||
repaired = []
|
||||
for field, default in state.items():
|
||||
if field not in cached:
|
||||
repaired.append(field)
|
||||
continue
|
||||
value = cached[field]
|
||||
if field in cls._COUNTER_FIELDS:
|
||||
ok = isinstance(value, int) and not isinstance(value, bool) and value >= 0
|
||||
elif field in cls._TIMESTAMP_FIELDS:
|
||||
# bool is a subclass of int, so True would pass as a timestamp
|
||||
# and then compare as 1.0 -- expiring a cooldown the instant it
|
||||
# opens, or (False) making the elapsed check never fire.
|
||||
ok = value is None or (
|
||||
isinstance(value, (int, float)) and not isinstance(value, bool)
|
||||
)
|
||||
elif field == 'circuit_state':
|
||||
# Membership first requires the value to be hashable: a list or
|
||||
# dict here would raise TypeError out of the repair itself,
|
||||
# which is the crash this whole path exists to prevent.
|
||||
ok = isinstance(value, str) and value in {
|
||||
member.value for member in CircuitState
|
||||
}
|
||||
else: # last_error
|
||||
ok = value is None or isinstance(value, str)
|
||||
if ok:
|
||||
state[field] = value
|
||||
else:
|
||||
repaired.append(field)
|
||||
# Anything the schema has since grown (degraded, degraded_reason) is
|
||||
# read with .get() by its callers, so carry it through untouched.
|
||||
for field, value in cached.items():
|
||||
if field not in state:
|
||||
state[field] = value
|
||||
return state, repaired
|
||||
|
||||
def get_health_state(self, plugin_id: str, force_reload: bool = False) -> Dict[str, Any]:
|
||||
"""Get current health state for a plugin.
|
||||
|
||||
@@ -99,11 +178,21 @@ class PluginHealthTracker:
|
||||
)
|
||||
return self._health_state[plugin_id]
|
||||
|
||||
# Fields the circuit breaker is rebuilt from after a restart. Everything
|
||||
# else in a health record is reporting, read only for display.
|
||||
_DURABLE_FIELDS = ('consecutive_failures', 'circuit_state',
|
||||
'circuit_opened_time', 'half_open_start_time')
|
||||
|
||||
def _durable(self, state: Dict[str, Any]) -> tuple:
|
||||
"""The part of a health record whose loss would change behaviour."""
|
||||
return tuple(state.get(field) for field in self._DURABLE_FIELDS)
|
||||
|
||||
def record_success(self, plugin_id: str) -> None:
|
||||
"""Record a successful plugin execution."""
|
||||
state = self.get_health_state(plugin_id)
|
||||
current_time = time.time()
|
||||
|
||||
durable_before = self._durable(state)
|
||||
|
||||
# Reset consecutive failures
|
||||
state['consecutive_failures'] = 0
|
||||
state['total_successes'] = state.get('total_successes', 0) + 1
|
||||
@@ -119,9 +208,20 @@ class PluginHealthTracker:
|
||||
# Shouldn't happen, but handle it
|
||||
state['circuit_state'] = CircuitState.CLOSED.value
|
||||
state['circuit_opened_time'] = None
|
||||
|
||||
self._save_health_state(plugin_id, state)
|
||||
|
||||
|
||||
# A healthy plugin reports success every cycle, and in that steady state
|
||||
# the only fields changed above are a counter and a timestamp that
|
||||
# nothing reads back after a restart. Persisting them anyway rewrites a
|
||||
# small file per plugin per cycle: on a rig running 24 plugins, a
|
||||
# five-minute sample measured 22 rewrites, about 4.4 a minute or 6,300 a
|
||||
# day. Those land on an SD card, where the cost is an erase-block cycle
|
||||
# rather than the 400 bytes involved, and where wear is what eventually
|
||||
# kills the card.
|
||||
# In-memory state is still updated every time, so the health API and web
|
||||
# UI show exactly what they did before; only the write is skipped.
|
||||
if self._durable(state) != durable_before:
|
||||
self._save_health_state(plugin_id, state)
|
||||
|
||||
def record_failure(self, plugin_id: str, error: Optional[Exception] = None) -> None:
|
||||
"""Record a failed plugin execution."""
|
||||
state = self.get_health_state(plugin_id)
|
||||
|
||||
@@ -14,7 +14,7 @@ import sys
|
||||
import subprocess
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, Tuple, Type
|
||||
from typing import Dict, Any, List, Optional, Tuple, Type
|
||||
import logging
|
||||
|
||||
from packaging.requirements import InvalidRequirement, Requirement
|
||||
@@ -45,6 +45,76 @@ def requirements_has_real_deps(requirements_file: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _extra_dependencies(dist_name: str, extras) -> Optional[List[Requirement]]:
|
||||
"""Dependencies a distribution declares *only* behind the given extras.
|
||||
|
||||
Returns None when the installed metadata cannot be read or parsed, so the
|
||||
caller can fall back to running pip rather than assuming anything.
|
||||
"""
|
||||
try:
|
||||
meta = importlib.metadata.metadata(dist_name)
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
return None
|
||||
|
||||
gated: List[Requirement] = []
|
||||
for raw in meta.get_all('Requires-Dist') or []:
|
||||
try:
|
||||
dep = Requirement(raw)
|
||||
except InvalidRequirement:
|
||||
return None
|
||||
if dep.marker is None:
|
||||
continue
|
||||
# Keep only what the distribution gates behind an extra we asked for:
|
||||
# satisfied when `extra` is that name, but not when no extra is
|
||||
# requested. A marker that holds either way (python_version, sys_platform)
|
||||
# belongs to the base install and is already covered by the version check.
|
||||
if dep.marker.evaluate({'extra': ''}):
|
||||
continue
|
||||
if any(dep.marker.evaluate({'extra': extra}) for extra in extras):
|
||||
gated.append(dep)
|
||||
return gated
|
||||
|
||||
|
||||
def _extras_are_satisfied(req: Requirement, _visited: Optional[set] = None) -> bool:
|
||||
"""Check the dependencies pulled in by req's extras are installed.
|
||||
|
||||
Follows extras through nested extras. A gated dependency can itself request
|
||||
one (`requests[socks]`), and checking only that `requests` is installed at
|
||||
an acceptable version says nothing about whether the socks extra's own
|
||||
dependency is there -- so the caller would skip pip and the plugin would
|
||||
fail at import instead. Plain dependencies are still checked one level
|
||||
deep, which is all that is needed to tell "the extra was installed" from
|
||||
"the extra was never installed".
|
||||
|
||||
`_visited` carries the (distribution, extras) pairs already seen, so a
|
||||
dependency cycle between extras terminates instead of recursing forever.
|
||||
Anything unreadable returns False, so the caller still falls through to pip.
|
||||
"""
|
||||
if _visited is None:
|
||||
_visited = set()
|
||||
marker = (req.name.lower(), frozenset(e.lower() for e in req.extras))
|
||||
if marker in _visited:
|
||||
# Already accounted for higher up the chain; treating a cycle as
|
||||
# satisfied here is safe because the outer frame still has to pass.
|
||||
return True
|
||||
_visited.add(marker)
|
||||
|
||||
gated = _extra_dependencies(req.name, req.extras)
|
||||
if gated is None:
|
||||
return False
|
||||
|
||||
for dep in gated:
|
||||
try:
|
||||
dep_version = importlib.metadata.version(dep.name)
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
return False
|
||||
if dep.specifier and not dep.specifier.contains(dep_version, prereleases=True):
|
||||
return False
|
||||
if dep.extras and not _extras_are_satisfied(dep, _visited):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def requirements_are_satisfied(requirements_file: str) -> bool:
|
||||
"""
|
||||
Check whether every real requirement line in requirements.txt is already
|
||||
@@ -76,9 +146,6 @@ def requirements_are_satisfied(requirements_file: str) -> bool:
|
||||
except InvalidRequirement:
|
||||
return False
|
||||
|
||||
if req.extras:
|
||||
return False # verifying extras' sub-dependencies isn't worth it here
|
||||
|
||||
if req.marker is not None and not req.marker.evaluate():
|
||||
continue # not applicable on this platform/interpreter
|
||||
|
||||
@@ -90,6 +157,9 @@ def requirements_are_satisfied(requirements_file: str) -> bool:
|
||||
if req.specifier and not req.specifier.contains(installed_version, prereleases=True):
|
||||
return False
|
||||
|
||||
if req.extras and not _extras_are_satisfied(req):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@@ -702,8 +772,8 @@ class PluginLoader:
|
||||
newer than the running core. Advisory only — never raises — so a
|
||||
plugin that guards optional features with try/except keeps working.
|
||||
"""
|
||||
from src import __version__ as core_version
|
||||
from src.plugin_system import compatibility
|
||||
core_version = compatibility.current_core_version()
|
||||
|
||||
compatible, _reason = compatibility.check(manifest, core_version)
|
||||
if compatible:
|
||||
|
||||
@@ -71,11 +71,15 @@ class PluginManager:
|
||||
self.plugin_loader = PluginLoader(logger=self.logger)
|
||||
self.plugin_executor = PluginExecutor(default_timeout=30.0, logger=self.logger)
|
||||
self.state_manager = PluginStateManager(logger=self.logger)
|
||||
self.schema_manager = SchemaManager(plugins_dir=self.plugins_dir, logger=self.logger)
|
||||
self.schema_manager = SchemaManager(plugins_dir=self.plugins_dir, logger=self.logger,
|
||||
config_manager=self.config_manager)
|
||||
|
||||
# Lock protecting plugin_manifests and plugin_directories from
|
||||
# concurrent mutation (background reconciliation) and reads (requests).
|
||||
self._discovery_lock = threading.RLock()
|
||||
#: Directories already reported as unloadable, so the warning is
|
||||
#: emitted once rather than on every discovery scan.
|
||||
self._skip_reported: set = set()
|
||||
|
||||
# Lock protecting plugin_last_update from concurrent mutation/iteration.
|
||||
# It's written from run_scheduled_updates()/update_all_plugins() (main
|
||||
@@ -125,6 +129,14 @@ class PluginManager:
|
||||
self._plugin_locks: Dict[str, threading.Lock] = {}
|
||||
self._plugin_locks_guard = threading.Lock()
|
||||
self._update_worker: Optional[threading.Thread] = None
|
||||
# Plugin ids whose update() has finished since the last time anyone
|
||||
# asked. Updates are dispatched to a worker thread, so a caller that
|
||||
# wants to know "whose data just changed" cannot learn it by diffing
|
||||
# plugin_last_update around run_scheduled_updates() -- that call only
|
||||
# enqueues, and the timestamp is stamped later, on the worker. See
|
||||
# run_scheduled_updates_with_changes().
|
||||
self._completed_updates: set = set()
|
||||
self._completed_updates_lock = threading.Lock()
|
||||
self._synchronous_updates = False
|
||||
if self.config_manager is not None:
|
||||
try:
|
||||
@@ -187,18 +199,59 @@ class PluginManager:
|
||||
continue
|
||||
|
||||
manifest_path = item / "manifest.json"
|
||||
if manifest_path.exists():
|
||||
try:
|
||||
with open(manifest_path, 'r', encoding='utf-8') as f:
|
||||
manifest = json.load(f)
|
||||
plugin_id = manifest.get('id')
|
||||
if plugin_id:
|
||||
plugin_ids.append(plugin_id)
|
||||
new_manifests[plugin_id] = manifest
|
||||
new_directories[plugin_id] = item
|
||||
except (json.JSONDecodeError, PermissionError, OSError) as e:
|
||||
self.logger.warning("Error reading manifest from %s: %s", manifest_path, e, exc_info=True)
|
||||
continue
|
||||
if not manifest_path.exists():
|
||||
# Once per directory per process. Discovery runs on every
|
||||
# web UI page load and every config reconcile, so warning
|
||||
# unconditionally would put a line in the journal each
|
||||
# time someone opened a page -- the same log-volume
|
||||
# problem this is meant to help diagnose.
|
||||
# A directory here that carries no manifest is not a
|
||||
# plugin. Said once, because the alternative is a plugin
|
||||
# that is enabled in config, enabled in plugin state,
|
||||
# present on disk, and simply absent from the running
|
||||
# process with nothing anywhere to say why. Working that
|
||||
# out afterwards means reading cache-file mtimes.
|
||||
if item.name not in self._skip_reported:
|
||||
self._skip_reported.add(item.name)
|
||||
self.logger.warning(
|
||||
"Skipping %s: no manifest.json, so it cannot be "
|
||||
"loaded as a plugin", item.name)
|
||||
continue
|
||||
try:
|
||||
with open(manifest_path, 'r', encoding='utf-8') as f:
|
||||
manifest = json.load(f)
|
||||
except (json.JSONDecodeError, PermissionError, OSError) as e:
|
||||
self.logger.warning("Error reading manifest from %s: %s", manifest_path, e, exc_info=True)
|
||||
continue
|
||||
|
||||
# json.load accepts any JSON value, so a manifest holding
|
||||
# null, [] or "text" parses and then raises AttributeError on
|
||||
# .get(). Nothing here catches that -- the outer handler takes
|
||||
# OSError/PermissionError only -- so a single malformed
|
||||
# manifest aborted the whole scan and every other plugin on
|
||||
# disk, however healthy, silently failed to register.
|
||||
if not isinstance(manifest, dict):
|
||||
if item.name not in self._skip_reported:
|
||||
self._skip_reported.add(item.name)
|
||||
self.logger.warning(
|
||||
"Skipping %s: its manifest.json is %s, not a JSON "
|
||||
"object", item.name, type(manifest).__name__)
|
||||
continue
|
||||
|
||||
plugin_id = manifest.get('id')
|
||||
if not plugin_id:
|
||||
# Parsed but unusable. This was the quietest path of all:
|
||||
# the manifest is read successfully and then dropped.
|
||||
if item.name not in self._skip_reported:
|
||||
self._skip_reported.add(item.name)
|
||||
self.logger.warning(
|
||||
"Skipping %s: its manifest.json has no \"id\", so "
|
||||
"there is nothing to register it under", item.name)
|
||||
continue
|
||||
|
||||
plugin_ids.append(plugin_id)
|
||||
new_manifests[plugin_id] = manifest
|
||||
new_directories[plugin_id] = item
|
||||
except (OSError, PermissionError) as e:
|
||||
self.logger.error("Error scanning directory %s: %s", directory, e, exc_info=True)
|
||||
|
||||
@@ -578,6 +631,17 @@ class PluginManager:
|
||||
|
||||
return self.load_plugin(plugin_id)
|
||||
|
||||
def discovered_plugin_ids(self) -> set:
|
||||
"""Snapshot of the discovered plugin ids, taken under the discovery lock.
|
||||
|
||||
Callers on other threads (the config watcher) must not iterate
|
||||
``plugin_manifests`` directly: discovery rebuilds it entry by entry, so
|
||||
an unsynchronised reader can see a half-populated mapping or raise
|
||||
"dictionary changed size during iteration".
|
||||
"""
|
||||
with self._discovery_lock:
|
||||
return set(self.plugin_manifests)
|
||||
|
||||
def get_plugin(self, plugin_id: str) -> Optional[Any]:
|
||||
"""
|
||||
Get a loaded plugin instance by ID.
|
||||
@@ -1025,6 +1089,7 @@ class PluginManager:
|
||||
if success:
|
||||
with self._plugin_last_update_lock:
|
||||
self.plugin_last_update[plugin_id] = scheduled_time
|
||||
self._note_update_completed(plugin_id)
|
||||
self.state_manager.record_update(plugin_id)
|
||||
self.state_manager.set_state(plugin_id, PluginState.ENABLED)
|
||||
if self.health_tracker:
|
||||
@@ -1089,28 +1154,41 @@ class PluginManager:
|
||||
|
||||
def run_scheduled_updates_with_changes(self, current_time: Optional[float] = None) -> List[str]:
|
||||
"""
|
||||
Like run_scheduled_updates(), but also returns the plugin_ids whose
|
||||
plugin_last_update timestamp actually advanced during this call.
|
||||
Like run_scheduled_updates(), but also reports which plugins have
|
||||
fresh data -- the ids whose update() has finished since the last
|
||||
call, not necessarily the ones enqueued by this one.
|
||||
|
||||
The before/after snapshots and the update pass itself are each
|
||||
individually lock-protected against concurrent plugin_last_update
|
||||
mutation (Vegas mode calls this from its own background
|
||||
update-tick thread, racing the main render loop's plugin updates),
|
||||
so callers get an atomic "who got fresh data" answer without
|
||||
reaching into plugin_last_update themselves. The lock is not held
|
||||
across the update pass so slow/blocking plugin update() calls don't
|
||||
serialize against other plugin_last_update readers.
|
||||
That distinction is the whole point. This used to snapshot
|
||||
plugin_last_update, call run_scheduled_updates(), and diff. But
|
||||
run_scheduled_updates() only *enqueues*: the work runs on the
|
||||
update worker and the timestamp is stamped there, after this method
|
||||
has already returned. The two snapshots were therefore always
|
||||
identical and the result was always empty, so Vegas never learned
|
||||
that any plugin's data had changed and kept scrolling whatever a
|
||||
segment was first built from -- last night's live game still drawn
|
||||
as live the next morning. The only path that ever worked was the
|
||||
synchronous kill-switch, where update() runs inline.
|
||||
|
||||
Reporting completions instead of enqueues costs a poll's worth of
|
||||
latency (the Vegas tick runs every ~4s) and is correct regardless of
|
||||
which side of the queue the work lands on.
|
||||
"""
|
||||
with self._plugin_last_update_lock:
|
||||
old_times = dict(self.plugin_last_update)
|
||||
|
||||
self.run_scheduled_updates(current_time)
|
||||
return self.drain_completed_updates()
|
||||
|
||||
with self._plugin_last_update_lock:
|
||||
return [
|
||||
plugin_id for plugin_id, new_time in self.plugin_last_update.items()
|
||||
if new_time > old_times.get(plugin_id, 0.0)
|
||||
]
|
||||
def _note_update_completed(self, plugin_id: str) -> None:
|
||||
"""Record that a plugin's update() finished, for the next poll."""
|
||||
with self._completed_updates_lock:
|
||||
self._completed_updates.add(plugin_id)
|
||||
|
||||
def drain_completed_updates(self) -> List[str]:
|
||||
"""Return and clear the plugin ids whose update() has since finished."""
|
||||
with self._completed_updates_lock:
|
||||
if not self._completed_updates:
|
||||
return []
|
||||
done = sorted(self._completed_updates)
|
||||
self._completed_updates.clear()
|
||||
return done
|
||||
|
||||
def update_all_plugins(self) -> None:
|
||||
"""
|
||||
@@ -1135,6 +1213,7 @@ class PluginManager:
|
||||
if success:
|
||||
with self._plugin_last_update_lock:
|
||||
self.plugin_last_update[plugin_id] = time.time()
|
||||
self._note_update_completed(plugin_id)
|
||||
self.state_manager.record_update(plugin_id)
|
||||
self.state_manager.set_state(plugin_id, PluginState.ENABLED)
|
||||
else:
|
||||
|
||||
@@ -6,14 +6,40 @@ with state transitions and queries.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from enum import Enum
|
||||
from typing import Optional, Dict, Any
|
||||
from typing import Optional, Dict, Any, Deque, List, Tuple
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
from src.logging_config import get_logger
|
||||
|
||||
|
||||
# The history is diagnostic only -- nothing reads the entries themselves, just
|
||||
# their count -- but it is appended to on the hot scheduling path: every update
|
||||
# cycle records RUNNING on reserve and ENABLED on finish. Unbounded, that is
|
||||
# 2,880 entries per plugin per day at the default 60s interval, which on a 1 GB
|
||||
# Pi exhausts memory in weeks.
|
||||
#
|
||||
# Two limits, because a single entry count answers the wrong question. What a
|
||||
# reader wants is "the last couple of hours", and how many transitions that is
|
||||
# depends entirely on the plugin's update interval -- which on a real board
|
||||
# spans 2s to 3600s. A flat 200 entries is 4.2 days for the slowest plugin and
|
||||
# 3.3 minutes for the fastest, so the plugin churning hardest, the one worth
|
||||
# looking at, keeps the least history.
|
||||
#
|
||||
# So: trim by AGE first, which makes the retained window comparable across
|
||||
# plugins whatever their cadence...
|
||||
STATE_HISTORY_MAX_AGE_SECONDS = 2 * 60 * 60
|
||||
|
||||
# ...and cap by COUNT second, purely as a memory ceiling for the fast pollers
|
||||
# whose age window would otherwise run to thousands of entries. At ~230 bytes
|
||||
# an entry this is ~0.5 MB per plugin worst case, and only plugins updating
|
||||
# faster than roughly every 4s can reach it.
|
||||
MAX_STATE_HISTORY_PER_PLUGIN = 2000
|
||||
|
||||
|
||||
class PluginState(Enum):
|
||||
"""Plugin state enumeration."""
|
||||
UNLOADED = "unloaded" # Plugin not loaded
|
||||
@@ -37,11 +63,43 @@ class PluginStateManager:
|
||||
self.logger = logger or get_logger(__name__)
|
||||
self._lock = threading.RLock()
|
||||
self._states: Dict[str, PluginState] = {}
|
||||
self._state_history: Dict[str, list] = {}
|
||||
# (monotonic timestamp, transition). The clock is monotonic so a DST
|
||||
# shift or an NTP step cannot make entries look old and flush the
|
||||
# history; the human-readable timestamp lives inside the transition.
|
||||
self._state_history: Dict[str, Deque[Tuple[float, Dict[str, Any]]]] = {}
|
||||
# Lifetime transition totals, kept separately so the count reported by
|
||||
# get_state_info() stays truthful once the history above starts rolling.
|
||||
self._state_transition_counts: Dict[str, int] = {}
|
||||
self._error_info: Dict[str, Dict[str, Any]] = {}
|
||||
self._last_update: Dict[str, datetime] = {}
|
||||
self._last_display: Dict[str, datetime] = {}
|
||||
|
||||
def _record_transition(
|
||||
self,
|
||||
plugin_id: str,
|
||||
transition: Dict[str, Any]
|
||||
) -> None:
|
||||
"""Append a transition to the plugin's bounded history.
|
||||
|
||||
Callers must already hold ``_lock``. The deque discards its oldest
|
||||
entry once it is full, so the history cannot grow without bound; the
|
||||
lifetime total is tracked separately for get_state_info().
|
||||
"""
|
||||
history = self._state_history.get(plugin_id)
|
||||
if history is None:
|
||||
history = deque(maxlen=MAX_STATE_HISTORY_PER_PLUGIN)
|
||||
self._state_history[plugin_id] = history
|
||||
now = time.monotonic()
|
||||
history.append((now, transition))
|
||||
# Age out first; the deque's maxlen is the backstop for plugins that
|
||||
# produce more than the ceiling within the window.
|
||||
cutoff = now - STATE_HISTORY_MAX_AGE_SECONDS
|
||||
while history and history[0][0] < cutoff:
|
||||
history.popleft()
|
||||
self._state_transition_counts[plugin_id] = (
|
||||
self._state_transition_counts.get(plugin_id, 0) + 1
|
||||
)
|
||||
|
||||
def set_state(
|
||||
self,
|
||||
plugin_id: str,
|
||||
@@ -60,16 +118,13 @@ class PluginStateManager:
|
||||
old_state = self._states.get(plugin_id, PluginState.UNLOADED)
|
||||
self._states[plugin_id] = state
|
||||
|
||||
if plugin_id not in self._state_history:
|
||||
self._state_history[plugin_id] = []
|
||||
|
||||
transition = {
|
||||
'timestamp': datetime.now(),
|
||||
'from': old_state.value,
|
||||
'to': state.value,
|
||||
'error': str(error) if error else None
|
||||
}
|
||||
self._state_history[plugin_id].append(transition)
|
||||
self._record_transition(plugin_id, transition)
|
||||
|
||||
# Store error info if transitioning to ERROR state
|
||||
if state == PluginState.ERROR and error:
|
||||
@@ -126,17 +181,29 @@ class PluginStateManager:
|
||||
state = self.get_state(plugin_id)
|
||||
return state == PluginState.ENABLED
|
||||
|
||||
def get_state_history(self, plugin_id: str) -> list:
|
||||
def get_state_history(self, plugin_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Get state transition history for a plugin.
|
||||
|
||||
|
||||
Retention is by age first -- transitions older than
|
||||
STATE_HISTORY_MAX_AGE_SECONDS are dropped -- and by count second, at
|
||||
MAX_STATE_HISTORY_PER_PLUGIN, which only binds for plugins updating
|
||||
fast enough to exceed it inside that window.
|
||||
|
||||
Args:
|
||||
plugin_id: Plugin identifier
|
||||
|
||||
|
||||
Returns:
|
||||
List of state transitions
|
||||
List of recent state transitions, oldest first. Both the list and
|
||||
the transition dicts are copies, so callers cannot mutate the
|
||||
manager's own history. The values inside a transition are all
|
||||
immutable, so a shallow copy per entry is enough.
|
||||
"""
|
||||
return self._state_history.get(plugin_id, [])
|
||||
with self._lock:
|
||||
return [
|
||||
dict(transition)
|
||||
for _stamp, transition in self._state_history.get(plugin_id, ())
|
||||
]
|
||||
|
||||
def set_error_info(self, plugin_id: str, error_info: Dict[str, Any]) -> None:
|
||||
"""
|
||||
@@ -179,9 +246,7 @@ class PluginStateManager:
|
||||
old_state = self._states.get(plugin_id, PluginState.UNLOADED)
|
||||
self._states[plugin_id] = state
|
||||
|
||||
if plugin_id not in self._state_history:
|
||||
self._state_history[plugin_id] = []
|
||||
self._state_history[plugin_id].append({
|
||||
self._record_transition(plugin_id, {
|
||||
'timestamp': datetime.now(),
|
||||
'from': old_state.value,
|
||||
'to': state.value,
|
||||
@@ -241,26 +306,40 @@ class PluginStateManager:
|
||||
Returns:
|
||||
Dictionary with state information
|
||||
"""
|
||||
state = self.get_state(plugin_id)
|
||||
info = {
|
||||
'state': state.value,
|
||||
'is_loaded': self.is_loaded(plugin_id),
|
||||
'is_enabled': self.is_enabled(plugin_id),
|
||||
'is_running': self.is_running(plugin_id),
|
||||
'is_error': self.is_error(plugin_id),
|
||||
'can_execute': self.can_execute(plugin_id),
|
||||
'last_update': self.get_last_update(plugin_id),
|
||||
'last_display': self.get_last_display(plugin_id),
|
||||
'error_info': self.get_error_info(plugin_id),
|
||||
'state_history_count': len(self.get_state_history(plugin_id))
|
||||
}
|
||||
# One snapshot, one critical section. Each field was read under its own
|
||||
# lock, so an unload running concurrently could be observed half-done:
|
||||
# 'state' read before clear_state() removed it and
|
||||
# 'state_history_count' read after, giving a caller a plugin that is
|
||||
# ENABLED with zero transitions. _lock is an RLock, so the helpers
|
||||
# below can still take it.
|
||||
with self._lock:
|
||||
state = self.get_state(plugin_id)
|
||||
info = {
|
||||
'state': state.value,
|
||||
'is_loaded': self.is_loaded(plugin_id),
|
||||
'is_enabled': self.is_enabled(plugin_id),
|
||||
'is_running': self.is_running(plugin_id),
|
||||
'is_error': self.is_error(plugin_id),
|
||||
'can_execute': self.can_execute(plugin_id),
|
||||
'last_update': self.get_last_update(plugin_id),
|
||||
'last_display': self.get_last_display(plugin_id),
|
||||
'error_info': self.get_error_info(plugin_id),
|
||||
'state_history_count': self._state_transition_counts.get(plugin_id, 0)
|
||||
}
|
||||
return info
|
||||
|
||||
def clear_state(self, plugin_id: str) -> None:
|
||||
"""Clear all state information for a plugin."""
|
||||
self._states.pop(plugin_id, None)
|
||||
self._state_history.pop(plugin_id, None)
|
||||
self._error_info.pop(plugin_id, None)
|
||||
self._last_update.pop(plugin_id, None)
|
||||
self._last_display.pop(plugin_id, None)
|
||||
"""Clear all state information for a plugin.
|
||||
|
||||
Held under ``_lock`` so the five dicts are dropped as one unit: every
|
||||
other mutator takes the lock, and without it a concurrent set_state()
|
||||
could interleave and leave a plugin with history but no state.
|
||||
"""
|
||||
with self._lock:
|
||||
self._states.pop(plugin_id, None)
|
||||
self._state_history.pop(plugin_id, None)
|
||||
self._state_transition_counts.pop(plugin_id, None)
|
||||
self._error_info.pop(plugin_id, None)
|
||||
self._last_update.pop(plugin_id, None)
|
||||
self._last_display.pop(plugin_id, None)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import time
|
||||
import logging
|
||||
import threading
|
||||
from typing import Dict, Optional, Any, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, fields
|
||||
|
||||
try:
|
||||
import psutil
|
||||
@@ -49,6 +49,20 @@ class ResourceMetrics:
|
||||
self.total_execution_time = self.total_execution_time / self.call_count
|
||||
|
||||
|
||||
#: How often a plugin's metrics are written to the cache, in seconds.
|
||||
#:
|
||||
#: Persisting on every call meant a small file rewritten roughly nine times a
|
||||
#: minute per plugin. On a rig with fourteen active plugins that was ~126
|
||||
#: writes a minute for metrics alone, and since each ~350-byte file costs a
|
||||
#: 4KB block plus an ext4 journal entry, it dominated the device's write
|
||||
#: volume -- on an SD card, which wears out.
|
||||
#:
|
||||
#: The in-memory copy stays authoritative and exact; only the cross-process
|
||||
#: snapshot the web UI reads is delayed, and telemetry up to half a minute old
|
||||
#: is still a fair description of a long-running plugin.
|
||||
_METRICS_PERSIST_INTERVAL = 30.0
|
||||
|
||||
|
||||
class PluginResourceMonitor:
|
||||
"""
|
||||
Monitors resource usage for plugins.
|
||||
@@ -75,6 +89,10 @@ class PluginResourceMonitor:
|
||||
# Resource metrics per plugin
|
||||
self._metrics: Dict[str, ResourceMetrics] = {}
|
||||
self._limits: Dict[str, ResourceLimits] = {}
|
||||
# When each plugin's metrics last reached the cache. Metrics change on
|
||||
# every call, so they cannot be de-duplicated the way health state can;
|
||||
# they are rate-limited instead. See _METRICS_PERSIST_INTERVAL.
|
||||
self._metrics_persisted_at: Dict[str, float] = {}
|
||||
|
||||
# Thread-local storage for execution tracking
|
||||
self._local = threading.local()
|
||||
@@ -102,6 +120,66 @@ class PluginResourceMonitor:
|
||||
"psutil not available - resource monitoring will be limited to execution time only"
|
||||
)
|
||||
|
||||
def _metrics_from_cache(self, plugin_id: str, cached: Any) -> "ResourceMetrics":
|
||||
"""Build metrics from a cached record, ignoring anything unrecognised.
|
||||
|
||||
ResourceMetrics(**cached) raises TypeError on a single unexpected key,
|
||||
and that exception escapes into plugin_manager, which reports it as
|
||||
"plugin <id> operation failed". Every plugin fails, and the plugin
|
||||
system never finishes initialising.
|
||||
|
||||
Seen on a live rig: every plugin failing with
|
||||
|
||||
ResourceMetrics.__init__() got an unexpected keyword argument
|
||||
'consecutive_failures'
|
||||
|
||||
which is a plugin_health field, not a metrics one. How a health-shaped
|
||||
record came to sit under a plugin_metrics key on that machine is not
|
||||
established -- a restored backup that mixed two machines' caches is the
|
||||
likeliest explanation -- but the loader should not be brittle enough for
|
||||
it to matter. plugin_health already repairs its records field by field
|
||||
rather than trusting whatever is on disk; this does the same.
|
||||
|
||||
Unknown keys are dropped and named once, so a genuine schema change is
|
||||
visible in the log instead of silently discarded.
|
||||
"""
|
||||
if not isinstance(cached, dict):
|
||||
self.logger.warning(
|
||||
"Ignoring cached metrics for %s: expected a mapping, got %s",
|
||||
plugin_id, type(cached).__name__)
|
||||
return ResourceMetrics()
|
||||
|
||||
known = {f.name for f in fields(ResourceMetrics)}
|
||||
unknown = sorted(set(cached) - known)
|
||||
if unknown:
|
||||
self.logger.warning(
|
||||
"Dropping unrecognised field(s) from cached metrics for %s: %s",
|
||||
plugin_id, ", ".join(unknown))
|
||||
# A dataclass does not enforce its annotations, so
|
||||
# ResourceMetrics(call_count="not a number") builds happily and only
|
||||
# blows up later, deep inside monitor_call ("can only concatenate str
|
||||
# (not \"int\") to str"). Coerce here, where there is still a cache
|
||||
# key to name in the warning.
|
||||
declared = {f.name: f.type for f in fields(ResourceMetrics)}
|
||||
usable = {}
|
||||
for key, value in cached.items():
|
||||
if key not in known:
|
||||
continue
|
||||
try:
|
||||
usable[key] = int(value) if declared[key] in ('int', int) else float(value)
|
||||
except (TypeError, ValueError):
|
||||
self.logger.warning(
|
||||
"Cached metrics for %s have a bad %s (%r); starting fresh",
|
||||
plugin_id, key, value)
|
||||
return ResourceMetrics()
|
||||
try:
|
||||
return ResourceMetrics(**usable)
|
||||
except (TypeError, ValueError) as e:
|
||||
self.logger.warning(
|
||||
"Cached metrics for %s unusable (%s); starting fresh",
|
||||
plugin_id, e)
|
||||
return ResourceMetrics()
|
||||
|
||||
def _get_metrics_key(self, plugin_id: str) -> str:
|
||||
"""Get cache key for plugin metrics."""
|
||||
return f"plugin_metrics:{plugin_id}"
|
||||
@@ -126,7 +204,7 @@ class PluginResourceMonitor:
|
||||
cache_key, max_age=None, memory_ttl=0 if force_reload else None
|
||||
)
|
||||
if cached:
|
||||
metrics = ResourceMetrics(**cached)
|
||||
metrics = self._metrics_from_cache(plugin_id, cached)
|
||||
else:
|
||||
metrics = ResourceMetrics()
|
||||
self._metrics[plugin_id] = metrics
|
||||
@@ -232,18 +310,8 @@ class PluginResourceMonitor:
|
||||
# CPU is harder to measure per-call, so we track it separately
|
||||
metrics.cpu_percent = self._get_process_cpu_percent()
|
||||
|
||||
# Persist metrics
|
||||
cache_key = self._get_metrics_key(plugin_id)
|
||||
self.cache_manager.set(cache_key, {
|
||||
'memory_mb': metrics.memory_mb,
|
||||
'cpu_percent': metrics.cpu_percent,
|
||||
'execution_time': metrics.execution_time,
|
||||
'call_count': metrics.call_count,
|
||||
'total_execution_time': metrics.total_execution_time,
|
||||
'max_execution_time': metrics.max_execution_time,
|
||||
'min_execution_time': metrics.min_execution_time if metrics.min_execution_time != float('inf') else 0.0,
|
||||
'last_update_time': metrics.last_update_time
|
||||
})
|
||||
# Persist metrics, at most once per interval per plugin.
|
||||
self._persist_metrics(plugin_id, metrics)
|
||||
|
||||
# Check limits
|
||||
if limits:
|
||||
@@ -363,6 +431,44 @@ class PluginResourceMonitor:
|
||||
summaries[plugin_id] = self.get_metrics_summary(plugin_id)
|
||||
return summaries
|
||||
|
||||
def _persist_metrics(self, plugin_id: str, metrics: ResourceMetrics,
|
||||
force: bool = False) -> None:
|
||||
"""Write a plugin's metrics to the cache, at most once per interval.
|
||||
|
||||
Caller must hold ``self._lock``.
|
||||
"""
|
||||
# Monotonic, not wall clock: these devices have no RTC, so the clock
|
||||
# jumps by however far off boot-time was the moment NTP first syncs.
|
||||
# A forward jump would allow an early write, a backward one would
|
||||
# stall the snapshot well past the interval.
|
||||
#
|
||||
# The sentinel for "never written" is None, not 0.0. monotonic() is
|
||||
# time since boot on Linux, and systemd starts this service *at* boot,
|
||||
# so `now - 0.0 < 30` was true for the first half-minute of every
|
||||
# single run -- the throttle swallowed the very first snapshot, which
|
||||
# is the one that matters most after a restart.
|
||||
now = time.monotonic()
|
||||
last_written = self._metrics_persisted_at.get(plugin_id)
|
||||
if (not force and last_written is not None
|
||||
and now - last_written < _METRICS_PERSIST_INTERVAL):
|
||||
return
|
||||
cache_key = self._get_metrics_key(plugin_id)
|
||||
self.cache_manager.set(cache_key, {
|
||||
'memory_mb': metrics.memory_mb,
|
||||
'cpu_percent': metrics.cpu_percent,
|
||||
'execution_time': metrics.execution_time,
|
||||
'call_count': metrics.call_count,
|
||||
'total_execution_time': metrics.total_execution_time,
|
||||
'max_execution_time': metrics.max_execution_time,
|
||||
'min_execution_time': (metrics.min_execution_time
|
||||
if metrics.min_execution_time != float('inf')
|
||||
else 0.0),
|
||||
'last_update_time': metrics.last_update_time,
|
||||
})
|
||||
# Only after the write lands. Marking it first would mean a failed
|
||||
# set() bought the next interval's silence without leaving a snapshot.
|
||||
self._metrics_persisted_at[plugin_id] = now
|
||||
|
||||
def reset_metrics(self, plugin_id: str) -> None:
|
||||
"""Reset metrics for a plugin."""
|
||||
with self._lock:
|
||||
@@ -370,4 +476,7 @@ class PluginResourceMonitor:
|
||||
self._metrics[plugin_id] = ResourceMetrics()
|
||||
cache_key = self._get_metrics_key(plugin_id)
|
||||
self.cache_manager.delete(cache_key)
|
||||
# Let the next call persist immediately rather than leaving the
|
||||
# deleted key absent for the rest of the interval.
|
||||
self._metrics_persisted_at.pop(plugin_id, None)
|
||||
|
||||
|
||||
@@ -26,7 +26,25 @@ class SchemaManager:
|
||||
- Cache invalidation on plugin changes
|
||||
"""
|
||||
|
||||
def __init__(self, plugins_dir: Optional[Path] = None, project_root: Optional[Path] = None, logger: Optional[logging.Logger] = None):
|
||||
# Plugin config keys that mean "where this device is". A plugin declaring
|
||||
# any of these in its schema gets the device-wide ``location`` block from
|
||||
# config.json as the *default* for that field, instead of whatever city the
|
||||
# plugin author happened to ship. A value the user set on the plugin itself
|
||||
# always wins -- this only ever replaces the schema default, so an explicit
|
||||
# per-plugin location is still honoured.
|
||||
#
|
||||
# Only these fully-namespaced keys are substituted. A bare ``state`` or
|
||||
# ``city`` key is deliberately left alone: plugins use those for unrelated
|
||||
# things (ledmatrix-elections' ``state`` is a two-letter code, not a place
|
||||
# name), and silently rewriting them would break those plugins.
|
||||
DEVICE_LOCATION_KEYS: Dict[str, str] = {
|
||||
'location_city': 'city',
|
||||
'location_state': 'state',
|
||||
'location_country': 'country',
|
||||
}
|
||||
|
||||
def __init__(self, plugins_dir: Optional[Path] = None, project_root: Optional[Path] = None,
|
||||
logger: Optional[logging.Logger] = None, config_manager: Optional[Any] = None):
|
||||
"""
|
||||
Initialize the Schema Manager.
|
||||
|
||||
@@ -34,10 +52,14 @@ class SchemaManager:
|
||||
plugins_dir: Base plugins directory path
|
||||
project_root: Project root directory path
|
||||
logger: Optional logger instance
|
||||
config_manager: Optional config manager, used to resolve the
|
||||
device-wide ``location`` that seeds plugin location defaults.
|
||||
Omitting it simply leaves schema defaults untouched.
|
||||
"""
|
||||
self.logger = logger or logging.getLogger(__name__)
|
||||
self.plugins_dir = plugins_dir
|
||||
self.project_root = project_root or Path.cwd()
|
||||
self.config_manager = config_manager
|
||||
|
||||
# Schema cache: plugin_id -> schema dict
|
||||
self._schema_cache: Dict[str, Dict[str, Any]] = {}
|
||||
@@ -212,10 +234,70 @@ class SchemaManager:
|
||||
|
||||
return defaults
|
||||
|
||||
def get_device_location(self) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Return the device-wide ``location`` block from config.json, or None.
|
||||
|
||||
This is the City/State/Country the user sets once under General
|
||||
settings. Returns None when there is no config manager wired, the
|
||||
config can't be read, or no location has been configured.
|
||||
"""
|
||||
if self.config_manager is None:
|
||||
return None
|
||||
try:
|
||||
config = self.config_manager.load_config()
|
||||
except Exception as e:
|
||||
# A config that can't be read must never stop defaults being
|
||||
# generated -- the plugin's own schema defaults still apply.
|
||||
self.logger.debug(f"Could not read device location from config: {e}")
|
||||
return None
|
||||
if not isinstance(config, dict):
|
||||
return None
|
||||
location = config.get('location')
|
||||
return location if isinstance(location, dict) else None
|
||||
|
||||
def apply_device_location(self, defaults: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Replace location-shaped schema defaults with the device's own location.
|
||||
|
||||
Without this, a plugin that ships ``"location_city": "Dallas"`` as its
|
||||
schema default silently reports Dallas weather (and centres its radar
|
||||
there) for every user who never opened that plugin's config form --
|
||||
even though they set their real city under General settings. The
|
||||
substituted value is still only a *default*: ``merge_with_defaults``
|
||||
lets any per-plugin value the user saved win over it.
|
||||
|
||||
Mutates and returns ``defaults`` for convenience.
|
||||
"""
|
||||
if not defaults:
|
||||
return defaults
|
||||
if not any(key in defaults for key in self.DEVICE_LOCATION_KEYS):
|
||||
return defaults
|
||||
|
||||
location = self.get_device_location()
|
||||
if not location:
|
||||
return defaults
|
||||
|
||||
for key, field in self.DEVICE_LOCATION_KEYS.items():
|
||||
if key not in defaults:
|
||||
continue
|
||||
value = location.get(field)
|
||||
# Only a non-empty string is a real answer; a blank or missing
|
||||
# field means "not configured", which leaves the schema default.
|
||||
if isinstance(value, str) and value.strip():
|
||||
defaults[key] = value.strip()
|
||||
|
||||
return defaults
|
||||
|
||||
def generate_default_config(self, plugin_id: str, use_cache: bool = True) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate default configuration for a plugin from its schema.
|
||||
|
||||
Location fields (see ``DEVICE_LOCATION_KEYS``) default to the device's
|
||||
configured location rather than the plugin author's. That substitution
|
||||
is applied on the way out rather than being cached, so changing the
|
||||
device location takes effect without invalidating the defaults cache.
|
||||
|
||||
Args:
|
||||
plugin_id: Plugin identifier
|
||||
use_cache: If True, return cached defaults if available
|
||||
@@ -225,7 +307,7 @@ class SchemaManager:
|
||||
"""
|
||||
# Check cache first
|
||||
if use_cache and plugin_id in self._defaults_cache:
|
||||
return self._defaults_cache[plugin_id].copy()
|
||||
return self.apply_device_location(self._defaults_cache[plugin_id].copy())
|
||||
|
||||
schema = self.load_schema(plugin_id, use_cache=use_cache)
|
||||
if not schema:
|
||||
@@ -249,10 +331,11 @@ class SchemaManager:
|
||||
if 'live_priority' not in defaults:
|
||||
defaults['live_priority'] = schema.get('properties', {}).get('live_priority', {}).get('default', False)
|
||||
|
||||
# Cache the defaults
|
||||
# Cache the defaults *before* the device location is layered on, so a
|
||||
# later change to the device location is picked up by the next call.
|
||||
self._defaults_cache[plugin_id] = defaults.copy()
|
||||
|
||||
return defaults
|
||||
return self.apply_device_location(defaults)
|
||||
|
||||
def validate_config_against_schema(self, config: Dict[str, Any], schema: Dict[str, Any],
|
||||
plugin_id: Optional[str] = None) -> Tuple[bool, List[str]]:
|
||||
|
||||
@@ -1467,8 +1467,11 @@ class PluginStoreManager:
|
||||
# already had. Allowing it costs them a plugin that raises
|
||||
# ModuleNotFoundError at load and is reported only as one line
|
||||
# in the journal. See docs/SPORTS_UNIFICATION.md (phase B4/B6).
|
||||
from src import __version__ as core_version
|
||||
from src.plugin_system import compatibility
|
||||
# On disk, not as imported: this process may predate the core
|
||||
# update that made the plugin compatible. See
|
||||
# compatibility.current_core_version.
|
||||
core_version = compatibility.current_core_version()
|
||||
|
||||
compatible, reason = compatibility.check(manifest, core_version)
|
||||
if not compatible:
|
||||
@@ -1602,6 +1605,26 @@ class PluginStoreManager:
|
||||
'error': f'Manifest missing required fields: {", ".join(missing_fields)}'
|
||||
}
|
||||
|
||||
# Refuse a plugin that needs a newer core than this one, exactly as
|
||||
# _install_plugin_impl does after its download. Sideloading is an
|
||||
# explicit act rather than an automatic store update, but the floor
|
||||
# is not advice about intent -- it is a statement that the plugin
|
||||
# cannot run here, and letting it through produces the same silent
|
||||
# PluginState.ERROR at load. This was the last of the three routes
|
||||
# in that skipped the check.
|
||||
#
|
||||
# Before the move, so the `finally` below removes the temp tree and
|
||||
# nothing half-installed is left behind.
|
||||
from src.plugin_system import compatibility
|
||||
core_version = compatibility.current_core_version()
|
||||
|
||||
compatible, reason = compatibility.check(manifest, core_version)
|
||||
if not compatible:
|
||||
self.logger.error(
|
||||
"Refusing to install %s from %s: %s",
|
||||
plugin_id, repo_url, reason)
|
||||
return {'success': False, 'error': reason}
|
||||
|
||||
# Validate version fields consistency (warnings only, not required)
|
||||
validation_errors = self._validate_manifest_version_fields(manifest)
|
||||
if validation_errors:
|
||||
@@ -2583,6 +2606,85 @@ class PluginStoreManager:
|
||||
self.logger.error(f"Error uninstalling plugin {plugin_id}: {e}")
|
||||
return False
|
||||
|
||||
def _gate_pulled_commit(self, plugin_id: str, plugin_path: Path,
|
||||
previous_sha: Optional[str]) -> bool:
|
||||
"""Apply the compatibility gate to a commit that arrived via git pull.
|
||||
|
||||
Every other route into an installed plugin goes through
|
||||
``install_plugin``, which gates in ``_install_plugin_impl``. This one
|
||||
did not: a ``git pull`` could deliver a manifest flooring above this
|
||||
core and nothing would notice until the plugin failed to load, which
|
||||
surfaces as one line in the journal and a scoreboard that silently
|
||||
stopped appearing.
|
||||
|
||||
Checked after the pull rather than before it, for the same reason
|
||||
``_install_plugin_impl`` checks after the download: the registry
|
||||
carries no compatibility field, so the incoming floor is only knowable
|
||||
once the new commit is on disk.
|
||||
|
||||
Undone with ``git reset --hard`` rather than by removing the directory.
|
||||
This is a live checkout, the previous commit is still in the object
|
||||
store, and the reset leaves the user on the exact version they were
|
||||
already running -- the same promise ``_reinstall_with_rollback`` makes,
|
||||
reached by the means this path actually has. It is also the gentler
|
||||
option: no window in which the plugin directory does not exist, and no
|
||||
``.standalone-backup-`` debris if the process dies mid-way.
|
||||
|
||||
A manifest that cannot be read is not evidence of incompatibility, so
|
||||
it allows. ``compatibility.check`` refuses only on evidence for the
|
||||
same reason: a wrong refusal breaks a working install, while a wrong
|
||||
allowance degrades to exactly the behaviour this path had before the
|
||||
gate existed.
|
||||
"""
|
||||
manifest_path = plugin_path / "manifest.json"
|
||||
try:
|
||||
with open(manifest_path, 'r', encoding='utf-8') as mf:
|
||||
manifest = json.load(mf)
|
||||
except (OSError, ValueError) as e:
|
||||
self.logger.warning(
|
||||
"Could not read %s after updating %s (%s); allowing the "
|
||||
"update, as an unreadable manifest declares no floor",
|
||||
manifest_path, plugin_id, e)
|
||||
return True
|
||||
|
||||
from src.plugin_system import compatibility
|
||||
core_version = compatibility.current_core_version()
|
||||
|
||||
compatible, reason = compatibility.check(manifest, core_version)
|
||||
if compatible:
|
||||
return True
|
||||
|
||||
self.logger.error("Refusing the update to %s: %s", plugin_id, reason)
|
||||
|
||||
if not previous_sha:
|
||||
self.logger.error(
|
||||
"Cannot roll %s back: the commit it was on before the pull is "
|
||||
"unknown. It is now on a version this core cannot run — "
|
||||
"reinstall it from the plugin store.", plugin_id)
|
||||
return False
|
||||
|
||||
# Safe by construction: update_plugin returns before pulling unless the
|
||||
# tree was clean or successfully stashed, so there are no uncommitted
|
||||
# tracked edits for --hard to discard. The stash is not popped on the
|
||||
# success path either, so the reset leaves the working tree exactly
|
||||
# where a successful pull would have. Say "commit", not "changes".
|
||||
reset = subprocess.run(
|
||||
['git', '-C', str(plugin_path), 'reset', '--hard', previous_sha],
|
||||
capture_output=True, text=True, timeout=60, check=False)
|
||||
if reset.returncode != 0:
|
||||
self.logger.error(
|
||||
"CRITICAL: could not roll %s back to commit %s: %s. It is left "
|
||||
"on a version this core cannot run; "
|
||||
"`git -C %s reset --hard %s` restores it.",
|
||||
plugin_id, previous_sha[:7],
|
||||
(reset.stderr or reset.stdout or '').strip(),
|
||||
plugin_path, previous_sha)
|
||||
else:
|
||||
self.logger.info(
|
||||
"Rolled %s back to commit %s; it stays on the version it was "
|
||||
"already running.", plugin_id, previous_sha[:7])
|
||||
return False
|
||||
|
||||
def _reinstall_with_rollback(self, plugin_id: str, plugin_path: Path) -> bool:
|
||||
"""Replace an installed plugin with a fresh install, atomically.
|
||||
|
||||
@@ -2860,6 +2962,8 @@ class PluginStoreManager:
|
||||
status_result = type('obj', (object,), {'stdout': '', 'stderr': 'Status check timed out'})()
|
||||
|
||||
stash_info = ""
|
||||
# Whether the pull can be undone without destroying work.
|
||||
tree_is_recoverable = not has_changes
|
||||
if has_changes:
|
||||
self.logger.info(f"Stashing local changes in {plugin_id} before update")
|
||||
try:
|
||||
@@ -2873,12 +2977,37 @@ class PluginStoreManager:
|
||||
)
|
||||
if stash_result.returncode == 0:
|
||||
stash_info = " (local changes were stashed)"
|
||||
tree_is_recoverable = True
|
||||
self.logger.info(f"Stashed local changes (including untracked files) for {plugin_id}")
|
||||
else:
|
||||
self.logger.warning(f"Failed to stash local changes for {plugin_id}: {stash_result.stderr}")
|
||||
except subprocess.TimeoutExpired:
|
||||
self.logger.warning(f"Stash operation timed out for {plugin_id}, proceeding with pull")
|
||||
|
||||
# Do not pull what cannot be un-pulled.
|
||||
#
|
||||
# The compatibility gate below can refuse the commit this
|
||||
# pull brings down, and its only way back is `git reset
|
||||
# --hard`, which discards uncommitted tracked edits. Those
|
||||
# edits are exactly what the stash above exists to protect,
|
||||
# so a stash that failed or timed out leaves the rollback
|
||||
# unable to run without destroying them.
|
||||
#
|
||||
# A pull does not necessarily refuse on a dirty tree -- git
|
||||
# merges happily as long as the incoming commit touches
|
||||
# different files -- so without this the update would
|
||||
# succeed, the gate would refuse, and the reset would take
|
||||
# the user's work with it. Refusing here costs an update in
|
||||
# a case that already went wrong; the alternative costs
|
||||
# data.
|
||||
if not tree_is_recoverable:
|
||||
self.logger.error(
|
||||
"Refusing to update %s: it has local changes that could "
|
||||
"not be stashed, and an incompatible update could then "
|
||||
"only be rolled back by discarding them. Commit or stash "
|
||||
"them by hand, then update.", plugin_id)
|
||||
return False
|
||||
|
||||
# Pull from the determined remote branch
|
||||
self.logger.info(f"Pulling from origin/{remote_pull_branch} for {plugin_id}...")
|
||||
pull_result = subprocess.run(
|
||||
@@ -2901,6 +3030,14 @@ class PluginStoreManager:
|
||||
elif updated_sha:
|
||||
self.logger.info(f"Plugin {plugin_id} updated to commit {updated_sha[:7]}{stash_info}")
|
||||
|
||||
# The install gate, at the only point on this path where
|
||||
# it can be answered. Every other route in goes through
|
||||
# install_plugin, which gates in _install_plugin_impl; this
|
||||
# one did not, so a pull could deliver a manifest flooring
|
||||
# above this core and nothing would notice.
|
||||
if not self._gate_pulled_commit(plugin_id, plugin_path, local_sha):
|
||||
return False
|
||||
|
||||
self._install_dependencies(plugin_path)
|
||||
return True
|
||||
|
||||
|
||||
@@ -62,6 +62,9 @@ class StartupValidator:
|
||||
# Validate plugins if plugin manager is available
|
||||
if self.plugin_manager:
|
||||
self._validate_plugins()
|
||||
|
||||
# Warn when the running systemd unit no longer matches the repo's
|
||||
self._validate_systemd_units()
|
||||
|
||||
is_valid = len(self.errors) == 0
|
||||
|
||||
@@ -74,6 +77,80 @@ class StartupValidator:
|
||||
|
||||
return (is_valid, self.errors.copy(), self.warnings.copy())
|
||||
|
||||
#: Units this project installs, and where each is installed to.
|
||||
_UNITS = (
|
||||
("systemd/ledmatrix.service", "/etc/systemd/system/ledmatrix.service"),
|
||||
("systemd/ledmatrix-web.service", "/etc/systemd/system/ledmatrix-web.service"),
|
||||
)
|
||||
|
||||
def _validate_systemd_units(self) -> None:
|
||||
"""Warn when an installed unit has drifted from the repo's template.
|
||||
|
||||
Nothing re-applies these after the first install. `git pull` -- which is
|
||||
what the web UI's update button runs -- brings a new template into the
|
||||
checkout, but nothing copies it to /etc/systemd/system and nothing runs
|
||||
`systemctl daemon-reload`, so the unit that actually runs is whatever
|
||||
first_time_install.sh wrote on day one.
|
||||
|
||||
That makes every hardening added to a unit inert on existing installs.
|
||||
Measured on one rig: the installed unit was thirteen days older than the
|
||||
repo's and differed in content, so a MemoryMax the repo had specified
|
||||
was not being enforced at all -- `systemctl show` reported
|
||||
MemoryMax=infinity.
|
||||
|
||||
A warning rather than an error, and certainly not a silent rewrite:
|
||||
editing files under /etc and restarting services is the installer's job,
|
||||
not something a display process should do to a machine while it boots.
|
||||
The remedy is to re-run scripts/install/install_service.sh.
|
||||
"""
|
||||
try:
|
||||
project_root = Path(__file__).resolve().parent.parent
|
||||
for template_rel, installed_path in self._UNITS:
|
||||
template = project_root / template_rel
|
||||
installed = Path(installed_path)
|
||||
if not template.is_file() or not installed.is_file():
|
||||
continue
|
||||
|
||||
# The template carries placeholders the installer substitutes,
|
||||
# so compare the substituted form rather than the raw file.
|
||||
expected = template.read_text(encoding="utf-8")
|
||||
expected = expected.replace("__PROJECT_ROOT_DIR__", str(project_root))
|
||||
expected = expected.replace("__USER__", "root")
|
||||
|
||||
try:
|
||||
actual = installed.read_text(encoding="utf-8")
|
||||
except PermissionError:
|
||||
continue
|
||||
|
||||
if self._unit_body(expected) != self._unit_body(actual):
|
||||
self.warnings.append(
|
||||
f"{installed.name} differs from {template_rel}; the "
|
||||
"installed unit is not refreshed by an update, so "
|
||||
"settings added to the template are not in effect. "
|
||||
"Re-run scripts/install/install_service.sh to apply them."
|
||||
)
|
||||
except OSError as e:
|
||||
self.logger.debug("Could not compare systemd units: %s", e)
|
||||
|
||||
@staticmethod
|
||||
def _unit_body(text: str) -> str:
|
||||
"""A unit's meaningful lines, in order: no comments, no blanks.
|
||||
|
||||
Order is preserved deliberately. This used to sort, which made the
|
||||
comparison insensitive to two changes that matter in a systemd unit:
|
||||
repeated directives such as ExecStartPre= and ExecStartPost= run in
|
||||
the order they appear, and a directive that moves between [Unit],
|
||||
[Service] and [Install] means something different -- or nothing --
|
||||
where it lands. A drift check that normalises those away reports no
|
||||
drift for a unit that has genuinely changed.
|
||||
"""
|
||||
lines = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#"):
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
def _validate_config(self) -> None:
|
||||
"""Validate configuration files."""
|
||||
try:
|
||||
|
||||
@@ -31,6 +31,18 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
#: Degradation threshold, as a fraction of target_fps. A marquee jitters a
|
||||
#: little all the time, so "anything under target" would report constantly and
|
||||
#: mean nothing; 90% of target is the point where a shortfall is real. At a
|
||||
#: 60fps target that is 54fps -- 55fps is a normal wobble and stays at DEBUG,
|
||||
#: which is deliberate, not an off-by-one.
|
||||
_FPS_HEALTHY_FRACTION = 0.9
|
||||
|
||||
#: A healthy marquee still reports this often, so silence means stopped
|
||||
#: rather than fine.
|
||||
_FPS_HEARTBEAT_INTERVAL = 300.0
|
||||
|
||||
|
||||
def _percentile(ordered: List[float], fraction: float) -> float:
|
||||
"""Nearest-rank percentile of an already-sorted list.
|
||||
|
||||
@@ -96,6 +108,11 @@ class VegasModeCoordinator:
|
||||
self._is_active = False
|
||||
self._is_paused = False
|
||||
self._should_stop = False
|
||||
# Frame-rate health, tracked across run_iteration() calls so the
|
||||
# heartbeat is one-per-interval rather than one-per-cycle, and so a
|
||||
# recovery spanning two cycles is still reported. Reset on start().
|
||||
self._fps_last_health_log = 0.0
|
||||
self._fps_was_degraded = False
|
||||
self._state_lock = threading.Lock()
|
||||
|
||||
# Live priority tracking
|
||||
@@ -248,6 +265,11 @@ class VegasModeCoordinator:
|
||||
self._is_active = True
|
||||
self._should_stop = False
|
||||
self._start_time = time.time()
|
||||
# A fresh run starts with a clean health slate: no stale
|
||||
# "was degraded" from the previous run, and a heartbeat that is
|
||||
# due immediately so the first sample confirms the marquee is up.
|
||||
self._fps_last_health_log = 0.0
|
||||
self._fps_was_degraded = False
|
||||
|
||||
# Line up the next group immediately, so the first extension is already
|
||||
# warm rather than stalling the scroll to fetch it.
|
||||
@@ -395,8 +417,18 @@ class VegasModeCoordinator:
|
||||
duration = self.render_pipeline.get_dynamic_duration()
|
||||
start_time = time.time()
|
||||
frame_count = 0
|
||||
fps_log_interval = 5.0 # Log FPS every 5 seconds
|
||||
last_fps_log_time = start_time
|
||||
fps_log_interval = 5.0 # Sample FPS every 5 seconds
|
||||
# Health state lives on the coordinator, not here: run_iteration() is
|
||||
# called once per cycle, so locals reset every few seconds. That made
|
||||
# `last_fps_health_log = 0.0` fire the "heartbeat" on the first sample
|
||||
# of every iteration rather than once per interval, and a recovery
|
||||
# that crossed an iteration boundary was never reported at all --
|
||||
# was_degraded had already gone back to False.
|
||||
# Monotonic, and deliberately not start_time: start_time is wall
|
||||
# clock and is used below to report the iteration's duration. Mixing
|
||||
# the two here would make every delta hugely negative and silence the
|
||||
# frame-rate reporting altogether.
|
||||
last_fps_log_time = time.monotonic()
|
||||
fps_frame_count = 0
|
||||
# A mean hides stutter completely. At 120fps a five-second window is
|
||||
# ~600 frames, so a 200ms freeze -- plainly visible on a marquee --
|
||||
@@ -408,7 +440,13 @@ class VegasModeCoordinator:
|
||||
logger.info("Starting Vegas iteration for %.1fs", duration)
|
||||
|
||||
while True:
|
||||
frame_started = time.time()
|
||||
# Monotonic, like the FPS window below. These devices have no RTC,
|
||||
# so the wall clock jumps by however wrong boot time was the moment
|
||||
# NTP first syncs. A backward jump makes frame_elapsed negative,
|
||||
# and `frame_interval - frame_elapsed` then sleeps for longer than
|
||||
# the whole budget -- the render loop stalls for the size of the
|
||||
# correction. A forward jump inflates p99 and worst-frame instead.
|
||||
frame_started = time.monotonic()
|
||||
|
||||
# Check for STATIC mode plugin that should pause scroll
|
||||
static_plugin = self._check_static_plugin_trigger()
|
||||
@@ -436,7 +474,7 @@ class VegasModeCoordinator:
|
||||
# quarter of the budget spent not rendering. Subtracting the work
|
||||
# already done keeps the pacing target while reclaiming that time,
|
||||
# and yields the GIL either way so other threads still run.
|
||||
frame_elapsed = time.time() - frame_started
|
||||
frame_elapsed = time.monotonic() - frame_started
|
||||
time.sleep(max(0.0, frame_interval - frame_elapsed))
|
||||
|
||||
# Measured before the sleep: time spent working, not pacing.
|
||||
@@ -448,16 +486,42 @@ class VegasModeCoordinator:
|
||||
frame_count += 1
|
||||
fps_frame_count += 1
|
||||
|
||||
# Periodic FPS logging
|
||||
current_time = time.time()
|
||||
# Periodic FPS logging. Reported at INFO only when the frame rate
|
||||
# is actually worth an operator's attention -- a shortfall against
|
||||
# target, or the recovery from one -- with a slow heartbeat so a
|
||||
# healthy marquee still shows a pulse.
|
||||
#
|
||||
# Measured over two hours on a running rig: 1410 samples, 98.5%
|
||||
# of them within 10% of target. The 1.5% that were not included a
|
||||
# reading of 8.6fps against a target of 60 -- a real stall, and
|
||||
# completely invisible inside 1389 lines reading "59.6".
|
||||
# Monotonic: every use of this value in the block below is a
|
||||
# duration, and these devices have no RTC, so the wall clock jumps
|
||||
# by however wrong boot time was the moment NTP first syncs. That
|
||||
# would not only mis-fire the heartbeat, it would corrupt the
|
||||
# frame rate itself, since fps is frames divided by this delta.
|
||||
current_time = time.monotonic()
|
||||
if current_time - last_fps_log_time >= fps_log_interval:
|
||||
fps = fps_frame_count / (current_time - last_fps_log_time)
|
||||
p99 = _percentile(sorted(frame_times), 0.99)
|
||||
logger.info(
|
||||
"Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms",
|
||||
fps, self.vegas_config.target_fps, fps_frame_count,
|
||||
p99 * 1000.0, frame_worst * 1000.0
|
||||
)
|
||||
target = self.vegas_config.target_fps
|
||||
degraded = target > 0 and fps < target * _FPS_HEALTHY_FRACTION
|
||||
due = (current_time - self._fps_last_health_log
|
||||
>= _FPS_HEARTBEAT_INTERVAL)
|
||||
if degraded or self._fps_was_degraded or due:
|
||||
logger.info(
|
||||
"Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms",
|
||||
fps, target, fps_frame_count,
|
||||
p99 * 1000.0, frame_worst * 1000.0
|
||||
)
|
||||
self._fps_last_health_log = current_time
|
||||
else:
|
||||
logger.debug(
|
||||
"Vegas FPS: %.1f (target: %d, frames: %d) p99 %.1fms worst %.1fms",
|
||||
fps, target, fps_frame_count,
|
||||
p99 * 1000.0, frame_worst * 1000.0
|
||||
)
|
||||
self._fps_was_degraded = degraded
|
||||
last_fps_log_time = current_time
|
||||
fps_frame_count = 0
|
||||
frame_worst = 0.0
|
||||
|
||||
@@ -83,7 +83,7 @@ class PluginAdapter:
|
||||
# into unrelated headlines once the strip refreshed to 9,505px.
|
||||
self._offset_shapes: dict = {}
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"PluginAdapter initialized: display=%dx%d",
|
||||
self.display_width, self.display_height
|
||||
)
|
||||
@@ -109,7 +109,7 @@ class PluginAdapter:
|
||||
Returns:
|
||||
List of PIL Images representing plugin content, or None if no content
|
||||
"""
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Getting content (class=%s)",
|
||||
plugin_id, plugin.__class__.__name__
|
||||
)
|
||||
@@ -118,7 +118,7 @@ class PluginAdapter:
|
||||
cached = self._get_cached(plugin_id)
|
||||
if cached is not None:
|
||||
total_width = sum(img.width for img in cached)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Using cached content: %d images, %dpx total",
|
||||
plugin_id, len(cached), total_width
|
||||
)
|
||||
@@ -126,46 +126,46 @@ class PluginAdapter:
|
||||
|
||||
# Try native Vegas content method first
|
||||
has_native = hasattr(plugin, 'get_vegas_content')
|
||||
logger.info("[%s] Has get_vegas_content: %s", plugin_id, has_native)
|
||||
logger.debug("[%s] Has get_vegas_content: %s", plugin_id, has_native)
|
||||
if has_native:
|
||||
content = self._get_native_content(plugin, plugin_id, offscreen_only)
|
||||
if content:
|
||||
total_width = sum(img.width for img in content)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Native content SUCCESS: %d images, %dpx total",
|
||||
plugin_id, len(content), total_width
|
||||
)
|
||||
return self._finalize(content, plugin_id, 'native', plugin)
|
||||
logger.info("[%s] Native content returned None", plugin_id)
|
||||
logger.debug("[%s] Native content returned None", plugin_id)
|
||||
|
||||
# Try to get scroll_helper's cached image (for scrolling plugins like stocks/odds)
|
||||
has_scroll_helper = hasattr(plugin, 'scroll_helper')
|
||||
logger.info("[%s] Has scroll_helper: %s", plugin_id, has_scroll_helper)
|
||||
logger.debug("[%s] Has scroll_helper: %s", plugin_id, has_scroll_helper)
|
||||
content = self._get_scroll_helper_content(plugin, plugin_id, offscreen_only)
|
||||
if content:
|
||||
total_width = sum(img.width for img in content)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] ScrollHelper content SUCCESS: %d images, %dpx total",
|
||||
plugin_id, len(content), total_width
|
||||
)
|
||||
return self._finalize(content, plugin_id, 'scroll_helper', plugin)
|
||||
if has_scroll_helper:
|
||||
logger.info("[%s] ScrollHelper content returned None", plugin_id)
|
||||
logger.debug("[%s] ScrollHelper content returned None", plugin_id)
|
||||
|
||||
if offscreen_only:
|
||||
# Display capture needs the shared canvas; leave it to the caller.
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Needs display capture, deferring to the render thread",
|
||||
plugin_id
|
||||
)
|
||||
return None
|
||||
|
||||
# Fall back to display capture
|
||||
logger.info("[%s] Trying fallback display capture...", plugin_id)
|
||||
logger.debug("[%s] Trying fallback display capture...", plugin_id)
|
||||
content = self._capture_display_content(plugin, plugin_id)
|
||||
if content:
|
||||
total_width = sum(img.width for img in content)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Fallback capture SUCCESS: %d images, %dpx total",
|
||||
plugin_id, len(content), total_width
|
||||
)
|
||||
@@ -226,7 +226,7 @@ class PluginAdapter:
|
||||
kept.append(result.image)
|
||||
|
||||
if not kept:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] All %d image(s) from %s were blank — contributing nothing",
|
||||
plugin_id, len(images), source
|
||||
)
|
||||
@@ -235,14 +235,14 @@ class PluginAdapter:
|
||||
trimmed_width = sum(img.width for img in kept)
|
||||
|
||||
if trimmed_width < self.config.min_plugin_width:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Trimmed content %dpx is below min_plugin_width %dpx — skipping",
|
||||
plugin_id, trimmed_width, self.config.min_plugin_width
|
||||
)
|
||||
return None
|
||||
|
||||
if trimmed_width != original_width or dropped_blank:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Trimmed %s content: %dpx -> %dpx (%.0f%% reclaimed), "
|
||||
"%d image(s) kept, %d blank dropped",
|
||||
plugin_id, source, original_width, trimmed_width,
|
||||
@@ -431,7 +431,7 @@ class PluginAdapter:
|
||||
"""
|
||||
if self._offset_shapes.get(plugin_id) != shape:
|
||||
if plugin_id in self._item_offsets:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Content is %s now, was %s — restarting the rotation "
|
||||
"rather than resuming at a position that no longer means "
|
||||
"anything", plugin_id, shape,
|
||||
@@ -579,7 +579,7 @@ class PluginAdapter:
|
||||
consumed += 1
|
||||
|
||||
if mode == 'truncate':
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Width budget %dpx: showing the first %d of %d row(s) "
|
||||
"(%dpx incl. gaps); the rest are not shown (overflow=truncate)",
|
||||
plugin_id, budget, len(selected), len(images), used
|
||||
@@ -587,7 +587,7 @@ class PluginAdapter:
|
||||
else:
|
||||
self._record_offset(
|
||||
plugin_id, (start + consumed) % len(images), shape)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Width budget %dpx: showing %d of %d row(s) (%dpx incl. gaps) "
|
||||
"from offset %d; remainder deferred to a later cycle",
|
||||
plugin_id, budget, len(selected), len(images), used, start
|
||||
@@ -636,7 +636,7 @@ class PluginAdapter:
|
||||
if mode != 'truncate':
|
||||
self._record_offset(
|
||||
plugin_id, 0 if end >= img.width else end, shape)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Width budget %dpx: cropped continuous %dpx image to "
|
||||
"[%d:%d] (no item gaps of %dpx+ to align to)%s",
|
||||
plugin_id, budget, img.width, offset, end, min_run,
|
||||
@@ -674,7 +674,7 @@ class PluginAdapter:
|
||||
self._record_offset(
|
||||
plugin_id, 0 if end >= img.width else end_index, shape)
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Width budget %dpx: cropped single %dpx image to [%d:%d] "
|
||||
"(%dpx) at item boundaries %d-%d of %d, %s",
|
||||
plugin_id, budget, img.width, start, end, end - start,
|
||||
@@ -698,7 +698,7 @@ class PluginAdapter:
|
||||
List of images or None
|
||||
"""
|
||||
try:
|
||||
logger.info("[%s] Native: calling get_vegas_content()", plugin_id)
|
||||
logger.debug("[%s] Native: calling get_vegas_content()", plugin_id)
|
||||
|
||||
# Tell the plugin how much width the ticker wants it to use, and
|
||||
# narrow the canvas for the duration of the call. A plugin that
|
||||
@@ -707,7 +707,7 @@ class PluginAdapter:
|
||||
# be explicit can read get_vegas_render_width().
|
||||
render_width = self.resolve_render_width(plugin, plugin_id)
|
||||
if render_width != self.display_width:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Native: requesting %dpx instead of %dpx",
|
||||
plugin_id, render_width, self.display_width
|
||||
)
|
||||
@@ -735,19 +735,19 @@ class PluginAdapter:
|
||||
plugin._vegas_render_width = None
|
||||
|
||||
if result is None:
|
||||
logger.info("[%s] Native: get_vegas_content() returned None", plugin_id)
|
||||
logger.debug("[%s] Native: get_vegas_content() returned None", plugin_id)
|
||||
return None
|
||||
|
||||
# Normalize to list
|
||||
if isinstance(result, Image.Image):
|
||||
images = [result]
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Native: got single Image %dx%d",
|
||||
plugin_id, result.width, result.height
|
||||
)
|
||||
elif isinstance(result, (list, tuple)):
|
||||
images = list(result)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Native: got %d items in list/tuple",
|
||||
plugin_id, len(images)
|
||||
)
|
||||
@@ -768,14 +768,14 @@ class PluginAdapter:
|
||||
)
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Native: item[%d] is %dx%d, mode=%s",
|
||||
plugin_id, i, img.width, img.height, img.mode
|
||||
)
|
||||
|
||||
# Ensure correct height
|
||||
if img.height != self.display_height:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Native: resizing item[%d]: %dx%d -> %dx%d",
|
||||
plugin_id, i, img.width, img.height,
|
||||
img.width, self.display_height
|
||||
@@ -793,13 +793,13 @@ class PluginAdapter:
|
||||
|
||||
if valid_images:
|
||||
total_width = sum(img.width for img in valid_images)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Native: SUCCESS - %d images, %dpx total width",
|
||||
plugin_id, len(valid_images), total_width
|
||||
)
|
||||
return valid_images
|
||||
|
||||
logger.info("[%s] Native: no valid images after validation", plugin_id)
|
||||
logger.debug("[%s] Native: no valid images after validation", plugin_id)
|
||||
return None
|
||||
|
||||
except (AttributeError, TypeError, ValueError, OSError) as e:
|
||||
@@ -833,20 +833,20 @@ class PluginAdapter:
|
||||
logger.debug("[%s] No scroll_helper attribute", plugin_id)
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Found scroll_helper: %s",
|
||||
plugin_id, type(scroll_helper).__name__
|
||||
)
|
||||
|
||||
cached_image = getattr(scroll_helper, 'cached_image', None)
|
||||
if cached_image is None:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] scroll_helper.cached_image is None, triggering content generation",
|
||||
plugin_id
|
||||
)
|
||||
if offscreen_only:
|
||||
# Generating it calls display(), which needs the canvas.
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] scroll_helper cache empty; deferring generation "
|
||||
"to the render thread", plugin_id
|
||||
)
|
||||
@@ -859,13 +859,13 @@ class PluginAdapter:
|
||||
return None
|
||||
|
||||
if not isinstance(cached_image, Image.Image):
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] scroll_helper.cached_image is not an Image: %s",
|
||||
plugin_id, type(cached_image).__name__
|
||||
)
|
||||
return None
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] scroll_helper.cached_image found: %dx%d, mode=%s",
|
||||
plugin_id, cached_image.width, cached_image.height, cached_image.mode
|
||||
)
|
||||
@@ -888,7 +888,7 @@ class PluginAdapter:
|
||||
|
||||
# Ensure correct height
|
||||
if img.height != self.display_height:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Resizing scroll_helper content: %dx%d -> %dx%d",
|
||||
plugin_id, img.width, img.height,
|
||||
img.width, self.display_height
|
||||
@@ -902,7 +902,7 @@ class PluginAdapter:
|
||||
if img.mode != 'RGB':
|
||||
img = img.convert('RGB')
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] ScrollHelper content ready: %dx%d",
|
||||
plugin_id, img.width, img.height
|
||||
)
|
||||
@@ -1002,7 +1002,7 @@ class PluginAdapter:
|
||||
with self._capture():
|
||||
# Method 1: Try _create_scrolling_display (stocks pattern)
|
||||
if hasattr(plugin, '_create_scrolling_display'):
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Triggering via _create_scrolling_display()",
|
||||
plugin_id
|
||||
)
|
||||
@@ -1010,7 +1010,7 @@ class PluginAdapter:
|
||||
plugin._create_scrolling_display()
|
||||
cached_image = getattr(scroll_helper, 'cached_image', None)
|
||||
if cached_image is not None and isinstance(cached_image, Image.Image):
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] _create_scrolling_display() SUCCESS: %dx%d",
|
||||
plugin_id, cached_image.width, cached_image.height
|
||||
)
|
||||
@@ -1022,7 +1022,7 @@ class PluginAdapter:
|
||||
|
||||
# Method 2: Try display(force_clear=True) which typically builds scroll content
|
||||
if hasattr(plugin, 'display'):
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Triggering via display(force_clear=True)",
|
||||
plugin_id
|
||||
)
|
||||
@@ -1031,12 +1031,12 @@ class PluginAdapter:
|
||||
plugin.display(force_clear=True)
|
||||
cached_image = getattr(scroll_helper, 'cached_image', None)
|
||||
if cached_image is not None and isinstance(cached_image, Image.Image):
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] display(force_clear=True) SUCCESS: %dx%d",
|
||||
plugin_id, cached_image.width, cached_image.height
|
||||
)
|
||||
return cached_image
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] display(force_clear=True) did not populate cached_image",
|
||||
plugin_id
|
||||
)
|
||||
@@ -1045,7 +1045,7 @@ class PluginAdapter:
|
||||
"[%s] display(force_clear=True) failed", plugin_id
|
||||
)
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Could not trigger scroll content generation",
|
||||
plugin_id
|
||||
)
|
||||
@@ -1077,15 +1077,15 @@ class PluginAdapter:
|
||||
try:
|
||||
# Save current display state
|
||||
original_image = self.display_manager.image.copy()
|
||||
logger.info("[%s] Fallback: saved original display state", plugin_id)
|
||||
logger.debug("[%s] Fallback: saved original display state", plugin_id)
|
||||
|
||||
# Ensure plugin has fresh data before capturing
|
||||
has_update_data = hasattr(plugin, 'update_data')
|
||||
logger.info("[%s] Fallback: has update_data=%s", plugin_id, has_update_data)
|
||||
logger.debug("[%s] Fallback: has update_data=%s", plugin_id, has_update_data)
|
||||
if has_update_data:
|
||||
try:
|
||||
plugin.update_data()
|
||||
logger.info("[%s] Fallback: update_data() called", plugin_id)
|
||||
logger.debug("[%s] Fallback: update_data() called", plugin_id)
|
||||
except (AttributeError, RuntimeError, OSError):
|
||||
logger.exception("[%s] Fallback: update_data() failed", plugin_id)
|
||||
|
||||
@@ -1097,41 +1097,41 @@ class PluginAdapter:
|
||||
# arrangement rather than one that has to be cropped afterwards.
|
||||
render_width = self.resolve_render_width(plugin, plugin_id)
|
||||
if render_width != self.display_width:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Fallback: rendering at %dpx instead of %dpx",
|
||||
plugin_id, render_width, self.display_width
|
||||
)
|
||||
|
||||
with self._capture(), self._render_at(render_width):
|
||||
self.display_manager.clear()
|
||||
logger.info("[%s] Fallback: display cleared, calling display()", plugin_id)
|
||||
logger.debug("[%s] Fallback: display cleared, calling display()", plugin_id)
|
||||
|
||||
# First try without force_clear (some plugins behave better this way)
|
||||
try:
|
||||
plugin.display()
|
||||
logger.info("[%s] Fallback: display() called successfully", plugin_id)
|
||||
logger.debug("[%s] Fallback: display() called successfully", plugin_id)
|
||||
except TypeError:
|
||||
# Plugin may require force_clear argument
|
||||
logger.info("[%s] Fallback: display() failed, trying with force_clear=True", plugin_id)
|
||||
logger.debug("[%s] Fallback: display() failed, trying with force_clear=True", plugin_id)
|
||||
plugin.display(force_clear=True)
|
||||
|
||||
# Capture the result
|
||||
captured = self.display_manager.image.copy()
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Fallback: captured frame %dx%d, mode=%s",
|
||||
plugin_id, captured.width, captured.height, captured.mode
|
||||
)
|
||||
|
||||
# Check if captured image has content (not all black)
|
||||
is_blank, bright_ratio = self._is_blank_image(captured, return_ratio=True)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Fallback: brightness check - %.3f%% bright pixels (threshold=0.5%%)",
|
||||
plugin_id, bright_ratio * 100
|
||||
)
|
||||
|
||||
if is_blank:
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Fallback: first capture blank, retrying with force_clear",
|
||||
plugin_id
|
||||
)
|
||||
@@ -1142,7 +1142,7 @@ class PluginAdapter:
|
||||
captured = self.display_manager.image.copy()
|
||||
|
||||
is_blank, bright_ratio = self._is_blank_image(captured, return_ratio=True)
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Fallback: retry brightness - %.3f%% bright pixels",
|
||||
plugin_id, bright_ratio * 100
|
||||
)
|
||||
@@ -1159,7 +1159,7 @@ class PluginAdapter:
|
||||
if captured.mode != 'RGB':
|
||||
captured = captured.convert('RGB')
|
||||
|
||||
logger.info(
|
||||
logger.debug(
|
||||
"[%s] Fallback: SUCCESS - captured %dx%d",
|
||||
plugin_id, captured.width, captured.height
|
||||
)
|
||||
|
||||
@@ -29,18 +29,16 @@ def success_response(
|
||||
Flask jsonify response
|
||||
"""
|
||||
response_data = create_success_response(data, message, metadata)
|
||||
|
||||
# Add request metadata if available
|
||||
if metadata is None:
|
||||
metadata = {}
|
||||
|
||||
# Add timing if request start time is available
|
||||
|
||||
# Timing is merged into whatever the caller passed, without inventing a
|
||||
# metadata block for responses that have neither.
|
||||
enriched = dict(metadata) if metadata is not None else {}
|
||||
if hasattr(request, 'start_time'):
|
||||
metadata['response_time_ms'] = int((time.time() - request.start_time) * 1000)
|
||||
|
||||
if metadata:
|
||||
response_data['metadata'] = metadata
|
||||
|
||||
enriched['response_time_ms'] = int((time.time() - request.start_time) * 1000)
|
||||
|
||||
if metadata is not None or enriched:
|
||||
response_data['metadata'] = enriched
|
||||
|
||||
return jsonify(response_data)
|
||||
|
||||
|
||||
|
||||
@@ -161,14 +161,17 @@ def create_success_response(
|
||||
"status": "success"
|
||||
}
|
||||
|
||||
# All three use `is not None` rather than truthiness: "" and {} are
|
||||
# values a caller chose to send, and dropping them silently would make
|
||||
# the response shape depend on the data.
|
||||
if data is not None:
|
||||
response["data"] = data
|
||||
|
||||
if message:
|
||||
|
||||
if message is not None:
|
||||
response["message"] = message
|
||||
|
||||
if metadata:
|
||||
|
||||
if metadata is not None:
|
||||
response["metadata"] = metadata
|
||||
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@@ -89,7 +89,11 @@ class WebInterfaceError:
|
||||
self.category = category or self._infer_category(error_code)
|
||||
self.details = details
|
||||
self.context = context or {}
|
||||
self.suggested_fixes = suggested_fixes or self._get_default_suggestions(error_code)
|
||||
# `is None`, not truthiness: an explicit [] means "this caller has
|
||||
# no suggestions to offer", which the default list would override.
|
||||
self.suggested_fixes = (
|
||||
suggested_fixes if suggested_fixes is not None
|
||||
else self._get_default_suggestions(error_code))
|
||||
self.original_error = original_error
|
||||
|
||||
def _infer_category(self, error_code: ErrorCode) -> ErrorCategory:
|
||||
|
||||
@@ -5,7 +5,7 @@ Provides functions for identifying, masking, separating, and filtering
|
||||
secret fields in plugin configurations based on JSON Schema x-secret markers.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Set, Tuple
|
||||
from typing import Any, Dict, Optional, Set, Tuple
|
||||
|
||||
|
||||
def find_secret_fields(properties: Dict[str, Any], prefix: str = '') -> Set[str]:
|
||||
@@ -143,6 +143,12 @@ def mask_secret_fields(config: Dict[str, Any], schema_properties: Dict[str, Any]
|
||||
return result
|
||||
|
||||
|
||||
#: What a masked secret looks like on the wire. Named because the write path
|
||||
#: has to recognise it coming back: a client that renders the mask and posts
|
||||
#: it unchanged must not store the mask as if it were the secret.
|
||||
SECRET_MASK = '\u2022' * 8
|
||||
|
||||
|
||||
def mask_all_secret_values(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Blanket-mask every non-empty value in a secrets config dict.
|
||||
|
||||
@@ -156,15 +162,25 @@ def mask_all_secret_values(config: Dict[str, Any]) -> Dict[str, Any]:
|
||||
Returns:
|
||||
A copy with all real values replaced by ``'••••••••'``.
|
||||
"""
|
||||
masked: Dict[str, Any] = {}
|
||||
for k, v in config.items():
|
||||
if isinstance(v, dict):
|
||||
masked[k] = mask_all_secret_values(v)
|
||||
elif v not in (None, '') and not (isinstance(v, str) and v.startswith('YOUR_')):
|
||||
masked[k] = '••••••••'
|
||||
else:
|
||||
masked[k] = v
|
||||
return masked
|
||||
return {k: _mask_value(v) for k, v in config.items()}
|
||||
|
||||
|
||||
def _mask_value(value: Any) -> Any:
|
||||
"""Mask one value, recursing through dicts and lists.
|
||||
|
||||
A list used to be masked as though it were a scalar, so
|
||||
``accounts: [{"name": "a", "token": "..."}]`` came back as a single
|
||||
``'••••••••'``. Nothing leaked, but the caller could no longer see how
|
||||
many entries there were or any of their non-secret fields, and the raw
|
||||
editor was shown a string where the file holds an array.
|
||||
"""
|
||||
if isinstance(value, dict):
|
||||
return {k: _mask_value(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_mask_value(item) for item in value]
|
||||
if value in (None, '') or (isinstance(value, str) and value.startswith('YOUR_')):
|
||||
return value
|
||||
return SECRET_MASK
|
||||
|
||||
|
||||
def remove_empty_secrets(secrets: Dict[str, Any]) -> Dict[str, Any]:
|
||||
@@ -186,6 +202,133 @@ def remove_empty_secrets(secrets: Dict[str, Any]) -> Dict[str, Any]:
|
||||
nested = remove_empty_secrets(v)
|
||||
if nested:
|
||||
result[k] = nested
|
||||
elif isinstance(v, list):
|
||||
# Lists used to fall through to the scalar branch below and be
|
||||
# kept verbatim, blanks and all. Because lists merge by
|
||||
# *replacement*, saving any unrelated setting then wrote
|
||||
# [{"token": ""}, ...] straight over the stored list and
|
||||
# destroyed every credential in it.
|
||||
pruned = _prune_secret_list(v)
|
||||
if pruned is not None:
|
||||
result[k] = pruned
|
||||
elif v is not None and not (isinstance(v, str) and v.strip() == ''):
|
||||
result[k] = v
|
||||
return result
|
||||
|
||||
|
||||
def _prune_secret_list(items: list) -> Optional[list]:
|
||||
"""Strip blanks from inside a list of secrets, preserving every index.
|
||||
|
||||
The rest of the system treats a secrets list as *parallel* to the regular
|
||||
one -- ``sec[i]`` holds the secret fields of item ``i``, and ``{}`` means
|
||||
"item i has none" (see ConfigManager._strip_secrets_recursive). So an
|
||||
emptied dict item stays ``{}``: putting ``None`` there makes that list stop
|
||||
looking parallel, and the stripper then drops the whole key from the main
|
||||
config, taking the non-secret fields with it.
|
||||
|
||||
A blank *scalar* becomes ``None``, meaning "no update at this index" --
|
||||
:func:`merge_secrets` substitutes whatever is stored there. Returns
|
||||
``None`` when nothing in the list carries a real value, so the caller drops
|
||||
the key and leaves the stored list untouched.
|
||||
"""
|
||||
pruned: list = []
|
||||
has_real_value = False
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
kept = remove_empty_secrets(item)
|
||||
pruned.append(kept)
|
||||
has_real_value = has_real_value or bool(kept)
|
||||
elif isinstance(item, list):
|
||||
sub = _prune_secret_list(item)
|
||||
pruned.append(sub if sub is not None else [])
|
||||
has_real_value = has_real_value or sub is not None
|
||||
elif item is not None and not (isinstance(item, str) and item.strip() == ''):
|
||||
pruned.append(item)
|
||||
has_real_value = True
|
||||
else:
|
||||
pruned.append(None)
|
||||
return pruned if has_real_value else None
|
||||
|
||||
|
||||
def merge_secrets(stored: Any, incoming: Any) -> Any:
|
||||
"""Merge submitted secrets over stored ones, element-wise inside lists.
|
||||
|
||||
``deep_merge`` replaces a list wholesale. For secrets that is destructive:
|
||||
an incoming list that carries a real value for one entry and ``None`` for
|
||||
the rest would drop the stored credentials of every other entry. Here a
|
||||
list merges by index, and ``None`` means "keep what is stored".
|
||||
|
||||
Entries are matched by *position*, which is what the config form gives us
|
||||
-- there is no schema-declared identity to key on, and it is the same
|
||||
contract ConfigManager._strip_secrets_recursive already relies on. The
|
||||
incoming list's length wins, so deleting an item deletes its secrets;
|
||||
an item the client left blank keeps whatever is stored at that index.
|
||||
"""
|
||||
if isinstance(stored, dict) and isinstance(incoming, dict):
|
||||
merged = dict(stored)
|
||||
for key, value in incoming.items():
|
||||
merged[key] = (merge_secrets(stored[key], value)
|
||||
if key in stored else value)
|
||||
return merged
|
||||
if isinstance(stored, list) and isinstance(incoming, list):
|
||||
# The incoming list sets the length -- the regular config's list is
|
||||
# authoritative about how many items exist, and this one runs parallel
|
||||
# to it. Removing an entry must therefore remove its secrets too.
|
||||
merged_list = []
|
||||
for index, item in enumerate(incoming):
|
||||
stored_item = stored[index] if index < len(stored) else None
|
||||
merged_list.append(stored_item if item is None
|
||||
else merge_secrets(stored_item, item))
|
||||
return merged_list
|
||||
if incoming is None:
|
||||
return stored
|
||||
return incoming
|
||||
|
||||
|
||||
def strip_masked_values(secrets: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Remove values a client echoed back rather than changed.
|
||||
|
||||
The counterpart to :func:`mask_all_secret_values`. A client that GETs the
|
||||
masked secrets, edits one field and POSTs the whole object back is sending
|
||||
``SECRET_MASK`` for every field it did not touch. Storing those would
|
||||
replace each untouched credential with eight bullet characters.
|
||||
|
||||
Drops the mask and, like :func:`remove_empty_secrets`, blank values -- so
|
||||
the caller can merge the result onto what is already stored and have
|
||||
"unchanged" mean unchanged. Empty nested dicts are pruned.
|
||||
"""
|
||||
result: Dict[str, Any] = {}
|
||||
for k, v in secrets.items():
|
||||
if isinstance(v, dict):
|
||||
nested = strip_masked_values(v)
|
||||
if nested:
|
||||
result[k] = nested
|
||||
elif isinstance(v, list):
|
||||
# A list is merged by replacement, not element by element -- there
|
||||
# is no identity to match entries on -- so a list that still holds
|
||||
# a mask cannot be merged safely: keeping it would store bullets,
|
||||
# and keeping the submitted entries alone would drop whichever the
|
||||
# client did not send back. Dropping the key leaves the stored
|
||||
# list untouched, which is what an untouched list should do.
|
||||
#
|
||||
# The consequence, deliberately: editing one secret inside a list
|
||||
# through this endpoint requires sending real values for all of
|
||||
# them. Sending some masks leaves the whole list as it was.
|
||||
if not _contains_mask(v):
|
||||
result[k] = v
|
||||
elif v is None:
|
||||
continue
|
||||
elif isinstance(v, str) and (v.strip() == '' or v == SECRET_MASK):
|
||||
continue
|
||||
else:
|
||||
result[k] = v
|
||||
return result
|
||||
|
||||
|
||||
def _contains_mask(value: Any) -> bool:
|
||||
"""True when a mask sentinel survives anywhere inside ``value``."""
|
||||
if isinstance(value, dict):
|
||||
return any(_contains_mask(v) for v in value.values())
|
||||
if isinstance(value, list):
|
||||
return any(_contains_mask(item) for item in value)
|
||||
return value == SECRET_MASK
|
||||
|
||||
@@ -43,10 +43,15 @@ def validate_image_url(url: str) -> Tuple[bool, Optional[str]]:
|
||||
if any(handler in url_lower for handler in ['onerror=', 'onload=', 'onclick=']):
|
||||
return False, "Event handlers not allowed in URLs"
|
||||
|
||||
# Reject directory traversal anywhere, not only in relative paths:
|
||||
# http://host/../secret is as much a traversal attempt as /../secret.
|
||||
if '..' in url:
|
||||
return False, "Invalid path: directory traversal not allowed"
|
||||
|
||||
# Allow relative paths starting with /
|
||||
if url.startswith('/'):
|
||||
# Validate it's a safe relative path (no directory traversal)
|
||||
if '..' in url or url.startswith('//'):
|
||||
# // would be a protocol-relative URL, not a local path
|
||||
if url.startswith('//'):
|
||||
return False, "Invalid relative path"
|
||||
return True, None
|
||||
|
||||
@@ -104,10 +109,11 @@ def validate_file_upload(filename: str, max_size_mb: int = 10,
|
||||
if '..' in filename or '/' in filename or '\\' in filename:
|
||||
return False, "Filename contains invalid characters"
|
||||
|
||||
# Check extension if specified
|
||||
# Check extension if specified. Both sides are lowercased: the caller's
|
||||
# list is as likely to hold '.TTF' as the filename is.
|
||||
if allowed_extensions:
|
||||
file_ext = Path(filename).suffix.lower()
|
||||
if file_ext not in allowed_extensions:
|
||||
if file_ext not in [ext.lower() for ext in allowed_extensions]:
|
||||
return False, f"File extension must be one of: {', '.join(allowed_extensions)}"
|
||||
|
||||
return True, None
|
||||
@@ -147,7 +153,8 @@ def validate_numeric_range(value: float, min_val: Optional[float] = None,
|
||||
Returns:
|
||||
Tuple of (is_valid, error_message)
|
||||
"""
|
||||
if not isinstance(value, (int, float)):
|
||||
# bool is an int subclass, so True would otherwise validate as 1.
|
||||
if not isinstance(value, (int, float)) or isinstance(value, bool):
|
||||
return False, "Value must be a number"
|
||||
|
||||
if min_val is not None and value < min_val:
|
||||
@@ -183,11 +190,19 @@ def validate_string_length(text: str, min_length: Optional[int] = None,
|
||||
|
||||
def sanitize_plugin_config(config: dict) -> dict:
|
||||
"""
|
||||
Sanitize plugin configuration input to prevent injection.
|
||||
|
||||
Restrict a plugin config to safe key names and value types.
|
||||
|
||||
Drops keys that are not plain identifiers and values that are not
|
||||
JSON-ish scalars, lists, or dicts, recursing into the latter two.
|
||||
|
||||
String values are returned **unescaped**: output escaping is the
|
||||
template layer's job, and escaping here would store the escaped form
|
||||
in config.json. Do not read this function as XSS protection for
|
||||
rendered output.
|
||||
|
||||
Args:
|
||||
config: Configuration dictionary
|
||||
|
||||
|
||||
Returns:
|
||||
Sanitized configuration dictionary
|
||||
"""
|
||||
|
||||
Regular → Executable
Regular → Executable
@@ -10,8 +10,8 @@ WorkingDirectory=__PROJECT_ROOT_DIR__
|
||||
ExecStart=/usr/bin/python3 __PROJECT_ROOT_DIR__/scripts/utils/wifi_monitor_daemon.py --interval 30
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
StandardOutput=syslog
|
||||
StandardError=syslog
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=ledmatrix-wifi-monitor
|
||||
|
||||
[Install]
|
||||
|
||||
@@ -8,9 +8,38 @@ Type=simple
|
||||
User=root
|
||||
WorkingDirectory=__PROJECT_ROOT_DIR__
|
||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||
# glibc gives each allocating thread its own malloc arena, up to 8 x CPU count,
|
||||
# and an arena that has grown is never handed back to the OS. This process runs
|
||||
# 9 threads on a 3-core Pi, so the ceiling is 24 arenas -- and a rig measured at
|
||||
# 1030 MB resident held 23 large anonymous mappings on 64 MB-aligned addresses,
|
||||
# 920 MB of them, while the live data it was actually holding (widest scroll
|
||||
# strip seen: 35,746 x 64) accounts for roughly 15 MB. That gap is arena bloat,
|
||||
# not leaked objects: RSS was flat across repeated sampling, not climbing.
|
||||
#
|
||||
# Capping the arenas trades a little allocator concurrency for a large amount of
|
||||
# resident memory on a device that has neither to spare. 2 is the usual value;
|
||||
# raise it if frame times regress.
|
||||
Environment=MALLOC_ARENA_MAX=2
|
||||
ExecStart=/usr/bin/python3 __PROJECT_ROOT_DIR__/run.py
|
||||
Restart=on-failure
|
||||
# Restart=always, not on-failure: run.py exiting 0 (a clean shutdown path taken
|
||||
# for a reason that no longer applies, e.g. a config reload) would otherwise leave
|
||||
# the service stopped and the panel dark indefinitely, with systemd considering
|
||||
# that a successful outcome and never bringing it back.
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
# Memory ceiling as a share of physical RAM, so one unit file suits a 512 MB
|
||||
# Pi Zero 2 W and an 8 GB Pi 5 alike. This is a backstop, not a tuning knob: it
|
||||
# turns "the board runs out of memory, stops being able to fork, and takes sshd
|
||||
# and the panel down together until someone pulls the plug" into "this one
|
||||
# service restarts".
|
||||
#
|
||||
# NOTE: Raspberry Pi firmware boots the kernel with cgroup_disable=memory, and
|
||||
# systemd accepts this setting and then silently ignores it. Verify with:
|
||||
# grep memory /sys/fs/cgroup/cgroup.controllers
|
||||
# If that prints nothing, add "cgroup_enable=memory cgroup_memory=1" to
|
||||
# /boot/firmware/cmdline.txt (all on line 1) and reboot. first_time_install.sh
|
||||
# does this for you.
|
||||
MemoryMax=85%
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=ledmatrix
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""
|
||||
Shared scaffolding for api_v3 blueprint tests.
|
||||
|
||||
Not a test module (the leading underscore keeps pytest from collecting
|
||||
it). It is the pytest-fixture equivalent of ``_make_client()`` in
|
||||
test_uninstall_and_reconcile_endpoint.py, which is unittest-style and
|
||||
requires ``self.addCleanup``.
|
||||
|
||||
The api_v3 blueprint keeps its managers as attributes on a module-level
|
||||
singleton, not in Flask app state, so replacing them with mocks leaks
|
||||
into every later test that imports api_v3 unless the originals are put
|
||||
back. ``api_v3_client`` snapshots and restores them around each test.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
|
||||
# Every manager attribute the blueprint reads. Anything missing here keeps
|
||||
# whatever a previously-run test left on the singleton.
|
||||
API_V3_MANAGER_ATTRS = (
|
||||
'config_manager', 'plugin_manager', 'plugin_store_manager',
|
||||
'plugin_state_manager', 'saved_repositories_manager', 'schema_manager',
|
||||
'operation_queue', 'operation_history', 'cache_manager',
|
||||
)
|
||||
|
||||
_SENTINEL = object()
|
||||
|
||||
|
||||
def build_app(blueprint):
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
app.config['SECRET_KEY'] = 'test'
|
||||
app.register_blueprint(blueprint, url_prefix='/api/v3')
|
||||
return app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_v3_module():
|
||||
"""The api_v3 module with every manager replaced by a MagicMock.
|
||||
|
||||
Restores the original attributes afterwards. Tests point individual
|
||||
managers at real objects (a ConfigManager over tmp_path, say) or set
|
||||
them to None to exercise the not-initialized branches.
|
||||
"""
|
||||
from web_interface.blueprints import api_v3 as module
|
||||
|
||||
originals = {
|
||||
name: getattr(module.api_v3, name, _SENTINEL)
|
||||
for name in API_V3_MANAGER_ATTRS
|
||||
}
|
||||
for name in API_V3_MANAGER_ATTRS:
|
||||
setattr(module.api_v3, name, MagicMock())
|
||||
# Default to the direct path; queue tests opt in explicitly.
|
||||
module.api_v3.operation_queue = None
|
||||
|
||||
yield module
|
||||
|
||||
for name, original in originals.items():
|
||||
if original is _SENTINEL:
|
||||
if hasattr(module.api_v3, name):
|
||||
try:
|
||||
delattr(module.api_v3, name)
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
setattr(module.api_v3, name, original)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_v3_client(api_v3_module):
|
||||
"""Flask test client wired to the mocked blueprint."""
|
||||
return build_app(api_v3_module.api_v3).test_client()
|
||||
@@ -0,0 +1,226 @@
|
||||
"""
|
||||
Endpoint tests for POST /plugins/calendar/upload-credentials.
|
||||
|
||||
The endpoint takes an uploaded Google OAuth credentials file, writes it
|
||||
into the calendar plugin's directory as credentials.json at mode 0600, and
|
||||
copies any previous file aside first. It had no tests.
|
||||
|
||||
Regression coverage for two fixed bugs:
|
||||
- The OAuth-shape check sat inside `except Exception: pass`, so a valid
|
||||
JSON document that is not an object — a bare `42`, a list, a string —
|
||||
raised TypeError on the membership test, was swallowed, and got saved
|
||||
as credentials.json anyway.
|
||||
- Each overwrite created a timestamped backup and nothing ever removed
|
||||
them, so every re-upload left another complete copy of the user's OAuth
|
||||
client credentials in the plugin directory, indefinitely.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402
|
||||
|
||||
URL = "/api/v3/plugins/calendar/upload-credentials"
|
||||
|
||||
VALID_CREDENTIALS = {
|
||||
"installed": {
|
||||
"client_id": "abc.apps.googleusercontent.com",
|
||||
"client_secret": "shh",
|
||||
"redirect_uris": ["http://localhost"],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plugin_dir(tmp_path, api_v3_module):
|
||||
directory = tmp_path / "plugins" / "calendar"
|
||||
directory.mkdir(parents=True)
|
||||
api_v3_module.api_v3.plugin_manager.get_plugin_directory.return_value = str(directory)
|
||||
return directory
|
||||
|
||||
|
||||
def upload(client, content, filename="credentials.json"):
|
||||
# bytes are sent verbatim (to exercise malformed input); anything else
|
||||
# is serialized, so None becomes the JSON literal null rather than an
|
||||
# empty body.
|
||||
payload = content if isinstance(content, bytes) else json.dumps(content).encode()
|
||||
return client.post(
|
||||
URL,
|
||||
data={"file": (io.BytesIO(payload), filename)},
|
||||
content_type="multipart/form-data",
|
||||
)
|
||||
|
||||
|
||||
def backups(plugin_dir):
|
||||
return sorted(plugin_dir.glob("credentials.json.backup.*"))
|
||||
|
||||
|
||||
class TestRequestValidation:
|
||||
def test_no_file_part_is_a_400(self, api_v3_client, plugin_dir):
|
||||
response = api_v3_client.post(URL, data={}, content_type="multipart/form-data")
|
||||
assert response.status_code == 400
|
||||
assert "No file provided" in response.get_json()["message"]
|
||||
|
||||
def test_empty_filename_is_a_400(self, api_v3_client, plugin_dir):
|
||||
response = upload(api_v3_client, VALID_CREDENTIALS, filename="")
|
||||
assert response.status_code == 400
|
||||
|
||||
@pytest.mark.parametrize("filename", ["creds.txt", "creds.pem", "creds"])
|
||||
def test_non_json_extension_is_a_400(self, api_v3_client, plugin_dir, filename):
|
||||
response = upload(api_v3_client, VALID_CREDENTIALS, filename=filename)
|
||||
assert response.status_code == 400
|
||||
assert "JSON file" in response.get_json()["message"]
|
||||
|
||||
def test_uppercase_json_extension_accepted(self, api_v3_client, plugin_dir):
|
||||
assert upload(api_v3_client, VALID_CREDENTIALS,
|
||||
filename="CREDENTIALS.JSON").status_code == 200
|
||||
|
||||
def test_oversized_file_is_a_400(self, api_v3_client, plugin_dir):
|
||||
response = upload(api_v3_client, b"x" * (1024 * 1024 + 1))
|
||||
assert response.status_code == 400
|
||||
assert "1MB" in response.get_json()["message"]
|
||||
assert not (plugin_dir / "credentials.json").exists()
|
||||
|
||||
def test_invalid_json_is_a_400(self, api_v3_client, plugin_dir):
|
||||
response = upload(api_v3_client, b"{not json")
|
||||
assert response.status_code == 400
|
||||
assert "not valid JSON" in response.get_json()["message"]
|
||||
assert not (plugin_dir / "credentials.json").exists()
|
||||
|
||||
def test_missing_plugin_directory_is_a_404(self, api_v3_client, api_v3_module, tmp_path):
|
||||
api_v3_module.api_v3.plugin_manager.get_plugin_directory.return_value = str(
|
||||
tmp_path / "not-installed")
|
||||
assert upload(api_v3_client, VALID_CREDENTIALS).status_code == 404
|
||||
|
||||
|
||||
class TestOAuthShapeValidation:
|
||||
def test_installed_key_accepted(self, api_v3_client, plugin_dir):
|
||||
assert upload(api_v3_client, VALID_CREDENTIALS).status_code == 200
|
||||
|
||||
def test_web_key_accepted(self, api_v3_client, plugin_dir):
|
||||
assert upload(api_v3_client, {"web": {"client_id": "x"}}).status_code == 200
|
||||
|
||||
def test_object_without_oauth_keys_is_a_400(self, api_v3_client, plugin_dir):
|
||||
response = upload(api_v3_client, {"something": "else"})
|
||||
assert response.status_code == 400
|
||||
assert "valid Google OAuth" in response.get_json()["message"]
|
||||
assert not (plugin_dir / "credentials.json").exists()
|
||||
|
||||
@pytest.mark.parametrize("content", [42, "a string", [1, 2, 3], True, None])
|
||||
def test_valid_json_that_is_not_an_object_is_rejected(
|
||||
self, api_v3_client, plugin_dir, content):
|
||||
# Regression: `'installed' not in 42` raises TypeError, which the
|
||||
# bare `except Exception: pass` swallowed — the file was then saved
|
||||
# as credentials.json despite being unusable as credentials.
|
||||
response = upload(api_v3_client, content)
|
||||
assert response.status_code == 400
|
||||
assert "valid Google OAuth" in response.get_json()["message"]
|
||||
assert not (plugin_dir / "credentials.json").exists()
|
||||
|
||||
|
||||
class TestSaving:
|
||||
def test_file_written_with_contents_intact(self, api_v3_client, plugin_dir):
|
||||
response = upload(api_v3_client, VALID_CREDENTIALS)
|
||||
assert response.status_code == 200
|
||||
saved = json.loads((plugin_dir / "credentials.json").read_text())
|
||||
assert saved == VALID_CREDENTIALS
|
||||
|
||||
def test_response_reports_the_path(self, api_v3_client, plugin_dir):
|
||||
body = upload(api_v3_client, VALID_CREDENTIALS).get_json()
|
||||
assert body["path"].endswith("credentials.json")
|
||||
|
||||
def test_permissions_are_owner_only(self, api_v3_client, plugin_dir):
|
||||
upload(api_v3_client, VALID_CREDENTIALS)
|
||||
mode = stat.S_IMODE((plugin_dir / "credentials.json").stat().st_mode)
|
||||
assert mode == 0o600
|
||||
|
||||
def test_first_upload_creates_no_backup(self, api_v3_client, plugin_dir):
|
||||
upload(api_v3_client, VALID_CREDENTIALS)
|
||||
assert backups(plugin_dir) == []
|
||||
|
||||
def test_overwrite_backs_up_the_previous_file(self, api_v3_client, plugin_dir):
|
||||
(plugin_dir / "credentials.json").write_text(json.dumps({"installed": {"old": 1}}))
|
||||
upload(api_v3_client, VALID_CREDENTIALS)
|
||||
assert len(backups(plugin_dir)) == 1
|
||||
assert json.loads(backups(plugin_dir)[0].read_text()) == {"installed": {"old": 1}}
|
||||
assert json.loads((plugin_dir / "credentials.json").read_text()) == VALID_CREDENTIALS
|
||||
|
||||
|
||||
class TestBackupPruning:
|
||||
def _seed(self, plugin_dir, count):
|
||||
"""Create `count` backups with distinct, increasing mtimes."""
|
||||
now = int(time.time())
|
||||
for i in range(count):
|
||||
path = plugin_dir / f"credentials.json.backup.{now - (count - i) * 10}"
|
||||
path.write_text(json.dumps({"installed": {"gen": i}}))
|
||||
os.utime(path, (now - (count - i) * 10, now - (count - i) * 10))
|
||||
|
||||
def test_old_backups_are_pruned(self, api_v3_client, plugin_dir):
|
||||
# Regression: nothing ever removed these, so a plugin directory
|
||||
# accumulated one full copy of the user's OAuth credentials per
|
||||
# re-upload, forever.
|
||||
(plugin_dir / "credentials.json").write_text(json.dumps({"installed": {"cur": 1}}))
|
||||
self._seed(plugin_dir, 7)
|
||||
assert len(backups(plugin_dir)) == 7
|
||||
|
||||
upload(api_v3_client, VALID_CREDENTIALS)
|
||||
assert len(backups(plugin_dir)) == 5
|
||||
|
||||
def test_the_newest_backups_are_the_ones_kept(self, api_v3_client, plugin_dir):
|
||||
(plugin_dir / "credentials.json").write_text(json.dumps({"installed": {"cur": 1}}))
|
||||
self._seed(plugin_dir, 7)
|
||||
|
||||
upload(api_v3_client, VALID_CREDENTIALS)
|
||||
remaining = backups(plugin_dir)
|
||||
# The just-created backup (of "cur") plus the four newest seeds.
|
||||
contents = [json.loads(p.read_text()) for p in remaining]
|
||||
assert {"installed": {"cur": 1}} in contents
|
||||
assert {"installed": {"gen": 0}} not in contents # oldest seed gone
|
||||
|
||||
def test_under_the_limit_nothing_is_removed(self, api_v3_client, plugin_dir):
|
||||
(plugin_dir / "credentials.json").write_text(json.dumps({"installed": {"cur": 1}}))
|
||||
self._seed(plugin_dir, 2)
|
||||
upload(api_v3_client, VALID_CREDENTIALS)
|
||||
assert len(backups(plugin_dir)) == 3 # 2 seeded + 1 new
|
||||
|
||||
def test_repeated_uploads_stay_bounded(
|
||||
self, api_v3_client, plugin_dir, api_v3_module, monkeypatch):
|
||||
# The backup filename carries int(time.time()), so uploads inside
|
||||
# the same second all write the same name and overwrite each other.
|
||||
# Advance a fake clock a second per round — otherwise this never
|
||||
# reaches six backups and the bound holds for the wrong reason.
|
||||
clock = {"now": int(time.time())}
|
||||
monkeypatch.setattr(
|
||||
api_v3_module, "time", SimpleNamespace(time=lambda: clock["now"]))
|
||||
for i in range(10):
|
||||
clock["now"] += 1
|
||||
upload(api_v3_client, {"installed": {"round": i}})
|
||||
os.utime(plugin_dir / "credentials.json",
|
||||
(clock["now"], clock["now"]))
|
||||
remaining = backups(plugin_dir)
|
||||
assert len(remaining) == 5
|
||||
# And they are the five most recent rounds, not an arbitrary five.
|
||||
kept = sorted(int(p.name.rsplit(".", 1)[1]) for p in remaining)
|
||||
assert kept == [clock["now"] - 4 + i for i in range(5)]
|
||||
|
||||
def test_unremovable_backup_does_not_fail_the_upload(
|
||||
self, api_v3_client, plugin_dir, monkeypatch):
|
||||
(plugin_dir / "credentials.json").write_text(json.dumps({"installed": {"cur": 1}}))
|
||||
self._seed(plugin_dir, 7)
|
||||
|
||||
def refuse(self):
|
||||
raise OSError("read-only filesystem")
|
||||
monkeypatch.setattr(Path, "unlink", refuse)
|
||||
|
||||
# Pruning is housekeeping; failing it must not lose the upload.
|
||||
assert upload(api_v3_client, VALID_CREDENTIALS).status_code == 200
|
||||
@@ -0,0 +1,302 @@
|
||||
"""
|
||||
Endpoint tests for /plugins/authenticate/spotify and .../ytm.
|
||||
|
||||
The Spotify step-2 handler writes a Python wrapper script to a temp file
|
||||
with the user's redirect URL embedded in it, then runs that file through
|
||||
subprocess. That is the most dangerous shape in the blueprint and had no
|
||||
tests: the URL is user input reaching generated source code.
|
||||
|
||||
The two endpoints are NOT symmetrical, despite the matching names. Only
|
||||
Spotify has a two-step flow, a wrapper script, and a redirect_url; YTM
|
||||
just runs its script directly.
|
||||
|
||||
Regression coverage for one fixed bug: the wrapper file was unlinked in
|
||||
the success/failure branch and again in the TimeoutExpired handler, so
|
||||
any other failure from subprocess.run — the interpreter missing, a fork
|
||||
failure, an interrupted call — left a temp file containing the user's
|
||||
redirect URL behind.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plugin_dir(tmp_path, api_v3_module):
|
||||
"""A plugin directory containing both auth scripts."""
|
||||
directory = tmp_path / "plugins" / "ledmatrix-music"
|
||||
directory.mkdir(parents=True)
|
||||
(directory / "authenticate_spotify.py").write_text("print('spotify')\n")
|
||||
(directory / "authenticate_ytm.py").write_text("print('ytm')\n")
|
||||
api_v3_module.api_v3.plugin_manager.get_plugin_directory.return_value = str(directory)
|
||||
return directory
|
||||
|
||||
|
||||
def completed(returncode=0, stdout="ok", stderr=""):
|
||||
return subprocess.CompletedProcess(
|
||||
args=["python3"], returncode=returncode, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
class TestSpotifyPreconditions:
|
||||
URL = "/api/v3/plugins/authenticate/spotify"
|
||||
|
||||
def test_missing_plugin_directory_is_404(self, api_v3_client, api_v3_module, tmp_path):
|
||||
api_v3_module.api_v3.plugin_manager.get_plugin_directory.return_value = str(
|
||||
tmp_path / "not-installed")
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code == 404
|
||||
assert response.get_json()["message"] == "Plugin not found"
|
||||
|
||||
def test_none_plugin_directory_is_404(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_manager.get_plugin_directory.return_value = None
|
||||
assert api_v3_client.post(self.URL, json={}).status_code == 404
|
||||
|
||||
def test_missing_auth_script_is_404(self, api_v3_client, plugin_dir):
|
||||
(plugin_dir / "authenticate_spotify.py").unlink()
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code == 404
|
||||
assert "script not found" in response.get_json()["message"]
|
||||
|
||||
|
||||
class TestSpotifyStepTwo:
|
||||
"""redirect_url present — the wrapper-script path."""
|
||||
|
||||
URL = "/api/v3/plugins/authenticate/spotify"
|
||||
|
||||
def test_success(self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run", return_value=completed(0, "done")):
|
||||
response = api_v3_client.post(self.URL, json={"redirect_url": "http://cb/?code=x"})
|
||||
assert response.status_code == 200
|
||||
body = response.get_json()
|
||||
assert body["status"] == "success"
|
||||
assert body["output"] == "done"
|
||||
|
||||
def test_script_failure_is_a_400_with_combined_output(self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run", return_value=completed(1, "out", "err")):
|
||||
response = api_v3_client.post(self.URL, json={"redirect_url": "http://cb/?code=x"})
|
||||
assert response.status_code == 400
|
||||
assert response.get_json()["output"] == "outerr"
|
||||
|
||||
def test_timeout_is_a_408(self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run",
|
||||
side_effect=subprocess.TimeoutExpired("python3", 120)):
|
||||
response = api_v3_client.post(self.URL, json={"redirect_url": "http://cb/?code=x"})
|
||||
assert response.status_code == 408
|
||||
assert "timed out" in response.get_json()["message"]
|
||||
|
||||
def test_runs_a_list_argv_never_a_shell(self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run", return_value=completed()) as run:
|
||||
api_v3_client.post(self.URL, json={"redirect_url": "http://cb/?code=x"})
|
||||
args, kwargs = run.call_args
|
||||
assert isinstance(args[0], list)
|
||||
assert args[0][0] == "python3"
|
||||
assert kwargs.get("shell") in (None, False)
|
||||
|
||||
def test_timeout_is_bounded(self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run", return_value=completed()) as run:
|
||||
api_v3_client.post(self.URL, json={"redirect_url": "http://cb/?code=x"})
|
||||
assert run.call_args.kwargs["timeout"] == 120
|
||||
|
||||
|
||||
class TestSpotifyWrapperCleanup:
|
||||
URL = "/api/v3/plugins/authenticate/spotify"
|
||||
|
||||
def _wrapper_paths_after(self, api_v3_client, run_mock):
|
||||
"""Run the endpoint and return the wrapper path subprocess saw."""
|
||||
seen = {}
|
||||
|
||||
def capture(args, **kwargs):
|
||||
seen["path"] = args[1]
|
||||
return run_mock(args, **kwargs)
|
||||
|
||||
with patch.object(subprocess, "run", side_effect=capture):
|
||||
api_v3_client.post(self.URL, json={"redirect_url": "http://cb/?code=x"})
|
||||
return seen["path"]
|
||||
|
||||
def test_removed_after_success(self, api_v3_client, plugin_dir):
|
||||
path = self._wrapper_paths_after(api_v3_client, lambda *a, **kw: completed())
|
||||
assert not os.path.exists(path)
|
||||
|
||||
def test_removed_after_script_failure(self, api_v3_client, plugin_dir):
|
||||
path = self._wrapper_paths_after(
|
||||
api_v3_client, lambda *a, **kw: completed(1, "out", "err"))
|
||||
assert not os.path.exists(path)
|
||||
|
||||
def test_removed_after_timeout(self, api_v3_client, plugin_dir):
|
||||
def raise_timeout(*a, **kw):
|
||||
raise subprocess.TimeoutExpired("python3", 120)
|
||||
path = self._wrapper_paths_after(api_v3_client, raise_timeout)
|
||||
assert not os.path.exists(path)
|
||||
|
||||
def test_removed_when_subprocess_cannot_start(self, api_v3_client, plugin_dir):
|
||||
# Regression: cleanup lived in the success/failure branch and in the
|
||||
# TimeoutExpired handler only. An OSError from subprocess.run itself
|
||||
# — no interpreter, fork failure — skipped both and left the wrapper,
|
||||
# which contains the user's redirect URL, on disk.
|
||||
def raise_oserror(*a, **kw):
|
||||
raise OSError("[Errno 12] Cannot allocate memory")
|
||||
path = self._wrapper_paths_after(api_v3_client, raise_oserror)
|
||||
assert not os.path.exists(path)
|
||||
|
||||
|
||||
class TestSpotifyRedirectUrlIsNotInjectable:
|
||||
"""The wrapper embeds redirect_url into generated Python source."""
|
||||
|
||||
URL = "/api/v3/plugins/authenticate/spotify"
|
||||
|
||||
ADVERSARIAL = [
|
||||
'''http://cb/?code=x"''',
|
||||
"""http://cb/?code=x'""",
|
||||
'http://cb/?code=x\\',
|
||||
'http://cb/?code=x\nimport os; os.system("id")',
|
||||
'http://cb/?code=x"""\nimport os\n"""',
|
||||
"http://cb/?code=x'''",
|
||||
'http://cb/?code=x\\"\\n',
|
||||
'"; import os; os.system("id"); "',
|
||||
]
|
||||
|
||||
def _wrapper_source(self, api_v3_client, redirect_url):
|
||||
captured = {}
|
||||
|
||||
def capture(args, **kwargs):
|
||||
captured["source"] = Path(args[1]).read_text()
|
||||
return completed()
|
||||
|
||||
with patch.object(subprocess, "run", side_effect=capture):
|
||||
api_v3_client.post(self.URL, json={"redirect_url": redirect_url})
|
||||
return captured["source"]
|
||||
|
||||
@pytest.mark.parametrize("redirect_url", ADVERSARIAL)
|
||||
def test_wrapper_is_still_valid_python(self, api_v3_client, plugin_dir, redirect_url):
|
||||
# If escaping failed, the generated file would not parse at all.
|
||||
source = self._wrapper_source(api_v3_client, redirect_url)
|
||||
ast.parse(source)
|
||||
|
||||
@pytest.mark.parametrize("redirect_url", ADVERSARIAL)
|
||||
def test_url_survives_as_one_string_literal(
|
||||
self, api_v3_client, plugin_dir, redirect_url):
|
||||
# Stronger than "it parses": the URL must still be a single string
|
||||
# assigned to redirect_url, not code that escaped into statements.
|
||||
source = self._wrapper_source(api_v3_client, redirect_url)
|
||||
tree = ast.parse(source)
|
||||
assigned = [
|
||||
node.value.value for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Assign)
|
||||
and isinstance(node.value, ast.Constant)
|
||||
and any(getattr(t, "id", None) == "redirect_url" for t in node.targets)
|
||||
]
|
||||
assert assigned == [redirect_url.strip()]
|
||||
|
||||
def test_injected_call_does_not_become_a_statement(self, api_v3_client, plugin_dir):
|
||||
source = self._wrapper_source(
|
||||
api_v3_client, 'http://cb/\nimport os; os.system("id")')
|
||||
tree = ast.parse(source)
|
||||
imported = {
|
||||
alias.name for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Import) for alias in node.names
|
||||
}
|
||||
# The wrapper legitimately imports sys, subprocess and os; what it
|
||||
# must not gain is a *call* smuggled in through the URL.
|
||||
calls = [
|
||||
node for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "system"
|
||||
]
|
||||
assert calls == []
|
||||
|
||||
|
||||
class TestSpotifyStepOne:
|
||||
"""No redirect_url — the OAuth-URL path, which imports the script."""
|
||||
|
||||
URL = "/api/v3/plugins/authenticate/spotify"
|
||||
|
||||
def test_script_without_credentials_helper_is_an_error(
|
||||
self, api_v3_client, plugin_dir):
|
||||
# The stub script defines neither get_auth_url nor
|
||||
# load_spotify_credentials, so no URL can be produced.
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code in (400, 500)
|
||||
assert response.get_json()["status"] == "error"
|
||||
|
||||
def test_unusable_credentials_do_not_leak_into_the_response(
|
||||
self, api_v3_client, plugin_dir):
|
||||
(plugin_dir / "authenticate_spotify.py").write_text(
|
||||
"def load_spotify_credentials():\n"
|
||||
" return ('id-abc', 'super-secret-value', None)\n"
|
||||
)
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert "super-secret-value" not in response.get_data(as_text=True)
|
||||
|
||||
def test_script_raising_on_import_is_handled(self, api_v3_client, plugin_dir):
|
||||
(plugin_dir / "authenticate_spotify.py").write_text("raise RuntimeError('boom')\n")
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code == 500
|
||||
assert response.get_json()["status"] == "error"
|
||||
|
||||
def test_bodyless_post_reaches_step_one(self, api_v3_client, plugin_dir):
|
||||
# Covered by the silent=True fix: previously a 500 from body parsing.
|
||||
response = api_v3_client.post(self.URL)
|
||||
assert response.status_code in (400, 500)
|
||||
assert response.get_json()["status"] == "error"
|
||||
|
||||
def test_whitespace_redirect_url_is_treated_as_absent(
|
||||
self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run", return_value=completed()) as run:
|
||||
api_v3_client.post(self.URL, json={"redirect_url": " "})
|
||||
# Step 2 never runs, so no wrapper is executed.
|
||||
run.assert_not_called()
|
||||
|
||||
|
||||
class TestYouTubeMusic:
|
||||
"""No wrapper script and no redirect_url — deliberately not symmetric."""
|
||||
|
||||
URL = "/api/v3/plugins/authenticate/ytm"
|
||||
|
||||
def test_missing_plugin_directory_is_404(self, api_v3_client, api_v3_module, tmp_path):
|
||||
api_v3_module.api_v3.plugin_manager.get_plugin_directory.return_value = str(
|
||||
tmp_path / "not-installed")
|
||||
assert api_v3_client.post(self.URL).status_code == 404
|
||||
|
||||
def test_missing_script_is_404(self, api_v3_client, plugin_dir):
|
||||
(plugin_dir / "authenticate_ytm.py").unlink()
|
||||
response = api_v3_client.post(self.URL)
|
||||
assert response.status_code == 404
|
||||
assert "script not found" in response.get_json()["message"]
|
||||
|
||||
def test_success(self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run", return_value=completed(0, "authorized")):
|
||||
response = api_v3_client.post(self.URL)
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["output"] == "authorized"
|
||||
|
||||
def test_failure_is_a_400_with_combined_output(self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run", return_value=completed(1, "out", "err")):
|
||||
response = api_v3_client.post(self.URL)
|
||||
assert response.status_code == 400
|
||||
assert response.get_json()["output"] == "outerr"
|
||||
|
||||
def test_timeout_is_a_408(self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run",
|
||||
side_effect=subprocess.TimeoutExpired("python3", 60)):
|
||||
assert api_v3_client.post(self.URL).status_code == 408
|
||||
|
||||
def test_runs_the_script_directly_without_a_shell(self, api_v3_client, plugin_dir):
|
||||
with patch.object(subprocess, "run", return_value=completed()) as run:
|
||||
api_v3_client.post(self.URL)
|
||||
args, kwargs = run.call_args
|
||||
assert args[0][0] == "python3"
|
||||
assert args[0][1].endswith("authenticate_ytm.py")
|
||||
assert kwargs.get("shell") in (None, False)
|
||||
assert kwargs["timeout"] == 60
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Non-finite JSON numbers must be rejected, not raise.
|
||||
|
||||
json.loads accepts Infinity/-Infinity/NaN by default (they are not valid JSON,
|
||||
but Python's parser emits them) and Flask's get_json passes them straight
|
||||
through. int(float('inf')) raises OverflowError, which is neither ValueError
|
||||
nor TypeError -- so validation blocks that carefully caught those let it
|
||||
through and Flask turned it into a 500.
|
||||
|
||||
The damage was not the status code. /config/dim-schedule answered with
|
||||
CONFIG_SAVE_FAILED and suggested "Check file permissions on config directory"
|
||||
and "Check available disk space" for what was actually an invalid number.
|
||||
|
||||
NaN already returned 400 (int(nan) raises ValueError), which is why this only
|
||||
showed up for the infinities.
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402
|
||||
|
||||
|
||||
#: (route, field) that returned 500 before OverflowError was caught. Both
|
||||
#: infinity signs are exercised: int() raises OverflowError for either, but
|
||||
#: only one of them was in the original report, and a guard that special-cased
|
||||
#: the sign would pass a one-sided test.
|
||||
NON_FINITE_ROUTES = [
|
||||
('/api/v3/config/dim-schedule', 'dim_brightness'),
|
||||
('/api/v3/errors/clear', 'max_age_hours'),
|
||||
('/api/v3/config/main', 'multiplexing'),
|
||||
('/api/v3/config/main', 'row_address_type'),
|
||||
]
|
||||
NON_FINITE_CASES = [
|
||||
(route, '{"%s": %s}' % (field, literal))
|
||||
for route, field in NON_FINITE_ROUTES
|
||||
for literal in ('Infinity', '-Infinity')
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route,body", NON_FINITE_CASES)
|
||||
def test_infinity_is_a_client_error_not_a_server_error(api_v3_client, route, body):
|
||||
"""Exactly 400, not merely "some 4xx".
|
||||
|
||||
Accepting any 4xx would let a 404 pass, so renaming one of these routes
|
||||
would leave the test green while testing nothing -- the failure mode this
|
||||
whole file exists to catch.
|
||||
"""
|
||||
response = api_v3_client.post(route, data=body, content_type='application/json')
|
||||
assert response.status_code == 400, (
|
||||
f"{route} with {body} answered {response.status_code}; expected 400"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route,body", [
|
||||
('/api/v3/config/dim-schedule', '{"dim_brightness": NaN}'),
|
||||
('/api/v3/errors/clear', '{"max_age_hours": NaN}'),
|
||||
])
|
||||
def test_nan_is_also_a_client_error(api_v3_client, route, body):
|
||||
"""int(nan) raises ValueError so this path already worked -- pinned so a
|
||||
refactor that narrows the except tuple cannot quietly break it."""
|
||||
response = api_v3_client.post(route, data=body, content_type='application/json')
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_a_valid_number_is_accepted(api_v3_client, api_v3_module, monkeypatch):
|
||||
"""Prove the widened except did not start swallowing ordinary input.
|
||||
|
||||
Asserting "not a 400" would not show that: the mocked save path fails for
|
||||
any input, so the assertion would hold even if validation had rejected the
|
||||
value. Give load_config a real dict and stub the atomic save, and the
|
||||
endpoint reaches its success response -- which only happens if 30 passed
|
||||
validation.
|
||||
"""
|
||||
api_v3_module.api_v3.config_manager.load_config.return_value = {}
|
||||
monkeypatch.setattr(api_v3_module, '_save_config_atomic',
|
||||
lambda *a, **k: (True, ''))
|
||||
response = api_v3_client.post(
|
||||
'/api/v3/config/dim-schedule',
|
||||
data='{"dim_brightness": 30}',
|
||||
content_type='application/json',
|
||||
)
|
||||
assert response.status_code == 200, response.get_data(as_text=True)[:200]
|
||||
@@ -0,0 +1,136 @@
|
||||
"""
|
||||
Regression tests: POST endpoints whose body is optional must accept a
|
||||
request that has no body at all.
|
||||
|
||||
Six handlers in api_v3 read their body as ``request.get_json() or {}``.
|
||||
The ``or {}`` states the intent plainly — every field is optional, so a
|
||||
bodyless POST should fall back to defaults. But ``get_json()`` without
|
||||
``silent=True`` raises ``UnsupportedMediaType`` when the request carries
|
||||
no JSON Content-Type, and it raises *before* ``or {}`` is evaluated. Each
|
||||
handler's catch-all then turned that into a 500.
|
||||
|
||||
So the natural way to call these endpoints — a POST with no body, which
|
||||
is what curl, a fetch() without options, and most HTTP clients send by
|
||||
default — failed on every one of them. The shipped UI always sends a JSON
|
||||
object, which is why this went unnoticed.
|
||||
|
||||
This file covers the endpoints whose bodyless behaviour is not already
|
||||
tested in their own suite.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402
|
||||
|
||||
|
||||
class TestOnDemandStart:
|
||||
URL = "/api/v3/display/on-demand/start"
|
||||
|
||||
def test_bodyless_post_is_not_a_server_error(self, api_v3_client, api_v3_module):
|
||||
response = api_v3_client.post(self.URL)
|
||||
# The endpoint may still reject the request on its own terms (no
|
||||
# plugin_id, nothing to display); what it must not do is fail with
|
||||
# a 500 raised out of body parsing.
|
||||
assert response.status_code != 500
|
||||
|
||||
def test_json_body_still_works(self, api_v3_client, api_v3_module):
|
||||
assert api_v3_client.post(self.URL, json={}).status_code != 500
|
||||
|
||||
|
||||
class TestResetPluginConfig:
|
||||
URL = "/api/v3/plugins/config/reset"
|
||||
|
||||
def test_bodyless_post_is_not_a_server_error(self, api_v3_client, api_v3_module):
|
||||
assert api_v3_client.post(self.URL).status_code != 500
|
||||
|
||||
def test_json_body_still_works(self, api_v3_client, api_v3_module):
|
||||
assert api_v3_client.post(self.URL, json={}).status_code != 500
|
||||
|
||||
|
||||
class TestDeleteOfTheDayJson:
|
||||
URL = "/api/v3/plugins/of-the-day/json/delete"
|
||||
|
||||
def test_bodyless_post_is_not_a_server_error(self, api_v3_client, api_v3_module):
|
||||
assert api_v3_client.post(self.URL).status_code != 500
|
||||
|
||||
def test_json_body_still_works(self, api_v3_client, api_v3_module):
|
||||
assert api_v3_client.post(self.URL, json={}).status_code != 500
|
||||
|
||||
|
||||
class TestPluginLimits:
|
||||
URL = "/api/v3/plugins/clock/limits"
|
||||
|
||||
def test_bodyless_post_is_not_a_server_error(self, api_v3_client, api_v3_module):
|
||||
assert api_v3_client.post(self.URL).status_code != 500
|
||||
|
||||
|
||||
class TestMissingBodyGivesTheDeclaredError:
|
||||
"""Handlers that answer "No data provided" must actually be able to.
|
||||
|
||||
A second group of handlers reads `data = request.get_json()` and then
|
||||
guards with `if not data: return 400`. That guard is unreachable for a
|
||||
request with no JSON body, because get_json() raises first — so the
|
||||
caller got a 500 "an error occurred; see logs for details" instead of
|
||||
the 400 the handler plainly intends to send.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"/api/v3/plugins/install",
|
||||
"/api/v3/plugins/install-from-url",
|
||||
"/api/v3/plugins/registry-from-url",
|
||||
"/api/v3/config/raw/main",
|
||||
"/api/v3/config/raw/secrets",
|
||||
"/api/v3/cache/delete",
|
||||
])
|
||||
def test_bodyless_post_gets_a_400_not_a_500(self, api_v3_client, api_v3_module, url):
|
||||
response = api_v3_client.post(url)
|
||||
assert response.status_code == 400, (
|
||||
f"{url} answered {response.status_code}: "
|
||||
f"{response.get_data(as_text=True)[:200]}")
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"/api/v3/plugins/install",
|
||||
"/api/v3/config/raw/main",
|
||||
])
|
||||
def test_malformed_json_gets_a_400_not_a_500(self, api_v3_client, api_v3_module, url):
|
||||
response = api_v3_client.post(
|
||||
url, data="{not json", content_type="application/json")
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
class TestNoBodyReadContradictsItsOwnGuard:
|
||||
SOURCE = Path(__file__).parent.parent / "web_interface/blueprints/api_v3.py"
|
||||
|
||||
def test_no_or_default_read_is_unguarded(self):
|
||||
"""`get_json() or <default>` is a contradiction without silent=True.
|
||||
|
||||
Writing `or {}` declares the body optional; omitting silent=True
|
||||
means the call raises before the default can apply.
|
||||
"""
|
||||
offenders = [
|
||||
line.strip() for line in self.SOURCE.read_text().splitlines()
|
||||
if "request.get_json()" in line and " or " in line
|
||||
]
|
||||
assert offenders == [], (
|
||||
"these reads declare a default but raise before reaching it; "
|
||||
f"use get_json(silent=True): {offenders}")
|
||||
|
||||
def test_no_not_data_guard_is_unreachable(self):
|
||||
"""A `if not data:` guard needs a read that can actually return None."""
|
||||
lines = self.SOURCE.read_text().splitlines()
|
||||
offenders = []
|
||||
for i, line in enumerate(lines):
|
||||
if re.search(r"=\s*request\.get_json\(\)\s*$", line):
|
||||
window = "\n".join(lines[i + 1:i + 3])
|
||||
if re.search(r"if\s+(not\s+data\b|data\s+is\s+None)", window):
|
||||
offenders.append(f"line {i + 1}: {line.strip()}")
|
||||
assert offenders == [], (
|
||||
"these handlers guard on a missing body but raise before the "
|
||||
f"guard runs; use get_json(silent=True): {offenders}")
|
||||
@@ -0,0 +1,302 @@
|
||||
"""
|
||||
Endpoint tests for POST /plugins/install and POST /plugins/install-from-url.
|
||||
|
||||
Both were only ever tested at the PluginStoreManager layer, so the route
|
||||
logic — the queue-vs-direct branch, schema invalidation, plugin discovery,
|
||||
state and history recording — was unexercised.
|
||||
|
||||
/plugins/install carries the same install logic twice: once inside the
|
||||
operation-queue callback and once in the direct fallback. The paired
|
||||
tests below assert both branches produce the same side effects, so the
|
||||
duplication cannot quietly drift.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402
|
||||
|
||||
INSTALL = "/api/v3/plugins/install"
|
||||
FROM_URL = "/api/v3/plugins/install-from-url"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def queued(api_v3_module):
|
||||
"""Enable the operation queue and run its callback synchronously."""
|
||||
queue = MagicMock()
|
||||
|
||||
def enqueue(operation_type, plugin_id, operation_callback=None):
|
||||
queue.callback_result = operation_callback(MagicMock())
|
||||
return "op-123"
|
||||
|
||||
queue.enqueue_operation.side_effect = enqueue
|
||||
api_v3_module.api_v3.operation_queue = queue
|
||||
return queue
|
||||
|
||||
|
||||
def side_effects(module):
|
||||
"""The manager calls a successful install is expected to make."""
|
||||
api = module.api_v3
|
||||
return {
|
||||
"schema_invalidated": api.schema_manager.invalidate_cache.call_args_list,
|
||||
"discovered": api.plugin_manager.discover_plugins.call_count,
|
||||
"loaded": api.plugin_manager.load_plugin.call_args_list,
|
||||
"state_set": api.plugin_state_manager.set_plugin_installed.call_args_list,
|
||||
"history": api.operation_history.record_operation.call_args_list,
|
||||
}
|
||||
|
||||
|
||||
class TestInstallValidation:
|
||||
def test_uninitialized_store_manager_is_a_500(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager = None
|
||||
response = api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
assert response.status_code == 500
|
||||
assert "not initialized" in response.get_json()["message"]
|
||||
|
||||
def test_missing_plugin_id_is_a_400(self, api_v3_client, api_v3_module):
|
||||
response = api_v3_client.post(INSTALL, json={})
|
||||
assert response.status_code == 400
|
||||
assert "plugin_id required" in response.get_json()["message"]
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.assert_not_called()
|
||||
|
||||
def test_empty_body_is_a_400(self, api_v3_client, api_v3_module):
|
||||
assert api_v3_client.post(INSTALL, json=None).status_code == 400
|
||||
|
||||
|
||||
class TestInstallDirectPath:
|
||||
"""operation_queue is None — the fallback branch."""
|
||||
|
||||
def test_success(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = True
|
||||
response = api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["status"] == "success"
|
||||
|
||||
def test_success_side_effects(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = True
|
||||
api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
effects = side_effects(api_v3_module)
|
||||
assert effects["schema_invalidated"] == [(("clock",), {})]
|
||||
assert effects["discovered"] == 1
|
||||
assert effects["loaded"] == [(("clock",), {})]
|
||||
assert effects["state_set"] == [(("clock",), {})]
|
||||
assert effects["history"][0].kwargs["status"] == "success"
|
||||
|
||||
def test_branch_forwarded_to_the_manager(self, api_v3_client, api_v3_module):
|
||||
manager = api_v3_module.api_v3.plugin_store_manager
|
||||
manager.install_plugin.return_value = True
|
||||
api_v3_client.post(INSTALL, json={"plugin_id": "clock", "branch": "dev"})
|
||||
manager.install_plugin.assert_called_once_with("clock", branch="dev")
|
||||
|
||||
def test_branch_named_in_the_message(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = True
|
||||
response = api_v3_client.post(INSTALL, json={"plugin_id": "clock", "branch": "dev"})
|
||||
assert "(branch: dev)" in response.get_json()["message"]
|
||||
|
||||
def test_failure_is_a_500(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = False
|
||||
response = api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
assert response.status_code == 500
|
||||
assert "Failed to install" in response.get_json()["message"]
|
||||
|
||||
def test_failure_mentions_missing_registry_entry(self, api_v3_client, api_v3_module):
|
||||
manager = api_v3_module.api_v3.plugin_store_manager
|
||||
manager.install_plugin.return_value = False
|
||||
manager.get_plugin_info.return_value = None
|
||||
response = api_v3_client.post(INSTALL, json={"plugin_id": "ghost"})
|
||||
assert "not found in registry" in response.get_json()["message"]
|
||||
|
||||
def test_failure_omits_registry_note_when_plugin_is_known(
|
||||
self, api_v3_client, api_v3_module):
|
||||
manager = api_v3_module.api_v3.plugin_store_manager
|
||||
manager.install_plugin.return_value = False
|
||||
manager.get_plugin_info.return_value = {"id": "clock"}
|
||||
response = api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
assert "not found in registry" not in response.get_json()["message"]
|
||||
|
||||
def test_failure_recorded_in_history(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = False
|
||||
api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
record = api_v3_module.api_v3.operation_history.record_operation.call_args
|
||||
assert record.kwargs["status"] == "failed"
|
||||
|
||||
def test_no_side_effects_on_failure(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = False
|
||||
api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
effects = side_effects(api_v3_module)
|
||||
assert effects["schema_invalidated"] == []
|
||||
assert effects["loaded"] == []
|
||||
assert effects["state_set"] == []
|
||||
|
||||
|
||||
class TestInstallQueuedPath:
|
||||
"""operation_queue present — the callback branch."""
|
||||
|
||||
def test_returns_an_operation_id(self, api_v3_client, api_v3_module, queued):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = True
|
||||
response = api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["data"]["operation_id"] == "op-123"
|
||||
|
||||
def test_message_says_queued(self, api_v3_client, api_v3_module, queued):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = True
|
||||
response = api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
assert "queued" in response.get_json()["message"]
|
||||
|
||||
def test_callback_success_side_effects(self, api_v3_client, api_v3_module, queued):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = True
|
||||
api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
effects = side_effects(api_v3_module)
|
||||
assert effects["schema_invalidated"] == [(("clock",), {})]
|
||||
assert effects["discovered"] == 1
|
||||
assert effects["loaded"] == [(("clock",), {})]
|
||||
assert effects["state_set"] == [(("clock",), {})]
|
||||
assert effects["history"][0].kwargs["status"] == "success"
|
||||
|
||||
def test_callback_reports_success(self, api_v3_client, api_v3_module, queued):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = True
|
||||
api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
assert queued.callback_result["success"] is True
|
||||
|
||||
def test_callback_failure_raises_for_the_queue(self, api_v3_client, api_v3_module, queued):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = False
|
||||
# The callback signals failure by raising, so the queue can mark the
|
||||
# operation failed; the route's catch-all turns it into a 500.
|
||||
response = api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
assert response.status_code == 500
|
||||
|
||||
def test_callback_failure_recorded_in_history(self, api_v3_client, api_v3_module, queued):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = False
|
||||
api_v3_client.post(INSTALL, json={"plugin_id": "clock"})
|
||||
record = api_v3_module.api_v3.operation_history.record_operation.call_args
|
||||
assert record.kwargs["status"] == "failed"
|
||||
|
||||
def test_branch_forwarded_from_the_callback(self, api_v3_client, api_v3_module, queued):
|
||||
manager = api_v3_module.api_v3.plugin_store_manager
|
||||
manager.install_plugin.return_value = True
|
||||
api_v3_client.post(INSTALL, json={"plugin_id": "clock", "branch": "dev"})
|
||||
manager.install_plugin.assert_called_once_with("clock", branch="dev")
|
||||
|
||||
|
||||
class TestInstallPathsAgree:
|
||||
"""The queue callback and the direct fallback duplicate the same logic."""
|
||||
|
||||
def _run(self, client, module, install_ok, queue):
|
||||
module.api_v3.plugin_store_manager.install_plugin.return_value = install_ok
|
||||
client.post(INSTALL, json={"plugin_id": "clock", "branch": "dev"})
|
||||
return side_effects(module)
|
||||
|
||||
def test_success_side_effects_match(self, api_v3_client, api_v3_module):
|
||||
direct = self._run(api_v3_client, api_v3_module, True, None)
|
||||
|
||||
# Reset and re-run through the queue.
|
||||
for mock in (api_v3_module.api_v3.schema_manager,
|
||||
api_v3_module.api_v3.plugin_manager,
|
||||
api_v3_module.api_v3.plugin_state_manager,
|
||||
api_v3_module.api_v3.operation_history):
|
||||
mock.reset_mock()
|
||||
queue = MagicMock()
|
||||
queue.enqueue_operation.side_effect = (
|
||||
lambda t, p, operation_callback=None: operation_callback(MagicMock()) and "op")
|
||||
api_v3_module.api_v3.operation_queue = queue
|
||||
queued = self._run(api_v3_client, api_v3_module, True, queue)
|
||||
|
||||
assert direct["schema_invalidated"] == queued["schema_invalidated"]
|
||||
assert direct["discovered"] == queued["discovered"]
|
||||
assert direct["loaded"] == queued["loaded"]
|
||||
assert direct["state_set"] == queued["state_set"]
|
||||
assert (direct["history"][0].kwargs["status"]
|
||||
== queued["history"][0].kwargs["status"])
|
||||
assert (direct["history"][0].kwargs["details"]
|
||||
== queued["history"][0].kwargs["details"])
|
||||
|
||||
def test_only_the_message_wording_differs(self, api_v3_client, api_v3_module):
|
||||
# Characterized: the direct path says "Plugin installed
|
||||
# successfully" while the queue callback says "Plugin clock
|
||||
# installed successfully". Cosmetic, and the queue's text is
|
||||
# internal to the operation record rather than the HTTP response.
|
||||
api_v3_module.api_v3.plugin_store_manager.install_plugin.return_value = True
|
||||
direct = api_v3_client.post(INSTALL, json={"plugin_id": "clock"}).get_json()
|
||||
assert direct["message"] == "Plugin installed successfully"
|
||||
|
||||
|
||||
class TestInstallFromUrl:
|
||||
def test_uninitialized_store_manager_is_a_500(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager = None
|
||||
assert api_v3_client.post(FROM_URL, json={"repo_url": "http://x"}).status_code == 500
|
||||
|
||||
def test_missing_repo_url_is_a_400(self, api_v3_client, api_v3_module):
|
||||
response = api_v3_client.post(FROM_URL, json={})
|
||||
assert response.status_code == 400
|
||||
assert "repo_url required" in response.get_json()["message"]
|
||||
|
||||
def test_success(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_from_url.return_value = {
|
||||
"success": True, "plugin_id": "clock", "name": "Clock"}
|
||||
response = api_v3_client.post(FROM_URL, json={"repo_url": "https://github.com/o/r"})
|
||||
assert response.status_code == 200
|
||||
body = response.get_json()
|
||||
assert body["plugin_id"] == "clock"
|
||||
assert body["name"] == "Clock"
|
||||
|
||||
def test_all_optional_arguments_forwarded(self, api_v3_client, api_v3_module):
|
||||
manager = api_v3_module.api_v3.plugin_store_manager
|
||||
manager.install_from_url.return_value = {"success": True, "plugin_id": "clock"}
|
||||
api_v3_client.post(FROM_URL, json={
|
||||
"repo_url": " https://github.com/o/r ",
|
||||
"plugin_id": "clock",
|
||||
"plugin_path": "plugins/clock",
|
||||
"branch": "dev",
|
||||
})
|
||||
manager.install_from_url.assert_called_once_with(
|
||||
repo_url="https://github.com/o/r",
|
||||
plugin_id="clock",
|
||||
plugin_path="plugins/clock",
|
||||
branch="dev",
|
||||
)
|
||||
|
||||
def test_success_invalidates_schema_and_loads_plugin(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_from_url.return_value = {
|
||||
"success": True, "plugin_id": "clock"}
|
||||
api_v3_client.post(FROM_URL, json={"repo_url": "http://x"})
|
||||
api_v3_module.api_v3.schema_manager.invalidate_cache.assert_called_once_with("clock")
|
||||
api_v3_module.api_v3.plugin_manager.load_plugin.assert_called_once_with("clock")
|
||||
|
||||
def test_success_without_plugin_id_skips_discovery(self, api_v3_client, api_v3_module):
|
||||
# install_from_url can succeed without naming the plugin; there is
|
||||
# then nothing to invalidate or load.
|
||||
api_v3_module.api_v3.plugin_store_manager.install_from_url.return_value = {
|
||||
"success": True, "plugin_id": None}
|
||||
api_v3_client.post(FROM_URL, json={"repo_url": "http://x"})
|
||||
api_v3_module.api_v3.schema_manager.invalidate_cache.assert_not_called()
|
||||
api_v3_module.api_v3.plugin_manager.load_plugin.assert_not_called()
|
||||
|
||||
def test_branch_from_result_included(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_from_url.return_value = {
|
||||
"success": True, "plugin_id": "clock", "branch": "dev"}
|
||||
body = api_v3_client.post(FROM_URL, json={"repo_url": "http://x"}).get_json()
|
||||
assert body["branch"] == "dev"
|
||||
assert "(branch: dev)" in body["message"]
|
||||
|
||||
def test_failure_reports_the_managers_error(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_from_url.return_value = {
|
||||
"success": False, "error": "repo not found"}
|
||||
response = api_v3_client.post(FROM_URL, json={"repo_url": "http://x"})
|
||||
assert response.status_code == 500
|
||||
assert response.get_json()["message"] == "repo not found"
|
||||
|
||||
def test_failure_without_error_uses_fallback_text(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_from_url.return_value = {
|
||||
"success": False}
|
||||
response = api_v3_client.post(FROM_URL, json={"repo_url": "http://x"})
|
||||
assert "Failed to install plugin from URL" in response.get_json()["message"]
|
||||
|
||||
def test_manager_exception_is_a_500(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.install_from_url.side_effect = (
|
||||
RuntimeError("boom"))
|
||||
assert api_v3_client.post(FROM_URL, json={"repo_url": "http://x"}).status_code == 500
|
||||
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
Endpoint tests for the plugin-registry routes in api_v3:
|
||||
POST /plugins/store/refresh and POST /plugins/registry-from-url.
|
||||
|
||||
Both reach out to the network through PluginStoreManager (mocked here) and
|
||||
had no endpoint-level coverage; registry-from-url in particular takes a
|
||||
user-supplied URL and hands it straight to the manager.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402
|
||||
|
||||
|
||||
class TestRefreshPluginStore:
|
||||
URL = "/api/v3/plugins/store/refresh"
|
||||
|
||||
def test_uninitialized_manager_is_a_500(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager = None
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code == 500
|
||||
assert "not initialized" in response.get_json()["message"]
|
||||
|
||||
def test_success_reports_plugin_count(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry.return_value = {
|
||||
"plugins": [{"id": "a"}, {"id": "b"}, {"id": "c"}]}
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["plugin_count"] == 3
|
||||
|
||||
def test_forces_a_refresh_rather_than_using_cache(self, api_v3_client, api_v3_module):
|
||||
manager = api_v3_module.api_v3.plugin_store_manager
|
||||
manager.fetch_registry.return_value = {"plugins": []}
|
||||
api_v3_client.post(self.URL, json={})
|
||||
manager.fetch_registry.assert_called_once_with(force_refresh=True)
|
||||
|
||||
def test_empty_registry_reports_zero(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry.return_value = {}
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.get_json()["plugin_count"] == 0
|
||||
|
||||
def test_no_body_is_accepted(self, api_v3_client, api_v3_module):
|
||||
# Regression: `request.get_json() or {}` says a missing body is
|
||||
# fine, but get_json() raises UnsupportedMediaType before `or {}`
|
||||
# is reached, so a bodyless POST — the natural way to call a
|
||||
# refresh endpoint — came back 500.
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry.return_value = {"plugins": []}
|
||||
assert api_v3_client.post(self.URL).status_code == 200
|
||||
|
||||
def test_body_without_json_content_type_is_accepted(
|
||||
self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry.return_value = {"plugins": []}
|
||||
response = api_v3_client.post(self.URL, data="", content_type="text/plain")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_malformed_json_body_falls_back_to_defaults(
|
||||
self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry.return_value = {"plugins": []}
|
||||
response = api_v3_client.post(
|
||||
self.URL, data="{not json", content_type="application/json")
|
||||
assert response.status_code == 200
|
||||
|
||||
@pytest.mark.parametrize("key", ["fetch_commit_info", "fetch_latest_versions"])
|
||||
def test_either_commit_info_key_extends_the_message(
|
||||
self, api_v3_client, api_v3_module, key):
|
||||
# fetch_latest_versions is the older spelling; both must work.
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry.return_value = {"plugins": []}
|
||||
response = api_v3_client.post(self.URL, json={key: True})
|
||||
assert "commit metadata" in response.get_json()["message"]
|
||||
|
||||
def test_message_stays_plain_without_the_flag(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry.return_value = {"plugins": []}
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.get_json()["message"] == "Plugin store refreshed"
|
||||
|
||||
def test_network_failure_is_a_500(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry.side_effect = (
|
||||
ConnectionError("github unreachable"))
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code == 500
|
||||
assert response.get_json()["message"] == "An error occurred; see logs for details"
|
||||
|
||||
def test_failure_body_carries_no_traceback_or_paths(
|
||||
self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry.side_effect = (
|
||||
RuntimeError("failed at /home/user/LEDMatrix/src/secret.py line 42"))
|
||||
body = api_v3_client.post(self.URL, json={}).get_json()
|
||||
assert "Traceback" not in str(body)
|
||||
# `details` is describe_exception output: one line, type-named,
|
||||
# credential-redacted. It may quote the message, but never a stack.
|
||||
assert body["details"].startswith("RuntimeError:")
|
||||
assert "\n" not in body["details"]
|
||||
|
||||
|
||||
class TestRegistryFromUrl:
|
||||
URL = "/api/v3/plugins/registry-from-url"
|
||||
|
||||
def test_uninitialized_manager_is_a_500(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager = None
|
||||
response = api_v3_client.post(self.URL, json={"repo_url": "http://x"})
|
||||
assert response.status_code == 500
|
||||
|
||||
def test_missing_repo_url_is_a_400(self, api_v3_client, api_v3_module):
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code == 400
|
||||
assert "repo_url required" in response.get_json()["message"]
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry_from_url.assert_not_called()
|
||||
|
||||
def test_success_returns_the_plugin_list(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry_from_url.return_value = {
|
||||
"plugins": [{"id": "clock"}]}
|
||||
response = api_v3_client.post(
|
||||
self.URL, json={"repo_url": "https://github.com/o/r"})
|
||||
assert response.status_code == 200
|
||||
body = response.get_json()
|
||||
assert body["plugins"] == [{"id": "clock"}]
|
||||
assert body["registry_url"] == "https://github.com/o/r"
|
||||
|
||||
def test_url_is_trimmed_before_use(self, api_v3_client, api_v3_module):
|
||||
manager = api_v3_module.api_v3.plugin_store_manager
|
||||
manager.fetch_registry_from_url.return_value = {"plugins": []}
|
||||
api_v3_client.post(self.URL, json={"repo_url": " https://github.com/o/r "})
|
||||
manager.fetch_registry_from_url.assert_called_once_with("https://github.com/o/r")
|
||||
|
||||
def test_registry_without_plugins_key_returns_empty_list(
|
||||
self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry_from_url.return_value = {
|
||||
"other": 1}
|
||||
response = api_v3_client.post(self.URL, json={"repo_url": "http://x"})
|
||||
assert response.get_json()["plugins"] == []
|
||||
|
||||
def test_no_registry_found_is_a_400(self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry_from_url.return_value = None
|
||||
response = api_v3_client.post(self.URL, json={"repo_url": "http://x/not-a-registry"})
|
||||
assert response.status_code == 400
|
||||
assert "Failed to fetch registry" in response.get_json()["message"]
|
||||
|
||||
@pytest.mark.parametrize("url", [
|
||||
"not a url",
|
||||
"javascript:alert(1)",
|
||||
"file:///etc/passwd",
|
||||
"http://localhost:8080/admin",
|
||||
])
|
||||
def test_unusable_urls_fail_cleanly(self, api_v3_client, api_v3_module, url):
|
||||
# Characterization: the handler performs no URL validation of its
|
||||
# own — whatever the manager makes of the URL decides the outcome.
|
||||
# What is pinned here is that a rejected URL produces a clean 400
|
||||
# rather than a traceback or a 500.
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry_from_url.return_value = None
|
||||
response = api_v3_client.post(self.URL, json={"repo_url": url})
|
||||
assert response.status_code == 400
|
||||
assert "Traceback" not in str(response.get_json())
|
||||
|
||||
def test_fetch_exception_is_a_500_without_internals(
|
||||
self, api_v3_client, api_v3_module):
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry_from_url.side_effect = (
|
||||
ValueError("parse failed in /srv/app/internal.py"))
|
||||
response = api_v3_client.post(self.URL, json={"repo_url": "http://x"})
|
||||
assert response.status_code == 500
|
||||
body = response.get_json()
|
||||
assert body["message"] == "An error occurred; see logs for details"
|
||||
assert "Traceback" not in str(body)
|
||||
|
||||
def test_non_string_repo_url_is_rejected(self, api_v3_client, api_v3_module):
|
||||
# Regression: .strip() on a non-string raised, and the catch-all
|
||||
# reported the caller's own mistake as a server fault.
|
||||
response = api_v3_client.post(self.URL, json={"repo_url": 12345})
|
||||
assert response.status_code == 400
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry_from_url.assert_not_called()
|
||||
|
||||
def test_blank_repo_url_is_rejected(self, api_v3_client, api_v3_module):
|
||||
response = api_v3_client.post(self.URL, json={"repo_url": " "})
|
||||
assert response.status_code == 400
|
||||
api_v3_module.api_v3.plugin_store_manager.fetch_registry_from_url.assert_not_called()
|
||||
@@ -0,0 +1,240 @@
|
||||
"""
|
||||
Endpoint tests for the /wifi/* routes in api_v3.
|
||||
|
||||
These routes drive the host's actual networking — connecting, dropping a
|
||||
connection, switching the radio off — and had no endpoint-level tests at
|
||||
all. WiFiManager is mocked throughout; nothing here may touch real
|
||||
networking.
|
||||
|
||||
Each handler does `from src.wifi_manager import WiFiManager` inside the
|
||||
function body, so the patch target is the class at its definition site.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from test._api_v3_test_helpers import api_v3_client, api_v3_module # noqa: F401,E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wifi_manager():
|
||||
"""Patch WiFiManager where it is defined; yield the instance mock."""
|
||||
with patch("src.wifi_manager.WiFiManager") as cls:
|
||||
instance = MagicMock()
|
||||
cls.return_value = instance
|
||||
yield instance
|
||||
|
||||
|
||||
class TestConnect:
|
||||
URL = "/api/v3/wifi/connect"
|
||||
|
||||
def test_success(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.connect_to_network.return_value = (True, "Connected to HomeNet")
|
||||
response = api_v3_client.post(self.URL, json={"ssid": "HomeNet", "password": "pw"})
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["message"] == "Connected to HomeNet"
|
||||
wifi_manager.connect_to_network.assert_called_once_with("HomeNet", "pw")
|
||||
|
||||
def test_missing_body_rejected(self, api_v3_client, wifi_manager):
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code == 400
|
||||
wifi_manager.connect_to_network.assert_not_called()
|
||||
|
||||
def test_missing_ssid_rejected(self, api_v3_client, wifi_manager):
|
||||
response = api_v3_client.post(self.URL, json={"password": "pw"})
|
||||
assert response.status_code == 400
|
||||
assert "SSID is required" in response.get_json()["message"]
|
||||
wifi_manager.connect_to_network.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("ssid", ["", " ", "\t"])
|
||||
def test_blank_ssid_rejected(self, api_v3_client, wifi_manager, ssid):
|
||||
response = api_v3_client.post(self.URL, json={"ssid": ssid})
|
||||
assert response.status_code == 400
|
||||
wifi_manager.connect_to_network.assert_not_called()
|
||||
|
||||
def test_ssid_is_trimmed(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.connect_to_network.return_value = (True, "ok")
|
||||
api_v3_client.post(self.URL, json={"ssid": " HomeNet "})
|
||||
wifi_manager.connect_to_network.assert_called_once_with("HomeNet", "")
|
||||
|
||||
def test_missing_password_becomes_empty_string(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.connect_to_network.return_value = (True, "ok")
|
||||
api_v3_client.post(self.URL, json={"ssid": "OpenNet"})
|
||||
wifi_manager.connect_to_network.assert_called_once_with("OpenNet", "")
|
||||
|
||||
def test_null_password_becomes_empty_string(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.connect_to_network.return_value = (True, "ok")
|
||||
api_v3_client.post(self.URL, json={"ssid": "OpenNet", "password": None})
|
||||
wifi_manager.connect_to_network.assert_called_once_with("OpenNet", "")
|
||||
|
||||
def test_failure_reports_the_managers_reason(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.connect_to_network.return_value = (False, "Bad password")
|
||||
response = api_v3_client.post(self.URL, json={"ssid": "HomeNet"})
|
||||
assert response.status_code == 400
|
||||
assert response.get_json()["message"] == "Bad password"
|
||||
|
||||
def test_failure_without_reason_uses_fallback_text(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.connect_to_network.return_value = (False, None)
|
||||
response = api_v3_client.post(self.URL, json={"ssid": "HomeNet"})
|
||||
assert response.status_code == 400
|
||||
assert response.get_json()["message"] == "Failed to connect to network"
|
||||
|
||||
def test_manager_exception_is_a_500_without_leaking_internals(
|
||||
self, api_v3_client, wifi_manager):
|
||||
wifi_manager.connect_to_network.side_effect = RuntimeError(
|
||||
"/usr/lib/secret/path blew up")
|
||||
response = api_v3_client.post(self.URL, json={"ssid": "HomeNet"})
|
||||
assert response.status_code == 500
|
||||
body = response.get_json()
|
||||
assert body["message"] == "An error occurred; see logs for details"
|
||||
# `details` comes from describe_exception, which is deliberately
|
||||
# safe to return (redacted, capped) — it names the type.
|
||||
assert "RuntimeError" in body["details"]
|
||||
|
||||
|
||||
class TestDisconnect:
|
||||
URL = "/api/v3/wifi/disconnect"
|
||||
|
||||
def test_success(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.disconnect_from_network.return_value = (True, "Disconnected")
|
||||
response = api_v3_client.post(self.URL)
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["message"] == "Disconnected"
|
||||
|
||||
def test_failure(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.disconnect_from_network.return_value = (False, "Not connected")
|
||||
response = api_v3_client.post(self.URL)
|
||||
assert response.status_code == 400
|
||||
assert response.get_json()["message"] == "Not connected"
|
||||
|
||||
def test_failure_without_reason_uses_fallback(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.disconnect_from_network.return_value = (False, "")
|
||||
response = api_v3_client.post(self.URL)
|
||||
assert response.get_json()["message"] == "Failed to disconnect from network"
|
||||
|
||||
def test_exception_is_a_500(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.disconnect_from_network.side_effect = OSError("nmcli missing")
|
||||
assert api_v3_client.post(self.URL).status_code == 500
|
||||
|
||||
|
||||
class TestApMode:
|
||||
ENABLE = "/api/v3/wifi/ap/enable"
|
||||
DISABLE = "/api/v3/wifi/ap/disable"
|
||||
|
||||
def test_enable_success(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.enable_ap_mode.return_value = (True, "AP enabled")
|
||||
response = api_v3_client.post(self.ENABLE, json={})
|
||||
assert response.status_code == 200
|
||||
wifi_manager.enable_ap_mode.assert_called_once_with(force=False)
|
||||
|
||||
@pytest.mark.parametrize("raw,expected", [
|
||||
(True, True), (False, False),
|
||||
("true", True), ("TRUE", True), ("1", True),
|
||||
("false", False), ("no", False), ("yes", False),
|
||||
(1, False), # only real True or the listed strings count
|
||||
])
|
||||
def test_force_coercion(self, api_v3_client, wifi_manager, raw, expected):
|
||||
wifi_manager.enable_ap_mode.return_value = (True, "ok")
|
||||
api_v3_client.post(self.ENABLE, json={"force": raw})
|
||||
wifi_manager.enable_ap_mode.assert_called_once_with(force=expected)
|
||||
|
||||
def test_enable_without_body(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.enable_ap_mode.return_value = (True, "ok")
|
||||
assert api_v3_client.post(self.ENABLE).status_code == 200
|
||||
|
||||
def test_enable_failure(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.enable_ap_mode.return_value = (False, "hostapd missing")
|
||||
response = api_v3_client.post(self.ENABLE, json={})
|
||||
assert response.status_code == 400
|
||||
assert response.get_json()["message"] == "hostapd missing"
|
||||
|
||||
def test_disable_success(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.disable_ap_mode.return_value = (True, "AP disabled")
|
||||
assert api_v3_client.post(self.DISABLE).status_code == 200
|
||||
|
||||
def test_disable_failure(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.disable_ap_mode.return_value = (False, "not running")
|
||||
assert api_v3_client.post(self.DISABLE).status_code == 400
|
||||
|
||||
def test_enable_exception_is_a_500(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.enable_ap_mode.side_effect = RuntimeError("boom")
|
||||
assert api_v3_client.post(self.ENABLE, json={}).status_code == 500
|
||||
|
||||
|
||||
class TestRadio:
|
||||
URL = "/api/v3/wifi/radio"
|
||||
|
||||
def test_get_state(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.get_wifi_radio_state.return_value = {
|
||||
"enabled": True, "ethernet_connected": False}
|
||||
response = api_v3_client.get(self.URL)
|
||||
assert response.status_code == 200
|
||||
assert response.get_json()["data"]["enabled"] is True
|
||||
|
||||
def test_get_state_exception_is_a_500(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.get_wifi_radio_state.side_effect = OSError("rfkill missing")
|
||||
assert api_v3_client.get(self.URL).status_code == 500
|
||||
|
||||
def test_enabled_is_required(self, api_v3_client, wifi_manager):
|
||||
response = api_v3_client.post(self.URL, json={})
|
||||
assert response.status_code == 400
|
||||
assert "enabled is required" in response.get_json()["message"]
|
||||
wifi_manager.set_wifi_radio.assert_not_called()
|
||||
|
||||
def test_enable_success(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.set_wifi_radio.return_value = (True, "Radio on", None)
|
||||
wifi_manager.get_wifi_radio_state.return_value = {"enabled": True}
|
||||
response = api_v3_client.post(self.URL, json={"enabled": True})
|
||||
assert response.status_code == 200
|
||||
wifi_manager.set_wifi_radio.assert_called_once_with(True, force=False)
|
||||
|
||||
@pytest.mark.parametrize("raw,expected", [
|
||||
(True, True), ("true", True), ("1", True), ("yes", True),
|
||||
(False, False), ("false", False), ("off", False), (0, False),
|
||||
])
|
||||
def test_enabled_coercion_is_string_aware(
|
||||
self, api_v3_client, wifi_manager, raw, expected):
|
||||
# bool("false") is True, so the endpoint parses strings explicitly
|
||||
# rather than trusting truthiness — it is a public contract, not
|
||||
# only the shipped UI which always sends real JSON booleans.
|
||||
wifi_manager.set_wifi_radio.return_value = (True, "ok", None)
|
||||
wifi_manager.get_wifi_radio_state.return_value = {}
|
||||
api_v3_client.post(self.URL, json={"enabled": raw})
|
||||
wifi_manager.set_wifi_radio.assert_called_once_with(expected, force=False)
|
||||
|
||||
def test_force_passed_through(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.set_wifi_radio.return_value = (True, "ok", None)
|
||||
wifi_manager.get_wifi_radio_state.return_value = {}
|
||||
api_v3_client.post(self.URL, json={"enabled": False, "force": "true"})
|
||||
wifi_manager.set_wifi_radio.assert_called_once_with(False, force=True)
|
||||
|
||||
def test_refusal_reports_reason(self, api_v3_client, wifi_manager):
|
||||
# Disabling the radio without Ethernet would lock the user out of
|
||||
# this very interface, so the manager can refuse with a reason.
|
||||
wifi_manager.set_wifi_radio.return_value = (
|
||||
False, "Refusing: no wired fallback", "no_ethernet")
|
||||
response = api_v3_client.post(self.URL, json={"enabled": False})
|
||||
assert response.status_code == 400
|
||||
body = response.get_json()
|
||||
assert body["reason"] == "no_ethernet"
|
||||
assert "Refusing" in body["message"]
|
||||
|
||||
def test_exception_is_a_500(self, api_v3_client, wifi_manager):
|
||||
wifi_manager.set_wifi_radio.side_effect = RuntimeError("boom")
|
||||
assert api_v3_client.post(self.URL, json={"enabled": True}).status_code == 500
|
||||
|
||||
|
||||
class TestNoRealNetworking:
|
||||
def test_wifi_manager_is_never_constructed_for_real(self, api_v3_client):
|
||||
# Guard against a future refactor moving the import to module level,
|
||||
# where the fixture's patch of the definition site would stop
|
||||
# applying and the tests would start driving real networking.
|
||||
with patch("src.wifi_manager.WiFiManager") as cls:
|
||||
cls.return_value.disconnect_from_network.return_value = (True, "ok")
|
||||
api_v3_client.post("/api/v3/wifi/disconnect")
|
||||
assert cls.called
|
||||
@@ -0,0 +1,450 @@
|
||||
"""A second request for a key already being fetched must join, not duplicate.
|
||||
|
||||
request_id embeds a millisecond timestamp and active_requests is keyed by it,
|
||||
so every submit looked new and nothing compared what was actually being
|
||||
fetched. On a real board the season-schedule cache_key is requested by both
|
||||
the Recent and the Upcoming manager: they miss the cache in the same
|
||||
millisecond and each start a full download and parse of the same payload.
|
||||
Measured on a running board, 138 background fetches in 24 hours arriving in
|
||||
pairs at identical timestamps -- half of them redundant.
|
||||
|
||||
The cost of a duplicate is a second download, a second JSON parse (the
|
||||
expensive part on a Pi), and a second parsed copy resident at the same time.
|
||||
Schedules on that board run from 256KB to 20MB. It also consumes a second of
|
||||
the three executor slots with identical work, which is what makes two large
|
||||
parses peak simultaneously.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from src.background_data_service import BackgroundDataService, FetchStatus
|
||||
|
||||
|
||||
PAYLOAD = {"events": [{"id": f"g{i}"} for i in range(20)]}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cache():
|
||||
m = MagicMock()
|
||||
m.get.return_value = None # always a miss: force the fetch path
|
||||
m.set.return_value = None
|
||||
return m
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def service(cache):
|
||||
svc = BackgroundDataService(cache, max_workers=3, request_timeout=5)
|
||||
yield svc
|
||||
svc.shutdown(wait=False)
|
||||
|
||||
|
||||
def _resp():
|
||||
r = Mock()
|
||||
r.json.return_value = PAYLOAD
|
||||
r.raise_for_status.return_value = None
|
||||
return r
|
||||
|
||||
|
||||
def _wait(service, req_id, timeout=5):
|
||||
deadline = time.time() + timeout
|
||||
while not service.is_request_complete(req_id) and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
|
||||
|
||||
class _BlockingSession:
|
||||
"""Holds the first fetch open so a second can be submitted mid-flight."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
self.release = threading.Event()
|
||||
self.started = threading.Event()
|
||||
|
||||
def get(self, *a, **k):
|
||||
self.calls += 1
|
||||
self.started.set()
|
||||
self.release.wait(timeout=5)
|
||||
return _resp()
|
||||
|
||||
|
||||
def test_a_second_submit_for_the_same_key_does_not_fetch_twice(service):
|
||||
session = _BlockingSession()
|
||||
with patch.object(service, "session", session):
|
||||
first = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="nba_2026",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
assert session.started.wait(timeout=5)
|
||||
|
||||
second = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="nba_2026",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
|
||||
assert second == first, "the joiner should share the in-flight request id"
|
||||
session.release.set()
|
||||
_wait(service, first)
|
||||
|
||||
assert session.calls == 1, f"the payload was fetched {session.calls} times"
|
||||
|
||||
|
||||
def test_the_joiner_still_gets_its_callback(service):
|
||||
session = _BlockingSession()
|
||||
seen = []
|
||||
with patch.object(service, "session", session):
|
||||
first = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=lambda r: seen.append("first"), max_retries=0)
|
||||
assert session.started.wait(timeout=5)
|
||||
joined = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=lambda r: seen.append("second"), max_retries=0)
|
||||
# Assert the coalescing happened, otherwise this passes trivially:
|
||||
# two independent requests would each fire their own callback and the
|
||||
# test would say nothing about the joined path.
|
||||
assert joined == first
|
||||
session.release.set()
|
||||
_wait(service, first)
|
||||
|
||||
deadline = time.time() + 5
|
||||
while len(seen) < 2 and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
assert sorted(seen) == ["first", "second"], (
|
||||
f"both submitters must be called back, got {seen}")
|
||||
|
||||
|
||||
def test_one_callback_raising_does_not_silence_the_other(service):
|
||||
session = _BlockingSession()
|
||||
seen = []
|
||||
|
||||
def boom(result):
|
||||
raise RuntimeError("consumer blew up")
|
||||
|
||||
with patch.object(service, "session", session):
|
||||
first = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=boom, max_retries=0)
|
||||
assert session.started.wait(timeout=5)
|
||||
joined = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=lambda r: seen.append("survivor"), max_retries=0)
|
||||
# Same reason: without coalescing these are separate requests and
|
||||
# neither callback can affect the other.
|
||||
assert joined == first
|
||||
session.release.set()
|
||||
_wait(service, first)
|
||||
|
||||
deadline = time.time() + 5
|
||||
while not seen and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
assert seen == ["survivor"]
|
||||
|
||||
|
||||
def test_different_keys_are_not_coalesced(service):
|
||||
session = _BlockingSession()
|
||||
with patch.object(service, "session", session):
|
||||
a = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/a", cache_key="key_a",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
assert session.started.wait(timeout=5)
|
||||
b = service.submit_fetch_request(
|
||||
sport="nhl", year=2026, url="https://x/b", cache_key="key_b",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
assert a != b, "different cache keys must not share a request"
|
||||
session.release.set()
|
||||
_wait(service, a)
|
||||
_wait(service, b)
|
||||
assert session.calls == 2
|
||||
|
||||
|
||||
def test_a_later_submit_after_completion_fetches_again(service):
|
||||
"""Dedupe is for concurrent requests only, not a second cache layer."""
|
||||
with patch.object(service.session, "get", side_effect=[_resp(), _resp()]) as get:
|
||||
first = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
_wait(service, first)
|
||||
second = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
_wait(service, second)
|
||||
assert first != second
|
||||
assert get.call_count == 2
|
||||
|
||||
|
||||
def test_cancelling_releases_the_key(service):
|
||||
"""A cancelled request must not wedge its key against future fetches."""
|
||||
session = _BlockingSession()
|
||||
with patch.object(service, "session", session):
|
||||
first = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
assert session.started.wait(timeout=5)
|
||||
service.cancel_request(first)
|
||||
assert "k" not in service._inflight_by_cache_key
|
||||
session.release.set()
|
||||
|
||||
|
||||
def test_a_stranded_index_entry_cannot_wedge_a_key(service):
|
||||
"""Defensive: the request is looked up, not trusted from the id alone."""
|
||||
service._inflight_by_cache_key["ghost"] = "no_such_request"
|
||||
with patch.object(service.session, "get", return_value=_resp()):
|
||||
req = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="ghost",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
_wait(service, req)
|
||||
assert service.get_result(req).success is True
|
||||
|
||||
|
||||
def test_the_deduplicated_count_is_reported(service):
|
||||
session = _BlockingSession()
|
||||
with patch.object(service, "session", session):
|
||||
first = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
assert session.started.wait(timeout=5)
|
||||
service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
max_retries=0)
|
||||
session.release.set()
|
||||
_wait(service, first)
|
||||
assert service.get_statistics().get("deduplicated_requests") == 1
|
||||
|
||||
|
||||
def test_a_cancelled_worker_cannot_overwrite_its_replacement(service, cache):
|
||||
"""Cancelling frees the key, so a replacement may already own it.
|
||||
|
||||
The worker cannot abort an HTTP call in flight, so when the cancelled one
|
||||
finally returns it must discard its response rather than write it. Without
|
||||
that, the sequence is: cancel A, submit B for the same key, B fetches and
|
||||
caches fresh data, A returns and overwrites it with the response nobody
|
||||
wanted -- and calls A's callbacks too.
|
||||
"""
|
||||
slow = _BlockingSession()
|
||||
stale = {"events": [{"id": "STALE"}]}
|
||||
slow_resp = Mock()
|
||||
slow_resp.json.return_value = stale
|
||||
slow_resp.raise_for_status.return_value = None
|
||||
|
||||
def blocked_get(*a, **k):
|
||||
slow.calls += 1
|
||||
slow.started.set()
|
||||
slow.release.wait(timeout=5)
|
||||
return slow_resp
|
||||
|
||||
called = []
|
||||
with patch.object(service.session, "get", side_effect=blocked_get):
|
||||
first = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=lambda r: called.append("cancelled_one"), max_retries=0)
|
||||
assert slow.started.wait(timeout=5)
|
||||
|
||||
service.cancel_request(first)
|
||||
assert "k" not in service._inflight_by_cache_key
|
||||
|
||||
# The replacement writes the fresh value while the cancelled fetch is held.
|
||||
fresh = {"events": [{"id": "FRESH"}]}
|
||||
fresh_resp = Mock()
|
||||
fresh_resp.json.return_value = fresh
|
||||
fresh_resp.raise_for_status.return_value = None
|
||||
with patch.object(service.session, "get", return_value=fresh_resp):
|
||||
second = service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s", cache_key="k",
|
||||
callback=lambda r: called.append("replacement"), max_retries=0)
|
||||
_wait(service, second)
|
||||
|
||||
assert cache.set.call_args[0][1] == fresh, "replacement must own the cache"
|
||||
|
||||
# Now let the cancelled fetch finish. It must write nothing and call nobody.
|
||||
# Wait for the worker to actually finish rather than sleeping: a fixed
|
||||
# sleep is a race under load, and a slow worker would make this pass for
|
||||
# the wrong reason. A cancelled request is still filed in
|
||||
# completed_requests, so that is the signal it has run to completion.
|
||||
writes_before = cache.set.call_count
|
||||
slow.release.set()
|
||||
deadline = time.time() + 5
|
||||
while first not in service.completed_requests and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
assert first in service.completed_requests, "cancelled worker never finished"
|
||||
|
||||
assert cache.set.call_count == writes_before, (
|
||||
"the cancelled worker wrote to the cache after its replacement")
|
||||
assert cache.set.call_args[0][1] == fresh, "stale data overwrote fresh"
|
||||
assert "cancelled_one" not in called, (
|
||||
"a cancelled request must not deliver callbacks")
|
||||
|
||||
|
||||
def test_request_ids_are_unique_within_a_millisecond(service):
|
||||
"""request_id was sport_year_milliseconds, which collides.
|
||||
|
||||
Two submits inside the same millisecond produced the SAME id, so one
|
||||
silently replaced the other in active_requests and completed_requests.
|
||||
Dedupe hands this id back to every joiner as their handle for
|
||||
get_result(), so uniqueness is now load-bearing rather than incidental.
|
||||
"""
|
||||
# Stub the executor rather than the session: this is about what submit
|
||||
# hands back, and letting 50 workers loose would outlive the patch and
|
||||
# make real network calls.
|
||||
with patch.object(service.executor, "submit"):
|
||||
ids = [
|
||||
service.submit_fetch_request(
|
||||
sport="nba", year=2026, url="https://x/s",
|
||||
cache_key=f"key_{i}", # distinct keys: no dedupe
|
||||
callback=lambda r: None, max_retries=0)
|
||||
for i in range(50)
|
||||
]
|
||||
assert len(set(ids)) == len(ids), "request ids collided"
|
||||
|
||||
|
||||
# --- cancellation must be terminal ------------------------------------------
|
||||
#
|
||||
# Cancelling used to be advisory: three separate paths wrote request.status
|
||||
# without checking whether the request had already been cancelled, so a cancel
|
||||
# could be silently undone and the work it was meant to stop went ahead.
|
||||
|
||||
URL = "http://example.invalid/scores"
|
||||
KEY = "sched_nfl_2025"
|
||||
|
||||
|
||||
class _CountingSession:
|
||||
"""Records whether an HTTP fetch was ever attempted."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def get(self, *a, **k):
|
||||
self.calls += 1
|
||||
return _resp()
|
||||
|
||||
|
||||
class _BlockingFailingSession(_CountingSession):
|
||||
"""Holds the fetch open, then fails it.
|
||||
|
||||
Cancelling while the worker is parked inside the HTTP call is the only way
|
||||
to reach the exception handler as a cancelled request. Cancel it before
|
||||
the call starts and the worker returns at the pre-start branch instead,
|
||||
which would leave the except path untested.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.started = threading.Event()
|
||||
self.release = threading.Event()
|
||||
|
||||
def get(self, *a, **k):
|
||||
self.calls += 1
|
||||
self.started.set()
|
||||
self.release.wait(timeout=5)
|
||||
raise requests.RequestException("connection reset")
|
||||
|
||||
|
||||
def _fill_every_worker_slot(service, slots=3):
|
||||
"""Occupy the pool so the next submit is queued rather than started.
|
||||
|
||||
This is what makes "cancel before the worker runs" deterministic instead
|
||||
of a race the test would win only sometimes. Returns the gate that
|
||||
releases the pool.
|
||||
"""
|
||||
gate = threading.Event()
|
||||
for _ in range(slots):
|
||||
service.executor.submit(gate.wait, 5)
|
||||
return gate
|
||||
|
||||
|
||||
def test_cancelling_before_the_worker_starts_stops_the_fetch(service, cache):
|
||||
"""The queued worker must honour a cancel, not overwrite it with IN_PROGRESS.
|
||||
|
||||
Between submit and the worker picking the job up, the request sits in the
|
||||
executor queue. Cancelling there is the cheapest possible cancel -- nothing
|
||||
has been downloaded yet -- and it was the one that did not work.
|
||||
"""
|
||||
gate = _fill_every_worker_slot(service)
|
||||
session = _CountingSession()
|
||||
delivered = []
|
||||
|
||||
with patch.object(service, 'session', session):
|
||||
rid = service.submit_fetch_request(
|
||||
"nfl", 2025, URL, KEY, max_retries=0, callback=delivered.append
|
||||
)
|
||||
assert service.cancel_request(rid) is True
|
||||
gate.set() # let the queued worker run
|
||||
_wait(service, rid)
|
||||
|
||||
assert session.calls == 0, (
|
||||
"cancelled before it started, yet the worker still downloaded the payload"
|
||||
)
|
||||
assert cache.set.call_count == 0, "a cancelled request wrote to the cache"
|
||||
assert delivered == [], "a cancelled request invoked its callbacks"
|
||||
|
||||
|
||||
def test_a_cancel_during_the_commit_is_refused(service, cache):
|
||||
"""Once the worker has claimed the commit, cancelling is too late.
|
||||
|
||||
The claim and the cancelled-check happen in one critical section, so a
|
||||
cancel arriving after it cannot retroactively abandon data already on its
|
||||
way to the cache. Letting it through stranded every joiner: the payload
|
||||
landed in the cache but the callbacks were suppressed, so a manager that
|
||||
joined this fetch waited for a call that never came.
|
||||
"""
|
||||
gate = _fill_every_worker_slot(service)
|
||||
late = {}
|
||||
delivered = []
|
||||
|
||||
def cancel_mid_write(key, data, *a, **k):
|
||||
late['returned'] = service.cancel_request(late['rid'])
|
||||
|
||||
cache.set.side_effect = cancel_mid_write
|
||||
|
||||
# Read the payload inside the callback. The service releases result.data
|
||||
# once every callback has been delivered, so inspecting the FetchResult
|
||||
# afterwards sees the released object, not what the caller was handed.
|
||||
def record(result):
|
||||
delivered.append((result.success, result.data))
|
||||
|
||||
with patch.object(service, 'session', _CountingSession()):
|
||||
late['rid'] = service.submit_fetch_request(
|
||||
"nfl", 2025, URL, KEY, max_retries=0, callback=record
|
||||
)
|
||||
gate.set() # only now can the worker reach the commit
|
||||
_wait(service, late['rid'])
|
||||
|
||||
assert late.get('returned') is False, (
|
||||
"cancelled a request that had already committed"
|
||||
)
|
||||
assert cache.set.call_count == 1, "the commit itself was lost"
|
||||
assert delivered == [(True, PAYLOAD)], (
|
||||
"data reached the cache but the callbacks were suppressed -- "
|
||||
"every joined submitter is left waiting forever"
|
||||
)
|
||||
|
||||
|
||||
def test_a_failure_after_cancelling_stays_cancelled(service, cache):
|
||||
"""A cancelled request that then errors must not resurface as FAILED.
|
||||
|
||||
The except path overwrote CANCELLED with FAILED, and the callback gate in
|
||||
the finally block only suppresses callbacks for CANCELLED -- so cancelling
|
||||
a request that was about to time out delivered a spurious error callback.
|
||||
"""
|
||||
session = _BlockingFailingSession()
|
||||
delivered = []
|
||||
|
||||
with patch.object(service, 'session', session):
|
||||
rid = service.submit_fetch_request(
|
||||
"nfl", 2025, URL, KEY, max_retries=0, callback=delivered.append
|
||||
)
|
||||
# Assert the worker is inside the HTTP call before cancelling,
|
||||
# otherwise this silently degrades into the pre-start case and the
|
||||
# exception handler is never exercised.
|
||||
assert session.started.wait(timeout=5)
|
||||
assert service.cancel_request(rid) is True
|
||||
session.release.set()
|
||||
_wait(service, rid)
|
||||
|
||||
assert session.calls == 1, "the fetch never started, so nothing could fail"
|
||||
|
||||
assert delivered == [], "a cancelled request delivered a failure callback"
|
||||
assert service.get_request_status(rid) is FetchStatus.CANCELLED, (
|
||||
"a cancelled request that then errored was reported as FAILED"
|
||||
)
|
||||
@@ -0,0 +1,280 @@
|
||||
"""A delivered fetch payload must not stay resident on the stored result.
|
||||
|
||||
BackgroundDataService kept the fetched body on the FetchResult it filed in
|
||||
`completed_requests`, which is swept only hourly and capped at 500 entries by
|
||||
count. For status records that is free; for a season schedule it is not. NCAA
|
||||
football's 2026 schedule is 946 games, and on a 1GB Pi 3B+ the parsed payload
|
||||
measured ~90MB -- a tenth of the board's memory, pinned for an hour after the
|
||||
consumer had already been handed it.
|
||||
|
||||
The cache-hit path was the worse of the two. It runs once per update interval
|
||||
per sport, mints a fresh request_id each time, and hands back whatever the
|
||||
cache returns -- so a memory-tier miss (the tier is capped at 150 entries)
|
||||
re-parses the payload from disk into a genuinely new object. Those accumulate
|
||||
as separate copies rather than shared references, which is the staircase seen
|
||||
in the field: RSS stepping up ~90MB per sport as seasons loaded and never
|
||||
coming back down.
|
||||
|
||||
Releasing is safe because the payload is written to the cache under the
|
||||
request's cache_key before the result is built, and that is where consumers
|
||||
read it from -- the callback is handed the object directly and the plugins use
|
||||
it only in passing before reading the cache back.
|
||||
|
||||
Requests submitted *without* a callback keep their payload: polling
|
||||
get_result() is then the only way to collect it, so releasing would break that
|
||||
contract.
|
||||
|
||||
And the release must happen after EVERY callback, not after each one. Callers
|
||||
that joined an in-flight fetch share a single FetchResult, so releasing per
|
||||
delivery strips the payload out from under everyone still queued -- see
|
||||
TestJoinersAllGetTheData.
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from src.background_data_service import BackgroundDataService
|
||||
|
||||
|
||||
PAYLOAD = {"events": [{"id": f"g{i}"} for i in range(50)]}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cache():
|
||||
m = MagicMock()
|
||||
m.get.return_value = None
|
||||
m.set.return_value = None
|
||||
return m
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def service(cache):
|
||||
svc = BackgroundDataService(cache, max_workers=2, request_timeout=5)
|
||||
yield svc
|
||||
svc.shutdown(wait=False)
|
||||
|
||||
|
||||
def _wait(service, req_id, timeout=5):
|
||||
"""Wait for the result to be FILED.
|
||||
|
||||
Enough for anything that is true by the time the worker stores the result:
|
||||
its success flag, its error, the cache write that happened during the
|
||||
fetch.
|
||||
"""
|
||||
deadline = time.time() + timeout
|
||||
while not service.is_request_complete(req_id) and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
|
||||
|
||||
def _wait_for_release(service, req_id, timeout=5):
|
||||
"""Wait for the payload to be RELEASED, which is strictly later.
|
||||
|
||||
The worker files the result, then runs the callback, then releases. So
|
||||
is_request_complete() goes true while the callback still has not run --
|
||||
waiting on it alone leaves a window in which `seen` is empty and the
|
||||
payload is still resident, and the assertions race the worker. It passes
|
||||
in practice only because a one-line callback usually beats the 20ms poll.
|
||||
|
||||
Release happens after the callback returns, so a released payload also
|
||||
means the callback has finished: one wait covers both.
|
||||
"""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
result = service.get_result(req_id)
|
||||
if result is not None and result.data is None:
|
||||
return
|
||||
time.sleep(0.02)
|
||||
raise AssertionError(
|
||||
f"payload for {req_id} was never released (callback may not have run)")
|
||||
|
||||
|
||||
def _resp():
|
||||
r = Mock()
|
||||
r.json.return_value = PAYLOAD
|
||||
r.raise_for_status.return_value = None
|
||||
return r
|
||||
|
||||
|
||||
class TestFetchPath:
|
||||
def test_callback_receives_the_payload_then_it_is_released(self, service, cache):
|
||||
seen = {}
|
||||
|
||||
def callback(result):
|
||||
# The consumer's one look at the data happens here.
|
||||
seen['events'] = len(result.data['events'])
|
||||
|
||||
with patch.object(service.session, "get", return_value=_resp()):
|
||||
req_id = service.submit_fetch_request(
|
||||
sport="ncaa_fb", year=2026, url="https://example.com/s",
|
||||
cache_key="ncaa_fb_2026", callback=callback, max_retries=0,
|
||||
)
|
||||
_wait_for_release(service, req_id)
|
||||
|
||||
assert seen['events'] == 50, "callback must still be handed the payload"
|
||||
|
||||
stored = service.get_result(req_id)
|
||||
assert stored is not None
|
||||
assert stored.success is True
|
||||
assert stored.data is None, "payload must not stay on the stored result"
|
||||
|
||||
def test_nothing_is_lost_the_cache_holds_it(self, service, cache):
|
||||
with patch.object(service.session, "get", return_value=_resp()):
|
||||
req_id = service.submit_fetch_request(
|
||||
sport="ncaa_fb", year=2026, url="https://example.com/s",
|
||||
cache_key="ncaa_fb_2026", callback=lambda r: None, max_retries=0,
|
||||
)
|
||||
_wait(service, req_id)
|
||||
|
||||
cache.set.assert_called_once()
|
||||
key, written = cache.set.call_args[0][:2]
|
||||
assert key == "ncaa_fb_2026"
|
||||
assert written == PAYLOAD, "the payload must be persisted before release"
|
||||
|
||||
def test_without_a_callback_the_payload_is_kept(self, service, cache):
|
||||
# Polling get_result() is then the only delivery mechanism.
|
||||
with patch.object(service.session, "get", return_value=_resp()):
|
||||
req_id = service.submit_fetch_request(
|
||||
sport="nfl", year=2026, url="https://example.com/s",
|
||||
cache_key="nfl_2026", max_retries=0,
|
||||
)
|
||||
_wait(service, req_id)
|
||||
|
||||
assert service.get_result(req_id).data == PAYLOAD
|
||||
|
||||
def test_a_failed_fetch_still_records_its_error(self, service, cache):
|
||||
with patch.object(service.session, "get", side_effect=Exception("boom")):
|
||||
req_id = service.submit_fetch_request(
|
||||
sport="nfl", year=2026, url="https://example.com/s",
|
||||
cache_key="nfl_2026", callback=lambda r: None, max_retries=0,
|
||||
)
|
||||
_wait(service, req_id)
|
||||
|
||||
stored = service.get_result(req_id)
|
||||
assert stored.success is False
|
||||
assert stored.error is not None
|
||||
|
||||
|
||||
class TestCacheHitPath:
|
||||
def test_cache_hit_releases_after_the_callback(self, service, cache):
|
||||
cache.get.return_value = PAYLOAD
|
||||
seen = {}
|
||||
|
||||
req_id = service.submit_fetch_request(
|
||||
sport="ncaa_fb", year=2026, url="https://example.com/s",
|
||||
cache_key="ncaa_fb_2026",
|
||||
callback=lambda r: seen.update(events=len(r.data['events'])),
|
||||
)
|
||||
|
||||
assert seen['events'] == 50
|
||||
assert service.get_result(req_id).data is None
|
||||
|
||||
def test_repeated_cache_hits_do_not_accumulate_payloads(self, service, cache):
|
||||
# The staircase: one entry per update interval per sport, each one
|
||||
# potentially a freshly parsed copy after a memory-tier miss.
|
||||
cache.get.return_value = PAYLOAD
|
||||
|
||||
for _ in range(25):
|
||||
service.submit_fetch_request(
|
||||
sport="ncaa_fb", year=2026, url="https://example.com/s",
|
||||
cache_key="ncaa_fb_2026", callback=lambda r: None,
|
||||
)
|
||||
|
||||
retained = [r for r in service.completed_requests.values() if r.data is not None]
|
||||
assert retained == [], f"{len(retained)} payloads still resident"
|
||||
|
||||
def test_cache_hit_without_a_callback_is_unchanged(self, service, cache):
|
||||
cache.get.return_value = PAYLOAD
|
||||
req_id = service.submit_fetch_request(
|
||||
sport="nfl", year=2026, url="https://example.com/s",
|
||||
cache_key="nfl_2026",
|
||||
)
|
||||
assert service.get_result(req_id).data == PAYLOAD
|
||||
|
||||
|
||||
class TestJoinersAllGetTheData:
|
||||
"""Deduplicated callers share one FetchResult; releasing between them
|
||||
empties it for the rest.
|
||||
|
||||
Not a corner case. A sport's recent, upcoming and live managers all ask for
|
||||
the same season schedule, so the second and third are joiners on almost
|
||||
every cycle. Releasing inside the delivery loop handed the payload to
|
||||
whichever ran first and gave the others `result.data is None`.
|
||||
|
||||
The consequence was worse than a quiet degradation, because consumers do
|
||||
`result.data.get('events')`: they raised AttributeError, the delivery loop
|
||||
caught it, and the whole failure surfaced as a single
|
||||
"Error in callback for request ..." line while that manager silently never
|
||||
received its schedule.
|
||||
"""
|
||||
|
||||
class _BlockingSession:
|
||||
"""Holds the fetch open so a second submit lands while in flight."""
|
||||
|
||||
def __init__(self):
|
||||
self.release = threading.Event()
|
||||
self.started = threading.Event()
|
||||
|
||||
def get(self, *a, **k):
|
||||
self.started.set()
|
||||
self.release.wait(timeout=5)
|
||||
return _resp()
|
||||
|
||||
def test_every_joiner_is_handed_the_payload(self, service):
|
||||
session = self._BlockingSession()
|
||||
seen = {}
|
||||
|
||||
def record(name):
|
||||
# Read it the way the sport managers do. `result.data['events']`
|
||||
# would raise TypeError on None; `.get` raises AttributeError,
|
||||
# which is the error actually seen in the field.
|
||||
def cb(result):
|
||||
seen[name] = result.data.get('events') if result.data else None
|
||||
return cb
|
||||
|
||||
with patch.object(service, "session", session):
|
||||
first = service.submit_fetch_request(
|
||||
sport="nhl", year=2026, url="https://x/s", cache_key="nhl_2026",
|
||||
callback=record("first"), max_retries=0)
|
||||
assert session.started.wait(timeout=5)
|
||||
|
||||
joined = service.submit_fetch_request(
|
||||
sport="nhl", year=2026, url="https://x/s", cache_key="nhl_2026",
|
||||
callback=record("second"), max_retries=0)
|
||||
# Without coalescing these are two independent fetches that each
|
||||
# own their result, and the test proves nothing about sharing.
|
||||
assert joined == first, "the joiner should share the in-flight id"
|
||||
|
||||
session.release.set()
|
||||
_wait(service, first)
|
||||
|
||||
deadline = time.time() + 5
|
||||
while len(seen) < 2 and time.time() < deadline:
|
||||
time.sleep(0.02)
|
||||
|
||||
assert set(seen) == {"first", "second"}, f"both must be called, got {seen}"
|
||||
for name, events in seen.items():
|
||||
assert events is not None, (
|
||||
f"{name!r} was handed a released payload: the result was "
|
||||
f"emptied before every callback had been delivered")
|
||||
assert len(events) == 50, f"{name!r} got {events!r}"
|
||||
|
||||
def test_the_payload_is_still_released_once_they_have_all_had_it(self, service):
|
||||
"""The memory fix must survive the ordering fix."""
|
||||
session = self._BlockingSession()
|
||||
|
||||
with patch.object(service, "session", session):
|
||||
first = service.submit_fetch_request(
|
||||
sport="nhl", year=2026, url="https://x/s", cache_key="nhl_2026",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
assert session.started.wait(timeout=5)
|
||||
service.submit_fetch_request(
|
||||
sport="nhl", year=2026, url="https://x/s", cache_key="nhl_2026",
|
||||
callback=lambda r: None, max_retries=0)
|
||||
session.release.set()
|
||||
_wait_for_release(service, first)
|
||||
|
||||
stored = service.get_result(first)
|
||||
assert stored is not None and stored.data is None, (
|
||||
"the payload must still be dropped once every callback has run")
|
||||
@@ -458,3 +458,26 @@ class TestDiskCacheWriteEconomy:
|
||||
cache = DiskCache(cache_dir=str(tmp_path))
|
||||
cache.set("k", {"when": datetime(2026, 7, 12, 10, 30)})
|
||||
assert cache.get("k") == {"when": "2026-07-12T10:30:00"}
|
||||
|
||||
|
||||
# --- the ceiling has to hold between cleanup sweeps ---------------------------
|
||||
|
||||
def test_memory_cache_enforces_ceiling_on_every_write():
|
||||
"""_cleanup_memory_cache only runs every cleanup_interval seconds (300 by
|
||||
default). If set() accepted entries without bound in between, a burst could
|
||||
take the cache far past max_size -- which is the unbounded growth the limit
|
||||
exists to prevent, and on a 1GB board the difference between a bounded cache
|
||||
and a Pi that cannot fork.
|
||||
"""
|
||||
from src.cache.memory_cache import MemoryCache
|
||||
|
||||
cache = MemoryCache(max_size=150, cleanup_interval=300.0)
|
||||
for i in range(1000):
|
||||
cache.set(f"k{i}", {"v": i})
|
||||
|
||||
assert len(cache._cache) <= 150
|
||||
# The timestamp map has to be evicted alongside the values, or it becomes
|
||||
# the leak instead.
|
||||
assert len(cache._timestamps) <= 150
|
||||
assert cache.get("k999") is not None, "the newest write must survive"
|
||||
assert cache.get("k0") is None, "the oldest must be the one evicted"
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""A checkbox group must not post back options it cannot show.
|
||||
|
||||
The enum that lets the widget draw checkboxes is also what validates the
|
||||
saved value. When a league retires a team code -- OAK for the Athletics, ARI
|
||||
for the Coyotes -- or a schema drops an option, a config that still holds the
|
||||
old value has nothing to render for it. The value stayed in the hidden
|
||||
``_data`` input regardless, because that input is seeded from the stored array
|
||||
and only rebuilt by ``updateCheckboxGroupData()`` on change. Editing any other
|
||||
field on that plugin therefore posted the stale value back, the schema
|
||||
rejected it, and the save endpoint returned 400
|
||||
``CONFIG_VALIDATION_FAILED`` -- so the whole plugin became uneditable until
|
||||
the user worked out which invisible entry was at fault.
|
||||
|
||||
Runtime was never affected: plugin loading treats schema violations as
|
||||
warn/degrade, and the stale code already matched no team. Only the web UI
|
||||
blocked.
|
||||
|
||||
These tests render the checkbox-group block lifted *out of the shipped
|
||||
template*, following test_enum_option_labels.py, so they exercise the
|
||||
production expression rather than a copy that could drift from it.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from jinja2 import DictLoader, Environment
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
CONFIG_FORM = (PROJECT_ROOT / 'web_interface' / 'templates' / 'v3' / 'partials'
|
||||
/ 'plugin_config.html')
|
||||
|
||||
# The checkbox-group branch: from its `{% elif %}` guard through the sentinel
|
||||
# hidden input that closes it. Anchored on the guard so the match cannot run on
|
||||
# into a neighbouring widget branch.
|
||||
BLOCK_RE = re.compile(
|
||||
r"\{%\s*elif x_widget == 'checkbox-group'\s*%\}(.*?)"
|
||||
r"<input type=\"hidden\" name=\"\{\{ full_key \}\}\[\]\" value=\"\">",
|
||||
re.S,
|
||||
)
|
||||
|
||||
|
||||
def _shipped_block() -> str:
|
||||
"""Return the live checkbox-group block lifted from plugin_config.html."""
|
||||
source = CONFIG_FORM.read_text(encoding='utf-8')
|
||||
match = BLOCK_RE.search(source)
|
||||
assert match, (
|
||||
'could not find the checkbox-group block in plugin_config.html — the '
|
||||
'template changed shape and this guard needs updating'
|
||||
)
|
||||
block = match.group(1)
|
||||
assert 'data-option-value' in block, 'extracted the wrong branch'
|
||||
assert '{% elif' not in block, 'extraction ran past the checkbox-group branch'
|
||||
return block
|
||||
|
||||
|
||||
def _render(prop: dict, value=None) -> str:
|
||||
env = Environment(loader=DictLoader({'f': _shipped_block()}), autoescape=True)
|
||||
return env.get_template('f').render(
|
||||
prop=prop, value=value, field_id='fid', full_key='k'
|
||||
)
|
||||
|
||||
|
||||
def _submitted(html: str) -> list:
|
||||
"""The array the form will actually post: the hidden _data input."""
|
||||
match = re.search(r'id="fid_data"[^>]*\svalue=\'([^\']*)\'', html)
|
||||
assert match, f'hidden _data input not found in:\n{html}'
|
||||
return json.loads(match.group(1).replace(''', "'"))
|
||||
|
||||
|
||||
def _checked(html: str) -> list:
|
||||
return re.findall(r'data-option-value="([^"]+)"[^>]*checked', html)
|
||||
|
||||
|
||||
MLB = {'type': 'array', 'items': {'type': 'string', 'enum': ['NYY', 'BOS', 'ATH']},
|
||||
'x-widget': 'checkbox-group'}
|
||||
|
||||
|
||||
def test_a_retired_code_is_not_posted_back() -> None:
|
||||
"""The regression: OAK became ATH, and OAK used to ride along on save."""
|
||||
html = _render(MLB, ['NYY', 'OAK'])
|
||||
assert _submitted(html) == ['NYY'], 'stale value would still be submitted'
|
||||
|
||||
|
||||
def test_the_dropped_value_is_named_rather_than_vanishing() -> None:
|
||||
html = _render(MLB, ['NYY', 'OAK'])
|
||||
assert 'OAK' in html
|
||||
assert 'data-stale-options' in html
|
||||
|
||||
|
||||
def test_valid_values_are_untouched_and_still_checked() -> None:
|
||||
html = _render(MLB, ['NYY', 'ATH'])
|
||||
assert _submitted(html) == ['NYY', 'ATH']
|
||||
assert sorted(_checked(html)) == ['ATH', 'NYY']
|
||||
assert 'data-stale-options' not in html
|
||||
|
||||
|
||||
def test_an_all_stale_selection_clears_rather_than_blocking() -> None:
|
||||
html = _render(MLB, ['OAK', 'SD'])
|
||||
assert _submitted(html) == []
|
||||
|
||||
|
||||
def test_an_empty_enum_leaves_the_value_alone() -> None:
|
||||
"""No options means nothing to validate against — filtering would wipe it."""
|
||||
prop = {'type': 'array', 'items': {'type': 'string'}, 'x-widget': 'checkbox-group'}
|
||||
html = _render(prop, ['ANYTHING', 'GOES'])
|
||||
assert _submitted(html) == ['ANYTHING', 'GOES']
|
||||
|
||||
|
||||
def test_unset_value_falls_back_to_the_default() -> None:
|
||||
prop = dict(MLB, default=['BOS'])
|
||||
html = _render(prop, None)
|
||||
assert _submitted(html) == ['BOS']
|
||||
assert _checked(html) == ['BOS']
|
||||
@@ -0,0 +1,143 @@
|
||||
"""GET /config/main must not hand out credentials.
|
||||
|
||||
The endpoint returned the raw config to anyone who could reach the port, and
|
||||
this web interface has no authentication of any kind. Measured against a live
|
||||
rig, an unauthenticated request returned:
|
||||
|
||||
github.api_token 40 chars
|
||||
incoming-packages.ha_token 183 chars
|
||||
jellyfin-now-playing.api_key 32 chars
|
||||
ledmatrix-weather.api_key 32 chars
|
||||
on-air.mqtt_password 8 chars
|
||||
youtube.api_key 20 chars
|
||||
youtube-stats.api_key 39 chars
|
||||
|
||||
A GitHub token and a Home Assistant long-lived token among them.
|
||||
|
||||
The x-secret masking the plugin config endpoints use does not apply here: this
|
||||
endpoint never consults a schema, and core keys such as github.api_token have
|
||||
no schema to carry the marker. Several of those fields *are* tagged x-secret in
|
||||
their plugin's schema and were still returned in full, which is what makes the
|
||||
schema route the wrong one to rely on for this endpoint.
|
||||
|
||||
Matching on field name is blunt. For a whole-config dump it is the right
|
||||
default: anything named like a credential should not leave the process, and a
|
||||
new plugin that adds a differently-shaped secret is covered without anyone
|
||||
remembering to tag it.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from web_interface.blueprints.api_v3 import (
|
||||
_looks_like_a_credential,
|
||||
_redact_credentials,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", [
|
||||
"password", "mqtt_password", "opensky_password", "passwd",
|
||||
"api_key", "apikey", "API_KEY", "flightaware_api_key",
|
||||
"token", "ha_token", "api_token", "access_token",
|
||||
"secret", "client_secret", "spotify_client_secret",
|
||||
"access_key", "private_key",
|
||||
])
|
||||
def test_credential_names_are_recognised(name):
|
||||
assert _looks_like_a_credential(name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", [
|
||||
"timezone", "city", "brightness", "enabled", "update_interval",
|
||||
"favorite_teams", "display_duration", "keyword",
|
||||
])
|
||||
def test_ordinary_names_are_left_alone(name):
|
||||
assert not _looks_like_a_credential(name)
|
||||
|
||||
|
||||
def test_the_measured_leak_is_closed():
|
||||
"""The exact shape taken off the rig."""
|
||||
config = {
|
||||
"github": {"api_token": "ghp_" + "x" * 36},
|
||||
"incoming-packages": {"ha_token": "y" * 183, "enabled": True},
|
||||
"jellyfin-now-playing": {"api_key": "z" * 32},
|
||||
"on-air": {"mqtt_password": "hunter22"},
|
||||
"youtube": {"api_key": "k" * 20},
|
||||
"timezone": "America/New_York",
|
||||
}
|
||||
out = _redact_credentials(config)
|
||||
assert out["github"]["api_token"] == ""
|
||||
assert out["incoming-packages"]["ha_token"] == ""
|
||||
assert out["jellyfin-now-playing"]["api_key"] == ""
|
||||
assert out["on-air"]["mqtt_password"] == ""
|
||||
assert out["youtube"]["api_key"] == ""
|
||||
# Everything else survives, or the config editor breaks.
|
||||
assert out["timezone"] == "America/New_York"
|
||||
assert out["incoming-packages"]["enabled"] is True
|
||||
|
||||
|
||||
def test_nested_and_listed_credentials_are_reached():
|
||||
config = {"a": {"b": {"c": {"password": "p"}}},
|
||||
"feeds": [{"name": "x", "api_key": "k"}, {"name": "y"}]}
|
||||
out = _redact_credentials(config)
|
||||
assert out["a"]["b"]["c"]["password"] == ""
|
||||
assert out["feeds"][0]["api_key"] == ""
|
||||
assert out["feeds"][0]["name"] == "x"
|
||||
|
||||
|
||||
def test_the_original_is_not_mutated():
|
||||
"""The caller holds the live config; redaction must not edit it in place."""
|
||||
config = {"github": {"api_token": "keepme"}}
|
||||
_redact_credentials(config)
|
||||
assert config["github"]["api_token"] == "keepme"
|
||||
|
||||
|
||||
def test_a_credential_shaped_container_is_still_walked():
|
||||
"""`secrets: {...}` is a section name, not a value to blank."""
|
||||
config = {"secrets": {"api_key": "k", "note": "keep"}}
|
||||
out = _redact_credentials(config)
|
||||
assert out["secrets"]["api_key"] == ""
|
||||
assert out["secrets"]["note"] == "keep"
|
||||
|
||||
|
||||
def test_non_dict_input_passes_through():
|
||||
assert _redact_credentials("plain") == "plain"
|
||||
assert _redact_credentials(7) == 7
|
||||
assert _redact_credentials(None) is None
|
||||
|
||||
|
||||
def test_the_endpoint_itself_redacts():
|
||||
"""Through the view function, not the helper.
|
||||
|
||||
The helper tests above all passed with the route still returning
|
||||
`config` -- reverting the one line that calls the redactor changed
|
||||
nothing, because nothing exercised the route. A property asserted on a
|
||||
helper is not a property asserted on the endpoint, and it is the endpoint
|
||||
that is exposed to the network.
|
||||
"""
|
||||
import json as _json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import flask
|
||||
|
||||
from web_interface.blueprints import api_v3 as mod
|
||||
|
||||
raw = {"github": {"api_token": "ghp_secret_value"},
|
||||
"timezone": "America/New_York"}
|
||||
|
||||
manager = MagicMock()
|
||||
manager.load_config.return_value = raw
|
||||
previous = getattr(mod.api_v3, "config_manager", None)
|
||||
mod.api_v3.config_manager = manager
|
||||
|
||||
app = flask.Flask(__name__)
|
||||
try:
|
||||
with app.test_request_context("/config/main"):
|
||||
response = mod.get_main_config()
|
||||
payload = response.get_json() if hasattr(response, "get_json") else _json.loads(response[0].data)
|
||||
finally:
|
||||
mod.api_v3.config_manager = previous
|
||||
|
||||
data = payload["data"]
|
||||
assert data["github"]["api_token"] == "", (
|
||||
"the endpoint returned the token; the redactor is not wired in")
|
||||
assert data["timezone"] == "America/New_York"
|
||||
# And the config the manager handed over is untouched.
|
||||
assert raw["github"]["api_token"] == "ghp_secret_value"
|
||||
@@ -0,0 +1,122 @@
|
||||
"""The gate must read the core version from disk, not from its own import.
|
||||
|
||||
`from src import __version__` binds whatever the process loaded at start. The
|
||||
web UI is a long-lived service of its own (ledmatrix-web.service), and updating
|
||||
the core replaces files on disk without restarting it -- the update route says
|
||||
so and asks the user to restart, but its prompt named only the *display*
|
||||
service, so a user who followed it left the web process holding the old number.
|
||||
|
||||
The plugin store's gate lives in that web process, so being stale by exactly
|
||||
one release is the case that bites: every plugin flooring on the release you
|
||||
just installed is refused, with a message blaming a core version that is
|
||||
already correct on disk.
|
||||
|
||||
Observed on hardware: after updating a rig to 3.3.0 and restarting only the
|
||||
display service, all eight sports scoreboards were refused with
|
||||
"supports LEDMatrix >=3.3.0, but this system is running 3.2.0" while
|
||||
src/__init__.py on that machine read 3.3.0.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from src.plugin_system import compatibility
|
||||
|
||||
|
||||
class TestCurrentCoreVersion:
|
||||
def test_it_reads_the_file_rather_than_the_imported_value(self, monkeypatch, tmp_path):
|
||||
# Simulate a process whose import predates the update: the module
|
||||
# object says 3.2.0 while the file on disk says 3.3.0.
|
||||
import src
|
||||
monkeypatch.setattr(src, "__version__", "3.2.0")
|
||||
fake = tmp_path / "__init__.py"
|
||||
fake.write_text('__version__ = "3.3.0"\n', encoding="utf-8")
|
||||
monkeypatch.setattr(compatibility, "_VERSION_FILE", fake)
|
||||
assert compatibility.current_core_version() == "3.3.0"
|
||||
|
||||
def test_it_matches_the_real_file_by_default(self):
|
||||
import src
|
||||
assert compatibility.current_core_version() == src.__version__
|
||||
|
||||
@pytest.mark.parametrize("body", [
|
||||
"__version__ = '3.4.1'\n",
|
||||
'__version__="3.4.1"\n',
|
||||
'"""doc"""\n\n__version__ = "3.4.1" # trailing comment\n',
|
||||
])
|
||||
def test_it_tolerates_the_ways_that_line_gets_written(self, monkeypatch, tmp_path, body):
|
||||
fake = tmp_path / "__init__.py"
|
||||
fake.write_text(body, encoding="utf-8")
|
||||
monkeypatch.setattr(compatibility, "_VERSION_FILE", fake)
|
||||
assert compatibility.current_core_version() == "3.4.1"
|
||||
|
||||
def test_a_missing_file_falls_back_to_the_import(self, monkeypatch, tmp_path):
|
||||
# Never worse than before: an unreadable file returns what the old
|
||||
# code would have returned.
|
||||
import src
|
||||
monkeypatch.setattr(src, "__version__", "3.2.0")
|
||||
monkeypatch.setattr(compatibility, "_VERSION_FILE", tmp_path / "gone.py")
|
||||
assert compatibility.current_core_version() == "3.2.0"
|
||||
|
||||
def test_a_file_without_the_line_falls_back(self, monkeypatch, tmp_path):
|
||||
import src
|
||||
monkeypatch.setattr(src, "__version__", "3.2.0")
|
||||
fake = tmp_path / "__init__.py"
|
||||
fake.write_text("# no version here\n", encoding="utf-8")
|
||||
monkeypatch.setattr(compatibility, "_VERSION_FILE", fake)
|
||||
assert compatibility.current_core_version() == "3.2.0"
|
||||
|
||||
def test_it_never_raises(self, monkeypatch, tmp_path):
|
||||
# This runs on the install path; an exception here would surface as a
|
||||
# failed update rather than a version mismatch.
|
||||
bad = tmp_path / "__init__.py"
|
||||
bad.write_bytes(b"\xff\xfe\x00 not utf-8 \xff")
|
||||
monkeypatch.setattr(compatibility, "_VERSION_FILE", bad)
|
||||
assert isinstance(compatibility.current_core_version(), str)
|
||||
|
||||
|
||||
class TestTheBugItFixes:
|
||||
def test_a_stale_import_no_longer_refuses_a_compatible_plugin(self, monkeypatch, tmp_path):
|
||||
"""The exact hardware failure, as a test."""
|
||||
import src
|
||||
monkeypatch.setattr(src, "__version__", "3.2.0") # what the process holds
|
||||
fake = tmp_path / "__init__.py"
|
||||
fake.write_text('__version__ = "3.3.0"\n', encoding="utf-8") # what is on disk
|
||||
monkeypatch.setattr(compatibility, "_VERSION_FILE", fake)
|
||||
|
||||
manifest = {"name": "Hockey Scoreboard", "min_ledmatrix_version": "3.3.0"}
|
||||
|
||||
stale_ok, _ = compatibility.check(manifest, src.__version__)
|
||||
assert stale_ok is False, "precondition: the stale value is what refused it"
|
||||
|
||||
fresh_ok, reason = compatibility.check(
|
||||
manifest, compatibility.current_core_version())
|
||||
assert fresh_ok is True, f"the disk version must allow it, got: {reason}"
|
||||
|
||||
def test_it_still_refuses_when_the_core_really_is_too_old(self, monkeypatch, tmp_path):
|
||||
# The gate must not become permissive: a genuinely old core still says no.
|
||||
fake = tmp_path / "__init__.py"
|
||||
fake.write_text('__version__ = "3.2.0"\n', encoding="utf-8")
|
||||
monkeypatch.setattr(compatibility, "_VERSION_FILE", fake)
|
||||
ok, reason = compatibility.check(
|
||||
{"name": "Hockey", "min_ledmatrix_version": "3.3.0"},
|
||||
compatibility.current_core_version())
|
||||
assert ok is False
|
||||
assert "3.2.0" in (reason or "")
|
||||
|
||||
|
||||
class TestCallSites:
|
||||
@pytest.mark.parametrize("module", [
|
||||
"src.plugin_system.store_manager",
|
||||
"src.plugin_system.plugin_loader",
|
||||
])
|
||||
def test_no_gate_binds_the_version_at_import(self, module):
|
||||
"""Catch a future call site reintroducing the stale read."""
|
||||
import inspect
|
||||
mod = importlib.import_module(module)
|
||||
source = inspect.getsource(mod)
|
||||
assert "from src import __version__ as core_version" not in source, (
|
||||
f"{module} binds __version__ at import; use "
|
||||
f"compatibility.current_core_version() so a long-lived process "
|
||||
f"sees a core update.")
|
||||
@@ -6,6 +6,7 @@ These tests cover the reconcile path that loads/unloads plugins and rebuilds
|
||||
the dispatch maps on the main thread when the enabled set changes.
|
||||
"""
|
||||
|
||||
import copy
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
||||
@@ -253,3 +254,182 @@ class TestEnabledSetChanged:
|
||||
{"a": {"enabled": True, "duration": 30}},
|
||||
{"a": {"enabled": True, "duration": 45}},
|
||||
) is False
|
||||
|
||||
|
||||
class TestEnabledPluginNotRunning:
|
||||
"""A plugin that fails validate_config() is enabled but absent, and the
|
||||
config edit that fixes it is nested inside the plugin's own section -- so
|
||||
the top-level ``enabled`` comparison never sees it. These cover the second
|
||||
gate that queues a reconcile in that case.
|
||||
"""
|
||||
|
||||
def test_nested_edit_is_invisible_to_the_enabled_set_check(self, test_display_controller):
|
||||
"""The original gate: proves why a second one is needed."""
|
||||
controller = test_display_controller
|
||||
old = {"hockey-scoreboard": {"enabled": True, "nhl": {"enabled": False}}}
|
||||
new = {"hockey-scoreboard": {"enabled": True, "nhl": {"enabled": True}}}
|
||||
# Enabling a league changes no top-level flag.
|
||||
assert controller._enabled_set_changed(old, new) is False
|
||||
|
||||
def test_queues_reconcile_when_enabled_plugin_is_absent(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}}
|
||||
controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"}
|
||||
controller.plugin_display_modes = {} # failed to load
|
||||
cfg = {"hockey-scoreboard": {"enabled": True, "nhl": {"enabled": True}}}
|
||||
assert controller._enabled_plugin_not_running(cfg) is True
|
||||
|
||||
def test_quiet_when_every_enabled_plugin_is_running(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}}
|
||||
controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"}
|
||||
controller.plugin_display_modes = {"hockey-scoreboard": ["nhl"]}
|
||||
cfg = {"hockey-scoreboard": {"enabled": True}}
|
||||
assert controller._enabled_plugin_not_running(cfg) is False
|
||||
|
||||
def test_disabled_plugin_does_not_queue(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}}
|
||||
controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"}
|
||||
controller.plugin_display_modes = {}
|
||||
cfg = {"hockey-scoreboard": {"enabled": False}}
|
||||
assert controller._enabled_plugin_not_running(cfg) is False
|
||||
|
||||
def test_non_plugin_sections_do_not_queue(self, test_display_controller):
|
||||
"""``schedule``/``display`` carry their own ``enabled`` and are never
|
||||
in plugin_display_modes -- without the manifest check they would queue
|
||||
a reconcile, and therefore a filesystem scan, on every config save."""
|
||||
controller = test_display_controller
|
||||
controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}}
|
||||
controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"}
|
||||
controller.plugin_display_modes = {"hockey-scoreboard": ["nhl"]}
|
||||
cfg = {
|
||||
"hockey-scoreboard": {"enabled": True},
|
||||
"schedule": {"enabled": True},
|
||||
"display": {"enabled": True},
|
||||
}
|
||||
assert controller._enabled_plugin_not_running(cfg) is False
|
||||
|
||||
def test_non_dict_section_is_ignored(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}}
|
||||
controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"}
|
||||
controller.plugin_display_modes = {}
|
||||
assert controller._enabled_plugin_not_running({"hockey-scoreboard": "nonsense"}) is False
|
||||
|
||||
def test_no_plugin_manager_is_quiet(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller.plugin_manager = None
|
||||
assert controller._enabled_plugin_not_running({"x": {"enabled": True}}) is False
|
||||
|
||||
|
||||
class TestReconcileQueuedThroughSubscriber:
|
||||
"""End-to-end through the real config-change subscriber, not the helper.
|
||||
|
||||
Without the second gate this is the four-day-outage path: the plugin is
|
||||
enabled, absent, and the save that enables its league sets no flag.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _subscriber(controller):
|
||||
subs = controller.config_service._subscribers['*']
|
||||
for cb in subs:
|
||||
if getattr(cb, '__name__', '') == '_controller_config_change':
|
||||
return cb
|
||||
raise AssertionError(f"controller subscriber not found among {subs}")
|
||||
|
||||
@staticmethod
|
||||
def _configs(controller, plugin_section_old, plugin_section_new):
|
||||
"""Build two full configs differing only inside the plugin section --
|
||||
the subscriber refreshes its cache from these, so they must be real."""
|
||||
base = copy.deepcopy(controller.config)
|
||||
old = copy.deepcopy(base)
|
||||
new = copy.deepcopy(base)
|
||||
old["hockey-scoreboard"] = plugin_section_old
|
||||
new["hockey-scoreboard"] = plugin_section_new
|
||||
return old, new
|
||||
|
||||
def test_nested_edit_queues_reconcile_for_absent_plugin(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}}
|
||||
controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"}
|
||||
controller.plugin_display_modes = {} # validate_config() said False
|
||||
controller._pending_plugin_reconcile = False
|
||||
|
||||
old, new = self._configs(
|
||||
controller,
|
||||
{"enabled": True, "nhl": {"enabled": False}},
|
||||
{"enabled": True, "nhl": {"enabled": True}},
|
||||
)
|
||||
# The original gate is blind to this edit ...
|
||||
assert controller._enabled_set_changed(old, new) is False
|
||||
self._subscriber(controller)(old, new)
|
||||
# ... but the reconcile is queued anyway.
|
||||
assert controller._pending_plugin_reconcile is True
|
||||
|
||||
def test_steady_state_does_not_queue_reconcile(self, test_display_controller):
|
||||
"""Everything enabled is running: an unrelated edit must not queue a
|
||||
reconcile, or every config save drags a filesystem scan onto the
|
||||
render thread."""
|
||||
controller = test_display_controller
|
||||
controller.plugin_manager.plugin_manifests = {"hockey-scoreboard": {}}
|
||||
controller.plugin_manager.discovered_plugin_ids.return_value = {"hockey-scoreboard"}
|
||||
controller.plugin_display_modes = {"hockey-scoreboard": ["nhl"]}
|
||||
controller._pending_plugin_reconcile = False
|
||||
|
||||
old, new = self._configs(
|
||||
controller,
|
||||
{"enabled": True, "scroll_speed": 1},
|
||||
{"enabled": True, "scroll_speed": 2},
|
||||
)
|
||||
self._subscriber(controller)(old, new)
|
||||
|
||||
assert controller._pending_plugin_reconcile is False
|
||||
|
||||
|
||||
class TestPendingReconcileNotLost:
|
||||
"""A config change arriving *during* reconcile must not be discarded.
|
||||
|
||||
The flag used to be cleared after a successful reconcile. Reconcile has
|
||||
already read its config by then, so that clear erased a request it never
|
||||
served and the newest config never reconciled -- the same "my save did
|
||||
nothing" symptom this path exists to prevent.
|
||||
"""
|
||||
|
||||
def test_request_arriving_during_reconcile_survives(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller._pending_plugin_reconcile = True
|
||||
|
||||
def reconcile_and_race():
|
||||
# The watcher thread queues another change while we are mid-flight.
|
||||
with controller._reconcile_flag_lock:
|
||||
controller._pending_plugin_reconcile = True
|
||||
return True
|
||||
|
||||
controller._reconcile_enabled_plugins = reconcile_and_race
|
||||
controller._service_pending_reconcile()
|
||||
|
||||
assert controller._pending_plugin_reconcile is True, \
|
||||
"a config change landing during reconcile was discarded"
|
||||
|
||||
def test_flag_cleared_on_a_quiet_success(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller._pending_plugin_reconcile = True
|
||||
controller._reconcile_enabled_plugins = lambda: True
|
||||
controller._service_pending_reconcile()
|
||||
assert controller._pending_plugin_reconcile is False
|
||||
|
||||
def test_retryable_failure_rearms(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller._pending_plugin_reconcile = True
|
||||
controller._reconcile_enabled_plugins = lambda: False
|
||||
controller._service_pending_reconcile()
|
||||
assert controller._pending_plugin_reconcile is True
|
||||
|
||||
def test_no_reconcile_when_nothing_pending(self, test_display_controller):
|
||||
controller = test_display_controller
|
||||
controller._pending_plugin_reconcile = False
|
||||
calls = []
|
||||
controller._reconcile_enabled_plugins = lambda: calls.append(1) or True
|
||||
controller._service_pending_reconcile()
|
||||
assert calls == []
|
||||
|
||||
@@ -58,7 +58,7 @@ def repos(tmp_path):
|
||||
def test_branch_with_upstream_uses_a_plain_pull(repos):
|
||||
args, note, error = resolve_pull_command(str(repos))
|
||||
assert error is None
|
||||
assert args == ['git', 'pull', '--rebase']
|
||||
assert args == ['git', 'pull', '--rebase', '--autostash']
|
||||
assert note == ''
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ def test_branch_without_upstream_falls_back_to_origin_branch(repos):
|
||||
|
||||
args, note, error = resolve_pull_command(str(repos))
|
||||
assert error is None
|
||||
assert args == ['git', 'pull', '--rebase', 'origin', 'audit']
|
||||
assert args == ['git', 'pull', '--rebase', '--autostash', 'origin', 'audit']
|
||||
assert 'audit' in note
|
||||
|
||||
|
||||
@@ -155,7 +155,7 @@ def test_switching_attaches_tracking_so_pull_needs_no_fallback(repos):
|
||||
|
||||
args, note, error = resolve_pull_command(str(repos))
|
||||
assert error is None
|
||||
assert args == ['git', 'pull', '--rebase']
|
||||
assert args == ['git', 'pull', '--rebase', '--autostash']
|
||||
assert note == ''
|
||||
|
||||
|
||||
@@ -200,3 +200,44 @@ def test_stash_option_lets_the_switch_through_and_keeps_the_work(repos):
|
||||
assert _git('branch', '--show-current', cwd=repos).stdout.strip() == 'other'
|
||||
# The edit is not lost — it is on the stash.
|
||||
assert 'switch to other' in _git('stash', 'list', cwd=repos).stdout
|
||||
|
||||
|
||||
class TestInstallerDoesNotBlockTheUpdateButton:
|
||||
"""first_time_install.sh chmods scripts that git tracked as 644.
|
||||
|
||||
With core.fileMode true -- the default on Linux -- that leaves five
|
||||
permanently modified tracked files on every machine that ran the
|
||||
installer, and `git pull --rebase` refuses to start:
|
||||
|
||||
error: cannot pull with rebase: You have unstaged changes.
|
||||
|
||||
Tracking them as executable makes the installer's chmod a no-op.
|
||||
"""
|
||||
|
||||
CHMODDED = [
|
||||
'first_time_install.sh',
|
||||
'start_display.sh',
|
||||
'stop_display.sh',
|
||||
'scripts/install/install_service.sh',
|
||||
'scripts/install/install_web_service.sh',
|
||||
]
|
||||
|
||||
def test_scripts_the_installer_chmods_are_tracked_executable(self):
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
out = subprocess.run(['git', 'ls-files', '-s', *self.CHMODDED],
|
||||
capture_output=True, text=True, cwd=str(root)).stdout
|
||||
modes = {line.split()[3]: line.split()[0] for line in out.strip().split('\n') if line}
|
||||
# git ls-files says nothing about a path it does not track, so a
|
||||
# renamed or deleted script would simply be absent here and the mode
|
||||
# check below would pass over it silently.
|
||||
untracked = sorted(set(self.CHMODDED) - set(modes))
|
||||
assert not untracked, (
|
||||
f"{untracked} are chmodded by the installer but not tracked by "
|
||||
"git, so their mode cannot be asserted at all")
|
||||
non_exec = sorted(f for f, m in modes.items() if m != '100755')
|
||||
assert not non_exec, (
|
||||
f"{non_exec} are chmodded by the installer but tracked non-executable, "
|
||||
"so every install leaves the working tree dirty and the update "
|
||||
"button cannot pull")
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
"""A healthy plugin must not rewrite its health record every cycle.
|
||||
|
||||
Every successful plugin update called record_success(), which persisted the
|
||||
record unconditionally. In steady state the only fields that had changed were
|
||||
total_successes and last_success_time -- a counter and a timestamp that
|
||||
health_monitor reads for display and that nothing reads back after a restart.
|
||||
|
||||
Measured on a rig running 24 plugins: about 17 health-file rewrites a minute,
|
||||
roughly 25,000 a day. Each is ~400 bytes, but they land on an SD card where
|
||||
the unit of cost is an erase-block cycle, not the byte count, and where wear is
|
||||
what eventually kills the card.
|
||||
|
||||
The circuit breaker still needs its own state to survive a restart, so the
|
||||
write is kept for exactly the fields it is rebuilt from -- and a failure, a
|
||||
circuit opening, or a recovery must still be written the moment it happens.
|
||||
"""
|
||||
import time
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
|
||||
from src.plugin_system.plugin_health import PluginHealthTracker, CircuitState
|
||||
|
||||
|
||||
class _Cache:
|
||||
"""Counts writes; serves back whatever was last written.
|
||||
|
||||
Both directions deep-copy, so this behaves like a real cache that
|
||||
serialises through a file. Storing by reference let the tracker keep
|
||||
mutating the object already in the store, so a record could appear to
|
||||
have been persisted when no write ever happened -- which is precisely
|
||||
what test_durable_state_survives_a_restart is supposed to detect.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.store = {}
|
||||
self.writes = 0
|
||||
|
||||
def set(self, key, data, ttl=None, **kwargs):
|
||||
self.writes += 1
|
||||
self.store[key] = copy.deepcopy(data)
|
||||
|
||||
def get(self, key, max_age=None, memory_ttl=None, **kwargs):
|
||||
return copy.deepcopy(self.store.get(key))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tracker():
|
||||
cache = _Cache()
|
||||
t = PluginHealthTracker(cache_manager=cache)
|
||||
return t, cache
|
||||
|
||||
|
||||
def test_steady_state_success_stops_writing(tracker):
|
||||
"""The regression: 100 healthy cycles used to be 100 SD writes."""
|
||||
t, cache = tracker
|
||||
t.record_success("weather")
|
||||
first = cache.writes
|
||||
for _ in range(100):
|
||||
t.record_success("weather")
|
||||
assert cache.writes == first, (
|
||||
f"{cache.writes - first} redundant writes across 100 healthy cycles"
|
||||
)
|
||||
|
||||
|
||||
def test_the_counters_are_still_accurate_in_memory(tracker):
|
||||
"""Skipping the write must not skip the bookkeeping."""
|
||||
t, _ = tracker
|
||||
for _ in range(10):
|
||||
t.record_success("weather")
|
||||
state = t.get_health_state("weather")
|
||||
assert state["total_successes"] == 10
|
||||
assert state["last_success_time"] is not None
|
||||
assert state["last_success_time"] <= time.time()
|
||||
|
||||
|
||||
def test_a_failure_is_written_immediately(tracker):
|
||||
t, cache = tracker
|
||||
t.record_success("weather")
|
||||
before = cache.writes
|
||||
t.record_failure("weather", RuntimeError("boom"))
|
||||
assert cache.writes > before, "a failure must reach disk"
|
||||
|
||||
|
||||
def test_recovery_after_failure_is_written(tracker):
|
||||
"""consecutive_failures returning to 0 is durable state changing."""
|
||||
t, cache = tracker
|
||||
t.record_failure("weather", RuntimeError("boom"))
|
||||
before = cache.writes
|
||||
t.record_success("weather")
|
||||
assert cache.writes > before, "recovery must reach disk"
|
||||
assert t.get_health_state("weather")["consecutive_failures"] == 0
|
||||
|
||||
|
||||
def test_a_closing_circuit_is_written(tracker):
|
||||
"""Success in half-open closes the circuit -- that must survive a restart."""
|
||||
t, cache = tracker
|
||||
state = t.get_health_state("weather")
|
||||
state["circuit_state"] = CircuitState.HALF_OPEN.value
|
||||
state["half_open_start_time"] = time.time()
|
||||
before = cache.writes
|
||||
t.record_success("weather")
|
||||
assert cache.writes > before, "a circuit transition must reach disk"
|
||||
assert t.get_health_state("weather")["circuit_state"] == CircuitState.CLOSED.value
|
||||
|
||||
|
||||
def test_durable_state_survives_a_restart(tracker):
|
||||
"""What is skipped must genuinely not matter to the breaker."""
|
||||
t, cache = tracker
|
||||
for _ in range(3):
|
||||
t.record_failure("weather", RuntimeError("boom"))
|
||||
for _ in range(50):
|
||||
t.record_success("weather")
|
||||
|
||||
revived = PluginHealthTracker(cache_manager=cache)
|
||||
state = revived.get_health_state("weather")
|
||||
assert state["consecutive_failures"] == 0
|
||||
assert state["circuit_state"] == CircuitState.CLOSED.value
|
||||
@@ -13,6 +13,7 @@ need root and mutate the system, so they are exercised manually instead.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -31,6 +32,16 @@ def run_lib(snippet: str, env: dict | None = None) -> subprocess.CompletedProces
|
||||
)
|
||||
|
||||
|
||||
def _fstype_of(path: object) -> str:
|
||||
"""Filesystem type backing ``path``, via the same tool the helper uses."""
|
||||
result = subprocess.run(
|
||||
["findmnt", "-no", "FSTYPE", "--target", str(path)],
|
||||
capture_output=True, text=True,
|
||||
env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin"},
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def call(fn: str, *args: object, env: dict | None = None) -> str:
|
||||
joined = " ".join(str(a) for a in args)
|
||||
result = run_lib(f"{fn} {joined}", env=env)
|
||||
@@ -195,8 +206,29 @@ class TestOomDetection:
|
||||
|
||||
class TestDiskBackedTmpdir:
|
||||
def test_returns_nothing_when_tmpdir_is_already_disk_backed(self, tmp_path):
|
||||
# tmp_path is on the regular filesystem, so the default must be kept.
|
||||
assert call("lm_disk_backed_tmpdir", env={"TMPDIR": str(tmp_path)}) == ""
|
||||
# Do not assume tmp_path is disk-backed. Debian 13 -- the platform this
|
||||
# helper exists for -- mounts /tmp as tmpfs, and pytest puts tmp_path
|
||||
# under /tmp, so this asserted against a *memory*-backed directory and
|
||||
# failed on the target platform while the helper behaved exactly as
|
||||
# designed. Search for a directory whose backing store is really disk.
|
||||
scratch = None
|
||||
disk_backed = None
|
||||
for candidate in (tmp_path, Path("/var/tmp"), LIB.parent):
|
||||
if _fstype_of(candidate) not in ("tmpfs", "ramfs", ""):
|
||||
if candidate is tmp_path:
|
||||
disk_backed = candidate
|
||||
else:
|
||||
scratch = Path(tempfile.mkdtemp(dir=str(candidate)))
|
||||
disk_backed = scratch
|
||||
break
|
||||
if disk_backed is None:
|
||||
pytest.skip("no disk-backed directory available to test against")
|
||||
try:
|
||||
assert call("lm_disk_backed_tmpdir",
|
||||
env={"TMPDIR": str(disk_backed)}) == ""
|
||||
finally:
|
||||
if scratch is not None:
|
||||
scratch.rmdir()
|
||||
|
||||
def test_redirects_away_from_a_memory_backed_tmpdir(self):
|
||||
# Debian 13 mounts /tmp as tmpfs, which would otherwise hold the whole
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Log lines must reach the journal with their real severity.
|
||||
|
||||
Everything this process writes to stdout lands in the journal as PRIORITY=6,
|
||||
whatever the Python level was, because journald has no other signal. Measured
|
||||
on a live rig over 24 hours: 55 lines containing " - ERROR - " and 13
|
||||
containing " - WARNING - ", every one of them recorded as informational. So
|
||||
|
||||
journalctl -p err -u ledmatrix
|
||||
|
||||
returned nothing while errors were being logged, and anyone triaging has to
|
||||
grep the message text instead. That is slower and it is wrong: a search for
|
||||
"oom" also matches the radar logging "zoom=9", which is exactly the false
|
||||
positive it produced during this audit.
|
||||
|
||||
systemd reads a leading "<N>" on each stdout line and uses it as the priority
|
||||
(sd-daemon(3)), so this needs no extra dependency -- and it must only be
|
||||
applied when systemd is actually reading, or the prefixes become literal noise
|
||||
in a terminal, the emulator, and test output.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from src.logging_config import JournalPriorityFormatter, _SYSLOG_PRIORITY, _under_systemd
|
||||
|
||||
|
||||
class _Plain(logging.Formatter):
|
||||
def format(self, record):
|
||||
return record.getMessage()
|
||||
|
||||
|
||||
def _record(level, msg="hello"):
|
||||
return logging.LogRecord("t", level, "f.py", 1, msg, None, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("level,expected", [
|
||||
(logging.CRITICAL, 2),
|
||||
(logging.ERROR, 3),
|
||||
(logging.WARNING, 4),
|
||||
(logging.INFO, 6),
|
||||
(logging.DEBUG, 7),
|
||||
])
|
||||
def test_each_level_maps_to_its_syslog_priority(level, expected):
|
||||
out = JournalPriorityFormatter(_Plain()).format(_record(level))
|
||||
assert out.startswith(f"<{expected}>"), out
|
||||
assert _SYSLOG_PRIORITY[level] == expected
|
||||
|
||||
|
||||
def test_error_and_info_are_distinguishable():
|
||||
"""The whole point: journalctl -p err must be able to tell them apart."""
|
||||
fmt = JournalPriorityFormatter(_Plain())
|
||||
assert fmt.format(_record(logging.ERROR))[:3] != fmt.format(_record(logging.INFO))[:3]
|
||||
|
||||
|
||||
def test_every_line_of_a_multiline_record_is_tagged():
|
||||
"""The journal splits them, and an untagged continuation loses its level.
|
||||
|
||||
A traceback is the case that matters -- it is the most important thing in
|
||||
the log and the longest.
|
||||
"""
|
||||
out = JournalPriorityFormatter(_Plain()).format(
|
||||
_record(logging.ERROR, "Traceback:\nline one\nline two"))
|
||||
lines = out.split("\n")
|
||||
assert len(lines) == 3
|
||||
assert all(line.startswith("<3>") for line in lines), lines
|
||||
|
||||
|
||||
def test_the_message_survives_intact():
|
||||
out = JournalPriorityFormatter(_Plain()).format(_record(logging.WARNING, "disk full"))
|
||||
assert out == "<4>disk full"
|
||||
|
||||
|
||||
def test_an_unknown_level_falls_back_to_info():
|
||||
out = JournalPriorityFormatter(_Plain()).format(_record(25))
|
||||
assert out.startswith("<6>")
|
||||
|
||||
|
||||
def _stdout_ids():
|
||||
"""The dev:ino systemd would publish for this process's stdout."""
|
||||
st = os.fstat(sys.stdout.fileno())
|
||||
return f"{st.st_dev}:{st.st_ino}"
|
||||
|
||||
|
||||
def test_prefixing_is_off_outside_systemd():
|
||||
"""Otherwise a terminal run, the emulator and pytest all show `<6>`."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
assert not _under_systemd()
|
||||
with patch.dict(os.environ, {"JOURNAL_STREAM": _stdout_ids()}):
|
||||
assert _under_systemd()
|
||||
|
||||
|
||||
def test_an_inherited_journal_stream_does_not_count():
|
||||
"""The variable outlives the descriptor it describes.
|
||||
|
||||
systemd sets JOURNAL_STREAM for the service, and every child inherits it
|
||||
-- including one whose stdout has been redirected to a pipe or a file.
|
||||
Trusting the variable alone put literal "<6>" prefixes into that captured
|
||||
output. Only a descriptor whose dev:ino actually matches is the journal.
|
||||
"""
|
||||
with patch.dict(os.environ, {"JOURNAL_STREAM": "8:12345"}):
|
||||
assert not _under_systemd(), \
|
||||
"a stale inherited JOURNAL_STREAM was treated as the journal"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", "not-a-pair", "8", "8:", ":12345",
|
||||
"eight:12345", "8:12345:9"])
|
||||
def test_a_malformed_journal_stream_is_not_the_journal(value):
|
||||
with patch.dict(os.environ, {"JOURNAL_STREAM": value}):
|
||||
assert not _under_systemd()
|
||||
|
||||
|
||||
def test_a_closed_stdout_is_not_the_journal():
|
||||
"""os.fstat raises rather than answers; that must not propagate."""
|
||||
with patch.dict(os.environ, {"JOURNAL_STREAM": "8:12345"}), \
|
||||
patch("src.logging_config.sys.stdout") as fake_stdout:
|
||||
fake_stdout.fileno.side_effect = ValueError("I/O operation on closed file")
|
||||
assert not _under_systemd()
|
||||
|
||||
|
||||
def test_setup_uses_the_wrapper_only_under_systemd():
|
||||
from src.logging_config import setup_logging
|
||||
|
||||
for env, expect_wrapped in (({}, False),
|
||||
({"JOURNAL_STREAM": _stdout_ids()}, True)):
|
||||
with patch.dict(os.environ, env, clear=True):
|
||||
setup_logging()
|
||||
handlers = [h for h in logging.getLogger().handlers
|
||||
if isinstance(h, logging.StreamHandler)]
|
||||
assert handlers, "no stream handler installed"
|
||||
wrapped = any(isinstance(h.formatter, JournalPriorityFormatter)
|
||||
for h in handlers)
|
||||
assert wrapped is expect_wrapped, (
|
||||
f"JOURNAL_STREAM={env}: wrapped={wrapped}, expected {expect_wrapped}")
|
||||
logging.getLogger().handlers.clear()
|
||||
@@ -89,11 +89,17 @@ class TestContextualFormatter:
|
||||
assert "hello" in out
|
||||
|
||||
def test_location_toggle(self):
|
||||
# Assert on the whole "module.func:lineno" token, not a bare ":42".
|
||||
# The formatted line starts with an HH:MM:SS timestamp, so a bare
|
||||
# ":{lineno}" also matches the clock whenever the minute or second
|
||||
# happens to equal the line number -- about 3% of runs, which is a
|
||||
# flaky failure with nothing wrong.
|
||||
record = make_record()
|
||||
location = f"{record.module}.{record.funcName}:{record.lineno}"
|
||||
with_loc = ContextualFormatter(include_location=True).format(record)
|
||||
without = ContextualFormatter(include_location=False).format(record)
|
||||
assert f":{record.lineno}" in with_loc
|
||||
assert f":{record.lineno}" not in without
|
||||
assert location in with_loc
|
||||
assert location not in without
|
||||
|
||||
def test_record_not_mutated_no_double_prefix(self):
|
||||
# Regression: a record is formatted once PER HANDLER. The formatter
|
||||
@@ -183,15 +189,27 @@ class TestSetupLogging:
|
||||
setup_logging()
|
||||
assert len(logging.getLogger().handlers) == 1
|
||||
|
||||
@staticmethod
|
||||
def _selected_formatter():
|
||||
"""The formatter setup_logging() chose, past any journald wrapper.
|
||||
|
||||
Under systemd the console handler's formatter is wrapped so each line
|
||||
carries its syslog priority. That wrapper is applied only when
|
||||
JOURNAL_STREAM is set, which is true in CI and false in a terminal, so
|
||||
asserting on the handler's formatter directly passes locally and fails
|
||||
on the runner. These tests are about which formatter format_type
|
||||
selects, so they look through the wrapper.
|
||||
"""
|
||||
formatter = logging.getLogger().handlers[0].formatter
|
||||
return getattr(formatter, "inner", formatter)
|
||||
|
||||
def test_json_format_selects_structured_formatter(self):
|
||||
setup_logging(format_type="json")
|
||||
assert isinstance(
|
||||
logging.getLogger().handlers[0].formatter, StructuredFormatter)
|
||||
assert isinstance(self._selected_formatter(), StructuredFormatter)
|
||||
|
||||
def test_readable_format_selects_contextual_formatter(self):
|
||||
setup_logging(format_type="readable")
|
||||
assert isinstance(
|
||||
logging.getLogger().handlers[0].formatter, ContextualFormatter)
|
||||
assert isinstance(self._selected_formatter(), ContextualFormatter)
|
||||
|
||||
def test_log_file_adds_file_handler(self, tmp_path):
|
||||
log_file = tmp_path / "test.log"
|
||||
|
||||
@@ -8,11 +8,27 @@ ensure_logo_directory, and the download_missing_logo function path
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, Mock, MagicMock
|
||||
|
||||
from src.logo_downloader import LogoDownloader
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
|
||||
from src.logo_downloader import (
|
||||
PLACEHOLDER_BG,
|
||||
PLACEHOLDER_MARKER,
|
||||
PLACEHOLDER_RETRY_SECONDS,
|
||||
PLACEHOLDER_SIZE,
|
||||
LogoDownloader,
|
||||
download_missing_logo,
|
||||
is_placeholder_logo,
|
||||
placeholder_age_seconds,
|
||||
refresh_placeholder_timestamp,
|
||||
should_attempt_download,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -127,3 +143,211 @@ class TestEnsureLogoDirectory:
|
||||
with patch("builtins.open", side_effect=mock_open):
|
||||
result = downloader.ensure_logo_directory(test_dir)
|
||||
assert result is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Placeholder detection and retry
|
||||
#
|
||||
# A failed download used to be cached as a placeholder wearing the real logo's
|
||||
# filename, and download_missing_logo returned early on "the file exists". One
|
||||
# transient failure therefore pinned a team to a grey box permanently.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestPlaceholderLogos:
|
||||
def _placeholder(self, tmp_path, abbrev="COLL"):
|
||||
downloader = LogoDownloader()
|
||||
assert downloader.create_placeholder_logo(abbrev, str(tmp_path)) is True
|
||||
return tmp_path / f"{abbrev}.png"
|
||||
|
||||
def test_generated_placeholder_is_recognised(self, tmp_path):
|
||||
assert is_placeholder_logo(self._placeholder(tmp_path)) is True
|
||||
|
||||
def test_real_logo_is_not_a_placeholder(self, tmp_path):
|
||||
real = tmp_path / "REAL.png"
|
||||
Image.new("RGBA", (500, 500), (12, 34, 56, 255)).save(real)
|
||||
assert is_placeholder_logo(real) is False
|
||||
|
||||
def test_legacy_unmarked_placeholder_is_recognised(self, tmp_path):
|
||||
"""Placeholders written before the marker existed must still be caught.
|
||||
|
||||
They are already sitting on users' disks; if they were not recognised
|
||||
those teams would stay grey boxes forever even after this fix.
|
||||
"""
|
||||
legacy = tmp_path / "LEGACY.png"
|
||||
Image.new("RGBA", PLACEHOLDER_SIZE, PLACEHOLDER_BG).save(legacy)
|
||||
assert is_placeholder_logo(legacy) is True
|
||||
|
||||
def test_same_size_but_different_colour_is_not_a_placeholder(self, tmp_path):
|
||||
real = tmp_path / "SMALL.png"
|
||||
Image.new("RGBA", PLACEHOLDER_SIZE, (10, 200, 10, 255)).save(real)
|
||||
assert is_placeholder_logo(real) is False
|
||||
|
||||
def test_missing_file_is_not_a_placeholder(self, tmp_path):
|
||||
assert is_placeholder_logo(tmp_path / "nope.png") is False
|
||||
|
||||
def test_existing_real_logo_short_circuits_without_downloading(self, tmp_path):
|
||||
real = tmp_path / "REAL.png"
|
||||
Image.new("RGBA", (500, 500), (1, 2, 3, 255)).save(real)
|
||||
with patch.object(LogoDownloader, "download_logo") as download:
|
||||
assert download_missing_logo(
|
||||
"afl", "1", "REAL", real, logo_url="http://example/x.png") is True
|
||||
download.assert_not_called()
|
||||
|
||||
def _age_placeholder(self, path, seconds):
|
||||
"""Rewrite a placeholder's marker so it reads as `seconds` old."""
|
||||
metadata = PngInfo()
|
||||
metadata.add_text(PLACEHOLDER_MARKER, str(time.time() - seconds))
|
||||
with Image.open(path) as img:
|
||||
img.copy().save(path, "PNG", pnginfo=metadata)
|
||||
|
||||
def test_stale_placeholder_triggers_a_retry(self, tmp_path):
|
||||
path = self._placeholder(tmp_path)
|
||||
self._age_placeholder(path, PLACEHOLDER_RETRY_SECONDS + 60)
|
||||
assert placeholder_age_seconds(path) > PLACEHOLDER_RETRY_SECONDS
|
||||
|
||||
with patch.object(LogoDownloader, "download_logo", return_value=True) as download:
|
||||
assert download_missing_logo(
|
||||
"afl", "1", "COLL", path,
|
||||
logo_url="http://example/coll.png") is True
|
||||
download.assert_called_once()
|
||||
|
||||
def test_placeholder_age_survives_an_mtime_touch(self, tmp_path):
|
||||
"""The age comes from the stamp, not the filesystem.
|
||||
|
||||
Anything that rewrites file times -- a backup restore, an rsync, a
|
||||
permissions fix script -- would otherwise reset the retry clock.
|
||||
"""
|
||||
path = self._placeholder(tmp_path)
|
||||
self._age_placeholder(path, PLACEHOLDER_RETRY_SECONDS + 60)
|
||||
now = time.time()
|
||||
os.utime(path, (now, now))
|
||||
assert placeholder_age_seconds(path) > PLACEHOLDER_RETRY_SECONDS
|
||||
|
||||
def test_fresh_placeholder_does_not_retry(self, tmp_path):
|
||||
"""Rate limiting: a placeholder written seconds ago must not re-download.
|
||||
|
||||
Without this the fix would trade a permanent grey box for an ESPN
|
||||
request on every frame.
|
||||
"""
|
||||
path = self._placeholder(tmp_path)
|
||||
with patch.object(LogoDownloader, "download_logo") as download:
|
||||
assert download_missing_logo(
|
||||
"afl", "1", "COLL", path,
|
||||
logo_url="http://example/coll.png") is True
|
||||
download.assert_not_called()
|
||||
|
||||
|
||||
class TestDownloadEligibility:
|
||||
"""One rule, shared by every download site.
|
||||
|
||||
The two bulk loops and the single-logo path each had their own idea of what
|
||||
counted as "already have it", which is how one of them ended up retrying
|
||||
fresh placeholders and the other skipping stale ones forever.
|
||||
"""
|
||||
|
||||
def _placeholder(self, tmp_path, abbrev="COLL"):
|
||||
assert LogoDownloader().create_placeholder_logo(abbrev, str(tmp_path))
|
||||
return tmp_path / f"{abbrev}.png"
|
||||
|
||||
def _age(self, path, seconds):
|
||||
metadata = PngInfo()
|
||||
metadata.add_text(PLACEHOLDER_MARKER, str(time.time() - seconds))
|
||||
with Image.open(path) as img:
|
||||
img.copy().save(path, "PNG", pnginfo=metadata)
|
||||
|
||||
def test_missing_file_is_eligible(self, tmp_path):
|
||||
assert should_attempt_download(tmp_path / "nope.png") is True
|
||||
|
||||
def test_real_logo_is_not_eligible(self, tmp_path):
|
||||
real = tmp_path / "REAL.png"
|
||||
Image.new("RGBA", (500, 500), (1, 2, 3, 255)).save(real)
|
||||
assert should_attempt_download(real) is False
|
||||
|
||||
def test_force_download_beats_a_real_logo(self, tmp_path):
|
||||
real = tmp_path / "REAL.png"
|
||||
Image.new("RGBA", (500, 500), (1, 2, 3, 255)).save(real)
|
||||
assert should_attempt_download(real, force_download=True) is True
|
||||
|
||||
def test_fresh_placeholder_is_not_eligible(self, tmp_path):
|
||||
assert should_attempt_download(self._placeholder(tmp_path)) is False
|
||||
|
||||
def test_stale_placeholder_is_eligible(self, tmp_path):
|
||||
path = self._placeholder(tmp_path)
|
||||
self._age(path, PLACEHOLDER_RETRY_SECONDS + 60)
|
||||
assert should_attempt_download(path) is True
|
||||
|
||||
def test_league_bulk_loop_skips_a_fresh_placeholder(self, tmp_path):
|
||||
"""A bulk pass honours the same back-off as everything else."""
|
||||
self._placeholder(tmp_path, "AAA")
|
||||
downloader = LogoDownloader()
|
||||
teams = [{"abbreviation": "AAA", "display_name": "A", "logo_url": "http://x/a.png"}]
|
||||
with patch.object(LogoDownloader, "get_logo_directory", return_value=str(tmp_path)):
|
||||
with patch.object(LogoDownloader, "fetch_teams_data", return_value={"sports": [{}]}):
|
||||
with patch.object(LogoDownloader, "extract_teams_from_data", return_value=teams):
|
||||
with patch.object(LogoDownloader, "download_logo") as download:
|
||||
downloader.download_missing_logos_for_league("nfl")
|
||||
download.assert_not_called()
|
||||
|
||||
def test_league_bulk_loop_retries_a_stale_placeholder(self, tmp_path):
|
||||
path = self._placeholder(tmp_path, "AAA")
|
||||
self._age(path, PLACEHOLDER_RETRY_SECONDS + 60)
|
||||
downloader = LogoDownloader()
|
||||
teams = [{"abbreviation": "AAA", "display_name": "A", "logo_url": "http://x/a.png"}]
|
||||
with patch.object(LogoDownloader, "get_logo_directory", return_value=str(tmp_path)):
|
||||
with patch.object(LogoDownloader, "fetch_teams_data", return_value={"sports": [{}]}):
|
||||
with patch.object(LogoDownloader, "extract_teams_from_data", return_value=teams):
|
||||
with patch.object(LogoDownloader, "download_logo", return_value=True) as download:
|
||||
downloader.download_missing_logos_for_league("nfl")
|
||||
download.assert_called_once()
|
||||
|
||||
def test_ncaa_bulk_loop_retries_a_stale_placeholder(self, tmp_path):
|
||||
"""This loop skipped placeholders forever; it now shares the rule."""
|
||||
path = self._placeholder(tmp_path, "AAA")
|
||||
self._age(path, PLACEHOLDER_RETRY_SECONDS + 60)
|
||||
downloader = LogoDownloader()
|
||||
teams = [{"abbreviation": "AAA", "display_name": "A",
|
||||
"logo_url": "http://x/a.png", "category": "FBS",
|
||||
"conference": "SEC"}]
|
||||
with patch.object(LogoDownloader, "get_logo_directory", return_value=str(tmp_path)):
|
||||
with patch.object(LogoDownloader, "fetch_teams_data", return_value={"sports": [{}]}):
|
||||
with patch.object(LogoDownloader, "extract_teams_from_data", return_value=teams):
|
||||
with patch.object(LogoDownloader, "download_logo", return_value=True) as download:
|
||||
downloader.download_all_ncaa_football_logos()
|
||||
download.assert_called_once()
|
||||
|
||||
def test_ncaa_bulk_loop_skips_a_fresh_placeholder(self, tmp_path):
|
||||
self._placeholder(tmp_path, "AAA")
|
||||
downloader = LogoDownloader()
|
||||
teams = [{"abbreviation": "AAA", "display_name": "A",
|
||||
"logo_url": "http://x/a.png", "category": "FBS",
|
||||
"conference": "SEC"}]
|
||||
with patch.object(LogoDownloader, "get_logo_directory", return_value=str(tmp_path)):
|
||||
with patch.object(LogoDownloader, "fetch_teams_data", return_value={"sports": [{}]}):
|
||||
with patch.object(LogoDownloader, "extract_teams_from_data", return_value=teams):
|
||||
with patch.object(LogoDownloader, "download_logo") as download:
|
||||
downloader.download_all_ncaa_football_logos()
|
||||
download.assert_not_called()
|
||||
|
||||
|
||||
class TestRefreshPlaceholderTimestamp:
|
||||
def test_restarts_the_back_off(self, tmp_path):
|
||||
assert LogoDownloader().create_placeholder_logo("COLL", str(tmp_path))
|
||||
path = tmp_path / "COLL.png"
|
||||
metadata = PngInfo()
|
||||
metadata.add_text(PLACEHOLDER_MARKER, str(time.time() - (PLACEHOLDER_RETRY_SECONDS + 60)))
|
||||
with Image.open(path) as img:
|
||||
img.copy().save(path, "PNG", pnginfo=metadata)
|
||||
assert should_attempt_download(path) is True
|
||||
|
||||
assert refresh_placeholder_timestamp(path) is True
|
||||
assert should_attempt_download(path) is False
|
||||
|
||||
def test_refuses_to_touch_a_real_logo(self, tmp_path):
|
||||
real = tmp_path / "REAL.png"
|
||||
Image.new("RGBA", (500, 500), (1, 2, 3, 255)).save(real)
|
||||
before = real.read_bytes()
|
||||
assert refresh_placeholder_timestamp(real) is False
|
||||
assert real.read_bytes() == before
|
||||
|
||||
def test_missing_file_is_not_an_error(self, tmp_path):
|
||||
assert refresh_placeholder_timestamp(tmp_path / "nope.png") is False
|
||||
|
||||
@@ -0,0 +1,496 @@
|
||||
"""
|
||||
Tests for src/common/logo_helper.py — logo loading, LRU caching, resizing,
|
||||
and download-with-fallback. Previously untested: nothing in test/ referenced
|
||||
this module at all.
|
||||
|
||||
Real PIL images under tmp_path are used rather than mocked ones, since
|
||||
load_logo() does real Path.exists() and Image.open() calls; only the HTTP
|
||||
session and the permission helpers are patched.
|
||||
|
||||
Regression coverage for two fixed bugs:
|
||||
- _download_logo wrote response.content to disk with no size cap and no
|
||||
check that the bytes decoded as an image, so a hostile or broken URL
|
||||
could leave arbitrary/oversized content cached in the assets directory.
|
||||
- get_cache_stats() divided by self.cache_size unguarded, raising
|
||||
ZeroDivisionError for a helper constructed with cache_size=0.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from src.common.logo_helper import MAX_LOGO_BYTES, LogoHelper
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_real_chmod(monkeypatch):
|
||||
# Keep the permission helpers out of the way: their own env detection
|
||||
# is not what these tests are about.
|
||||
monkeypatch.setattr("src.common.logo_helper.ensure_directory_permissions", MagicMock())
|
||||
monkeypatch.setattr("src.common.logo_helper.ensure_file_permissions", MagicMock())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def helper():
|
||||
return LogoHelper(display_width=64, display_height=32,
|
||||
logger=logging.getLogger("test.logo_helper"))
|
||||
|
||||
|
||||
def write_logo(path: Path, size=(20, 20), color=(255, 0, 0), fmt="PNG") -> Path:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
Image.new("RGB", size, color).save(path, format=fmt)
|
||||
return path
|
||||
|
||||
|
||||
def fake_response(content: bytes, chunk_size: int = 64 * 1024):
|
||||
"""Stand-in for a streamed requests.Response.
|
||||
|
||||
_download_logo opens `with session.get(..., stream=True)` and reads
|
||||
through iter_content(), so the fake has to be a context manager that
|
||||
yields the body in pieces rather than exposing it as .content.
|
||||
Chunking is the fake's own, not the caller's, so a test can dribble a
|
||||
body out in small pieces.
|
||||
"""
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value = response
|
||||
response.__exit__.return_value = False
|
||||
response.raise_for_status = MagicMock()
|
||||
|
||||
def _iter_content(*_args, **_kwargs):
|
||||
for i in range(0, len(content), chunk_size):
|
||||
yield content[i:i + chunk_size]
|
||||
|
||||
response.iter_content = _iter_content
|
||||
return response
|
||||
|
||||
|
||||
def endless_response(chunk: bytes = b"\x00" * 65536):
|
||||
"""A server that declares no length and never stops sending.
|
||||
|
||||
This is the case response.content could not survive: it buffers to
|
||||
completion, so the size check never got a chance to run.
|
||||
"""
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value = response
|
||||
response.__exit__.return_value = False
|
||||
response.raise_for_status = MagicMock()
|
||||
|
||||
def _iter_content(*_args, **_kwargs):
|
||||
while True:
|
||||
yield chunk
|
||||
|
||||
response.iter_content = _iter_content
|
||||
return response
|
||||
|
||||
|
||||
def png_bytes(size=(20, 20), color=(0, 128, 0)) -> bytes:
|
||||
import io
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", size, color).save(buf, format="PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
class TestLoadLogo:
|
||||
def test_loads_and_converts_to_rgba(self, helper, tmp_path):
|
||||
path = write_logo(tmp_path / "PHI.png")
|
||||
logo = helper.load_logo("PHI", path)
|
||||
assert logo is not None
|
||||
assert logo.mode == "RGBA"
|
||||
|
||||
def test_missing_file_returns_none(self, helper, tmp_path, caplog):
|
||||
with caplog.at_level(logging.WARNING):
|
||||
assert helper.load_logo("NOPE", tmp_path / "missing.png") is None
|
||||
assert "Logo not found" in caplog.text
|
||||
|
||||
def test_second_load_is_served_from_cache(self, helper, tmp_path):
|
||||
path = write_logo(tmp_path / "PHI.png")
|
||||
first = helper.load_logo("PHI", path)
|
||||
path.unlink() # cache hit must not touch the filesystem
|
||||
assert helper.load_logo("PHI", path) is first
|
||||
|
||||
def test_cache_key_includes_requested_size(self, helper, tmp_path):
|
||||
# A panel-size change must not hand back a logo sized for the old
|
||||
# dimensions, so the two sizes get separate cache entries.
|
||||
path = write_logo(tmp_path / "PHI.png", size=(100, 100))
|
||||
small = helper.load_logo("PHI", path, max_width=10, max_height=10)
|
||||
large = helper.load_logo("PHI", path, max_width=50, max_height=50)
|
||||
assert small is not large
|
||||
assert small.size != large.size
|
||||
assert len(helper._logo_cache) == 2
|
||||
|
||||
def test_default_size_is_one_and_a_half_display(self, helper, tmp_path):
|
||||
path = write_logo(tmp_path / "PHI.png", size=(500, 500))
|
||||
logo = helper.load_logo("PHI", path)
|
||||
assert logo.width <= int(64 * 1.5)
|
||||
assert logo.height <= int(32 * 1.5)
|
||||
|
||||
def test_smaller_image_is_not_upscaled(self, helper, tmp_path):
|
||||
path = write_logo(tmp_path / "PHI.png", size=(8, 8))
|
||||
assert helper.load_logo("PHI", path, max_width=64, max_height=64).size == (8, 8)
|
||||
|
||||
def test_larger_image_is_downscaled_preserving_aspect(self, helper, tmp_path):
|
||||
path = write_logo(tmp_path / "PHI.png", size=(200, 100))
|
||||
logo = helper.load_logo("PHI", path, max_width=50, max_height=50)
|
||||
assert logo.width <= 50 and logo.height <= 50
|
||||
assert logo.width == 50 and logo.height == 25 # 2:1 preserved
|
||||
|
||||
def test_string_path_accepted(self, helper, tmp_path):
|
||||
path = write_logo(tmp_path / "PHI.png")
|
||||
assert helper.load_logo("PHI", str(path)) is not None
|
||||
|
||||
def test_corrupt_file_returns_none(self, helper, tmp_path, caplog):
|
||||
bad = tmp_path / "bad.png"
|
||||
bad.write_bytes(b"not an image")
|
||||
with caplog.at_level(logging.ERROR):
|
||||
assert helper.load_logo("BAD", bad) is None
|
||||
assert "Error loading logo" in caplog.text
|
||||
|
||||
|
||||
class TestCacheManagement:
|
||||
def test_lru_evicts_oldest(self, tmp_path):
|
||||
helper = LogoHelper(64, 32, cache_size=2, logger=MagicMock())
|
||||
paths = [write_logo(tmp_path / f"T{i}.png") for i in range(3)]
|
||||
for i, path in enumerate(paths):
|
||||
helper.load_logo(f"T{i}", path)
|
||||
assert len(helper._logo_cache) == 2
|
||||
assert not any(k.startswith("T0_") for k in helper._logo_cache)
|
||||
|
||||
def test_cache_hit_refreshes_lru_position(self, tmp_path):
|
||||
helper = LogoHelper(64, 32, cache_size=2, logger=MagicMock())
|
||||
a, b, c = [write_logo(tmp_path / f"{n}.png") for n in ("A", "B", "C")]
|
||||
helper.load_logo("A", a)
|
||||
helper.load_logo("B", b)
|
||||
helper.load_logo("A", a) # A is now most-recently used
|
||||
helper.load_logo("C", c) # evicts B, not A
|
||||
assert any(k.startswith("A_") for k in helper._logo_cache)
|
||||
assert not any(k.startswith("B_") for k in helper._logo_cache)
|
||||
|
||||
def test_clear_cache_empties_both_structures(self, helper, tmp_path):
|
||||
helper.load_logo("PHI", write_logo(tmp_path / "PHI.png"))
|
||||
helper.clear_cache()
|
||||
assert helper._logo_cache == {}
|
||||
assert helper._cache_order == []
|
||||
|
||||
def test_cache_stats(self, tmp_path):
|
||||
helper = LogoHelper(64, 32, cache_size=4, logger=MagicMock())
|
||||
helper.load_logo("PHI", write_logo(tmp_path / "PHI.png"))
|
||||
stats = helper.get_cache_stats()
|
||||
assert stats["cached_logos"] == 1
|
||||
assert stats["cache_size_limit"] == 4
|
||||
assert stats["cache_usage_percent"] == 25
|
||||
|
||||
def test_zero_cache_size_does_not_divide_by_zero(self):
|
||||
# Regression: this raised ZeroDivisionError.
|
||||
stats = LogoHelper(64, 32, cache_size=0, logger=MagicMock()).get_cache_stats()
|
||||
assert stats["cache_usage_percent"] == 0
|
||||
assert stats["cache_size_limit"] == 0
|
||||
|
||||
|
||||
class TestLoadLogoWithDownload:
|
||||
def test_existing_file_skips_download(self, helper, tmp_path):
|
||||
path = write_logo(tmp_path / "PHI.png")
|
||||
helper.session.get = MagicMock()
|
||||
assert helper.load_logo_with_download("PHI", path, "http://x/logo.png") is not None
|
||||
helper.session.get.assert_not_called()
|
||||
|
||||
def test_downloads_then_loads(self, helper, tmp_path):
|
||||
path = tmp_path / "PHI.png"
|
||||
helper.session.get = MagicMock(return_value=fake_response(png_bytes()))
|
||||
logo = helper.load_logo_with_download("PHI", path, "http://x/logo.png")
|
||||
assert logo is not None
|
||||
assert path.exists()
|
||||
# stream=True is load-bearing: it is what lets the size cap apply
|
||||
# before the body is buffered.
|
||||
helper.session.get.assert_called_once_with(
|
||||
"http://x/logo.png", timeout=30, stream=True)
|
||||
|
||||
def test_download_failure_falls_back_to_placeholder(self, helper, tmp_path):
|
||||
helper.session.get = MagicMock(
|
||||
side_effect=requests.RequestException("connection reset"))
|
||||
logo = helper.load_logo_with_download(
|
||||
"PHI", tmp_path / "PHI.png", "http://x/logo.png",
|
||||
max_width=20, max_height=20)
|
||||
assert logo is not None and logo.size == (20, 20) # placeholder
|
||||
|
||||
def test_http_error_falls_back_to_placeholder(self, helper, tmp_path):
|
||||
response = fake_response(b"")
|
||||
response.raise_for_status.side_effect = requests.HTTPError("404")
|
||||
helper.session.get = MagicMock(return_value=response)
|
||||
logo = helper.load_logo_with_download(
|
||||
"PHI", tmp_path / "PHI.png", "http://x/logo.png",
|
||||
max_width=20, max_height=20)
|
||||
assert logo is not None and logo.size == (20, 20)
|
||||
|
||||
def test_no_url_and_no_file_gives_placeholder(self, helper, tmp_path):
|
||||
logo = helper.load_logo_with_download(
|
||||
"PHI", tmp_path / "missing.png", None, max_width=16, max_height=16)
|
||||
assert logo is not None and logo.size == (16, 16)
|
||||
|
||||
|
||||
class TestDownloadLogo:
|
||||
def test_writes_file_and_sets_permissions(self, helper, tmp_path):
|
||||
path = tmp_path / "assets" / "PHI.png"
|
||||
# Directory creation is ensure_directory_permissions' job, and the
|
||||
# autouse fixture stubs it out — so make the directory here.
|
||||
path.parent.mkdir()
|
||||
helper.session.get = MagicMock(return_value=fake_response(png_bytes()))
|
||||
with patch("src.common.logo_helper.ensure_directory_permissions") as dirs, \
|
||||
patch("src.common.logo_helper.ensure_file_permissions") as files:
|
||||
helper._download_logo("http://x/logo.png", path)
|
||||
assert path.exists()
|
||||
dirs.assert_called_once()
|
||||
files.assert_called_once()
|
||||
assert dirs.call_args[0][0] == path.parent
|
||||
|
||||
def test_oversized_response_is_rejected_without_writing(self, helper, tmp_path):
|
||||
# Regression: an unbounded response.content was written straight to
|
||||
# disk, so a hostile URL chose how many bytes landed in assets/.
|
||||
path = tmp_path / "huge.png"
|
||||
helper.session.get = MagicMock(
|
||||
return_value=fake_response(b"\x00" * (MAX_LOGO_BYTES + 1)))
|
||||
with pytest.raises(ValueError, match="exceeds the"):
|
||||
helper._download_logo("http://x/huge.png", path)
|
||||
assert not path.exists()
|
||||
|
||||
def test_unbounded_response_is_aborted_at_the_cap(self, helper, tmp_path):
|
||||
# Regression: the cap used to be checked against response.content,
|
||||
# which buffers the whole body first — so a server that omits
|
||||
# Content-Length and never stops sending exhausted memory before
|
||||
# the check could run. Streaming counts bytes as they arrive, so
|
||||
# this terminates instead of hanging.
|
||||
path = tmp_path / "endless.png"
|
||||
helper.session.get = MagicMock(return_value=endless_response())
|
||||
with pytest.raises(ValueError, match="exceeds the"):
|
||||
helper._download_logo("http://x/endless.png", path)
|
||||
assert not path.exists()
|
||||
|
||||
def test_no_partial_file_is_left_when_the_stream_dies(self, helper, tmp_path):
|
||||
# A transfer that fails midway must not leave a truncated logo
|
||||
# where the real one belongs — load_logo() would cache it.
|
||||
path = tmp_path / "cut.png"
|
||||
real = png_bytes()
|
||||
|
||||
def _dies_midway(*_args, **_kwargs):
|
||||
yield real[:20]
|
||||
raise OSError("connection reset")
|
||||
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value = response
|
||||
response.__exit__.return_value = False
|
||||
response.raise_for_status = MagicMock()
|
||||
response.iter_content = _dies_midway
|
||||
helper.session.get = MagicMock(return_value=response)
|
||||
|
||||
with pytest.raises(OSError):
|
||||
helper._download_logo("http://x/cut.png", path)
|
||||
assert not path.exists()
|
||||
assert list(tmp_path.glob("*.part")) == []
|
||||
|
||||
def test_concurrent_downloads_do_not_share_a_temp_file(self, helper, tmp_path):
|
||||
# Two plugins can ask for the same logo at once. A fixed
|
||||
# "<name>.part" would let them interleave writes into one file and
|
||||
# publish the mixture; each download gets its own temp name.
|
||||
path = tmp_path / "PHI.png"
|
||||
seen = []
|
||||
real_mkstemp = tempfile.mkstemp
|
||||
|
||||
def record(*args, **kwargs):
|
||||
fd, name = real_mkstemp(*args, **kwargs)
|
||||
seen.append(name)
|
||||
return fd, name
|
||||
|
||||
with patch("src.common.logo_helper.tempfile.mkstemp", side_effect=record):
|
||||
helper.session.get = MagicMock(return_value=fake_response(png_bytes()))
|
||||
helper._download_logo("http://x/logo.png", path)
|
||||
helper.session.get = MagicMock(return_value=fake_response(png_bytes()))
|
||||
helper._download_logo("http://x/logo.png", path)
|
||||
|
||||
assert len(seen) == 2 and seen[0] != seen[1]
|
||||
assert path.exists()
|
||||
assert list(tmp_path.glob("*.part")) == [] # both cleaned up
|
||||
|
||||
def test_request_failure_leaves_no_temp_file(self, helper, tmp_path):
|
||||
# mkstemp creates the file up front, so an error before any bytes
|
||||
# arrive still has something to clean up.
|
||||
helper.session.get = MagicMock(
|
||||
side_effect=requests.RequestException("connection reset"))
|
||||
with pytest.raises(requests.RequestException):
|
||||
helper._download_logo("http://x/logo.png", tmp_path / "PHI.png")
|
||||
assert list(tmp_path.glob("*")) == []
|
||||
|
||||
def test_non_image_response_is_deleted_and_raises(self, helper, tmp_path):
|
||||
# Regression: undecodable bytes stayed on disk, so every later
|
||||
# load_logo() call hit the corrupt file instead of re-downloading.
|
||||
path = tmp_path / "bad.png"
|
||||
helper.session.get = MagicMock(return_value=fake_response(b"<html>404</html>"))
|
||||
# Specifically Pillow's identify failure, not any OSError: the
|
||||
# point is that the bytes did not decode, and OSError alone would
|
||||
# also admit unrelated filesystem faults.
|
||||
with pytest.raises(UnidentifiedImageError):
|
||||
helper._download_logo("http://x/bad.png", path)
|
||||
assert not path.exists()
|
||||
assert list(tmp_path.glob("*.part")) == []
|
||||
|
||||
def test_decompression_bomb_is_deleted_and_raises(self, helper, tmp_path, monkeypatch):
|
||||
path = tmp_path / "bomb.png"
|
||||
helper.session.get = MagicMock(return_value=fake_response(png_bytes()))
|
||||
|
||||
class Bomb:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def load(self):
|
||||
raise Image.DecompressionBombError("too many pixels")
|
||||
|
||||
monkeypatch.setattr("src.common.logo_helper.Image.open", lambda *a, **kw: Bomb())
|
||||
with pytest.raises(Image.DecompressionBombError):
|
||||
helper._download_logo("http://x/bomb.png", path)
|
||||
assert not path.exists()
|
||||
|
||||
def test_bad_download_surfaces_as_placeholder_not_crash(self, helper, tmp_path):
|
||||
# The new guards raise, and load_logo_with_download's existing
|
||||
# broad except turns that into the placeholder path.
|
||||
helper.session.get = MagicMock(return_value=fake_response(b"garbage"))
|
||||
logo = helper.load_logo_with_download(
|
||||
"PHI", tmp_path / "PHI.png", "http://x/bad.png",
|
||||
max_width=12, max_height=12)
|
||||
assert logo is not None and logo.size == (12, 12)
|
||||
|
||||
|
||||
class TestLogoVariations:
|
||||
def test_plain_abbreviation_returns_itself(self, helper):
|
||||
assert helper.get_logo_variations("PHI") == ["PHI"]
|
||||
|
||||
def test_ampersand_expanded(self, helper):
|
||||
assert "TAAND M" in helper.get_logo_variations("TA& M")
|
||||
|
||||
def test_and_contracted(self, helper):
|
||||
assert "T&M" in helper.get_logo_variations("TANDM")
|
||||
|
||||
def test_special_case_appends_known_aliases(self, helper):
|
||||
variations = helper.get_logo_variations("TA&M")
|
||||
assert "TAMU" in variations and "TEXASAM" in variations
|
||||
assert "TAANDM" in variations # the generic & rule still applies
|
||||
|
||||
|
||||
class TestNormalizeAbbreviation:
|
||||
def test_uppercases_and_strips(self, helper):
|
||||
assert helper.normalize_abbreviation(" phi ") == "PHI"
|
||||
|
||||
def test_ampersand_becomes_and(self, helper):
|
||||
assert helper.normalize_abbreviation("TA&M") == "TAANDM"
|
||||
|
||||
def test_internal_spaces_removed(self, helper):
|
||||
assert helper.normalize_abbreviation("New York") == "NEWYORK"
|
||||
|
||||
def test_deliberately_differs_from_logo_downloader(self, helper):
|
||||
# Pinned, not a bug: LogoDownloader.normalize_abbreviation replaces
|
||||
# filesystem-unsafe characters but keeps spaces, and plugins call
|
||||
# that one. Changing either changes which logo filenames resolve on
|
||||
# existing installs. Both docstrings say so explicitly.
|
||||
from src.logo_downloader import LogoDownloader
|
||||
assert helper.normalize_abbreviation("New York") == "NEWYORK"
|
||||
assert LogoDownloader.normalize_abbreviation("New York") == "NEW YORK"
|
||||
|
||||
|
||||
class TestPlaceholderLogo:
|
||||
def test_uses_requested_dimensions(self, helper):
|
||||
assert helper._create_placeholder_logo("PHI", 30, 20).size == (30, 20)
|
||||
|
||||
def test_defaults_to_one_and_a_half_display(self, helper):
|
||||
assert helper._create_placeholder_logo("PHI").size == (96, 48)
|
||||
|
||||
def test_is_rgba(self, helper):
|
||||
assert helper._create_placeholder_logo("PHI", 10, 10).mode == "RGBA"
|
||||
|
||||
def test_invalid_dimensions_return_none(self, helper, caplog):
|
||||
with caplog.at_level(logging.ERROR):
|
||||
assert helper._create_placeholder_logo("PHI", -5, -5) is None
|
||||
assert "Error creating placeholder" in caplog.text
|
||||
|
||||
|
||||
class TestSessionConfiguration:
|
||||
def test_user_agent_and_accept_headers(self, helper):
|
||||
assert helper.session.headers["User-Agent"] == "LEDMatrix-Common/1.0"
|
||||
assert helper.session.headers["Accept"] == "image/*"
|
||||
|
||||
|
||||
class TestStalePlaceholderHandling:
|
||||
"""load_logo_with_download must not be fooled by a cached placeholder.
|
||||
|
||||
A placeholder wears the real logo's filename, so both the file cache and
|
||||
the in-memory cache can hold one and look like a hit.
|
||||
"""
|
||||
|
||||
def _placeholder(self, tmp_path, abbrev="COLL"):
|
||||
from src.logo_downloader import LogoDownloader
|
||||
assert LogoDownloader().create_placeholder_logo(abbrev, str(tmp_path))
|
||||
return tmp_path / f"{abbrev}.png"
|
||||
|
||||
def _make_stale(self, path):
|
||||
import time
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
from src.logo_downloader import PLACEHOLDER_MARKER, PLACEHOLDER_RETRY_SECONDS
|
||||
metadata = PngInfo()
|
||||
metadata.add_text(PLACEHOLDER_MARKER, str(time.time() - (PLACEHOLDER_RETRY_SECONDS + 60)))
|
||||
with Image.open(path) as img:
|
||||
img.copy().save(path, "PNG", pnginfo=metadata)
|
||||
|
||||
def test_fresh_placeholder_is_served_without_a_download(self, helper, tmp_path):
|
||||
path = self._placeholder(tmp_path)
|
||||
with patch.object(LogoHelper, "_download_logo") as download:
|
||||
assert helper.load_logo_with_download("COLL", path, "http://x/c.png") is not None
|
||||
download.assert_not_called()
|
||||
|
||||
def test_stale_placeholder_triggers_a_download(self, helper, tmp_path):
|
||||
path = self._placeholder(tmp_path)
|
||||
self._make_stale(path)
|
||||
with patch.object(LogoHelper, "_download_logo") as download:
|
||||
helper.load_logo_with_download("COLL", path, "http://x/c.png")
|
||||
download.assert_called_once()
|
||||
|
||||
def test_replacement_logo_is_not_masked_by_the_cached_placeholder(self, helper, tmp_path):
|
||||
"""The bug this guards: load_logo answers from cache before the disk.
|
||||
|
||||
Without invalidation the freshly downloaded logo would not appear until
|
||||
the process restarted.
|
||||
"""
|
||||
path = self._placeholder(tmp_path)
|
||||
first = helper.load_logo_with_download("COLL", path, "http://x/c.png")
|
||||
assert first is not None
|
||||
|
||||
self._make_stale(path)
|
||||
|
||||
def fake_download(_self, _url, file_path):
|
||||
Image.new("RGB", (500, 500), (7, 8, 9)).save(file_path, format="PNG")
|
||||
|
||||
with patch.object(LogoHelper, "_download_logo", fake_download):
|
||||
second = helper.load_logo_with_download("COLL", path, "http://x/c.png")
|
||||
|
||||
assert second is not None
|
||||
from src.logo_downloader import is_placeholder_logo
|
||||
assert is_placeholder_logo(path) is False
|
||||
assert second.getpixel((0, 0))[:3] == (7, 8, 9)
|
||||
|
||||
def test_failed_retry_restarts_the_back_off(self, helper, tmp_path):
|
||||
"""Otherwise a stale placeholder means a download attempt per call."""
|
||||
from src.logo_downloader import should_attempt_download
|
||||
path = self._placeholder(tmp_path)
|
||||
self._make_stale(path)
|
||||
assert should_attempt_download(path) is True
|
||||
|
||||
def boom(_self, _url, _file_path):
|
||||
raise OSError("network down")
|
||||
|
||||
with patch.object(LogoHelper, "_download_logo", boom):
|
||||
helper.load_logo_with_download("COLL", path, "http://x/c.png")
|
||||
|
||||
assert should_attempt_download(path) is False
|
||||
@@ -0,0 +1,128 @@
|
||||
"""A malformed metrics cache entry must not take every plugin down with it.
|
||||
|
||||
`ResourceMetrics(**cached)` raises TypeError on a single unexpected key, and
|
||||
that exception escapes into plugin_manager, which reports it per plugin as
|
||||
"plugin <id> operation failed". Every plugin fails and the plugin system never
|
||||
finishes initialising -- the health endpoint reports
|
||||
`plugin_system: not_initialized` while the display itself keeps running.
|
||||
|
||||
Seen on a live rig, once per plugin, continuously:
|
||||
|
||||
ERROR - src.plugin_system.plugin_manager - plugin geochron operation failed:
|
||||
ResourceMetrics.__init__() got an unexpected keyword argument
|
||||
'consecutive_failures'
|
||||
|
||||
`consecutive_failures` belongs to plugin_health, not to metrics. How a
|
||||
health-shaped record came to sit under a plugin_metrics key on that machine is
|
||||
not established -- a restored backup that mixed two machines' caches is the
|
||||
likeliest explanation, and the same rig had one restored onto it -- but a
|
||||
loader that turns one bad cache entry into a total outage is the part worth
|
||||
fixing. plugin_health already repairs its own records field by field rather
|
||||
than trusting what is on disk.
|
||||
"""
|
||||
import logging
|
||||
from dataclasses import fields
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from src.plugin_system.resource_monitor import PluginResourceMonitor, ResourceMetrics
|
||||
|
||||
|
||||
class _Cache:
|
||||
def __init__(self, payload=None):
|
||||
self.payload = payload
|
||||
|
||||
def get(self, key, max_age=None, memory_ttl=None, **kwargs):
|
||||
return self.payload
|
||||
|
||||
def set(self, key, data, ttl=None, **kwargs):
|
||||
pass
|
||||
|
||||
|
||||
def _monitor(payload):
|
||||
m = PluginResourceMonitor(cache_manager=_Cache(payload))
|
||||
m.logger = logging.getLogger("test")
|
||||
return m
|
||||
|
||||
|
||||
#: What the rig actually had under the metrics key.
|
||||
HEALTH_SHAPED = {
|
||||
"consecutive_failures": 0, "circuit_state": "closed",
|
||||
"circuit_opened_time": None, "half_open_start_time": None,
|
||||
"last_error": None, "last_failure_time": None,
|
||||
"last_success_time": 1_700_000_000.0, "total_failures": 0,
|
||||
"total_successes": 42,
|
||||
}
|
||||
|
||||
|
||||
def test_a_health_record_under_the_metrics_key_does_not_raise():
|
||||
"""The exact failure: it must degrade, not take the plugin system down."""
|
||||
monitor = _monitor(HEALTH_SHAPED)
|
||||
metrics = monitor.get_metrics(" plugin-a".strip())
|
||||
assert isinstance(metrics, ResourceMetrics)
|
||||
|
||||
|
||||
def test_recognised_fields_in_a_mixed_record_are_kept():
|
||||
"""Dropping the record wholesale would lose real history unnecessarily."""
|
||||
mixed = dict(HEALTH_SHAPED, call_count=7, memory_mb=12.5)
|
||||
metrics = _monitor(mixed).get_metrics("plugin-b")
|
||||
assert metrics.call_count == 7
|
||||
assert metrics.memory_mb == 12.5
|
||||
|
||||
|
||||
def test_a_clean_record_still_loads_unchanged():
|
||||
clean = {f.name: 3 for f in fields(ResourceMetrics)}
|
||||
metrics = _monitor(clean).get_metrics("plugin-c")
|
||||
for name in (f.name for f in fields(ResourceMetrics)):
|
||||
assert getattr(metrics, name) == 3
|
||||
|
||||
|
||||
def test_unknown_fields_are_named_in_the_log(caplog):
|
||||
"""Silently discarding them would hide a real schema change."""
|
||||
with caplog.at_level(logging.WARNING):
|
||||
_monitor(HEALTH_SHAPED).get_metrics("plugin-d")
|
||||
# getMessage(), not .message: the latter is only populated once a handler
|
||||
# formats the record, so the obvious spelling silently never matches.
|
||||
assert any("consecutive_failures" in r.getMessage() for r in caplog.records), \
|
||||
caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", ["a string", 42, ["a", "list"]])
|
||||
def test_a_non_mapping_cache_entry_does_not_raise(payload):
|
||||
metrics = _monitor(payload).get_metrics("plugin-e")
|
||||
assert isinstance(metrics, ResourceMetrics)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", [
|
||||
{"call_count": "not a number"},
|
||||
{"memory_mb": None},
|
||||
{"execution_time": {"nested": "junk"}},
|
||||
{"min_execution_time": ["a", "list"]},
|
||||
])
|
||||
def test_values_of_the_wrong_type_fall_back_to_usable_defaults(bad):
|
||||
"""isinstance() alone was not enough.
|
||||
|
||||
A dataclass does not enforce its annotations, so the bad value was simply
|
||||
stored and the old assertion passed -- then monitor_call() raised
|
||||
"can only concatenate str (not \"int\") to str" on the next call. The
|
||||
metrics must come back *usable*, not merely constructed.
|
||||
"""
|
||||
monitor = _monitor(bad)
|
||||
metrics = monitor.get_metrics("plugin-f")
|
||||
assert isinstance(metrics, ResourceMetrics)
|
||||
|
||||
field_name = next(iter(bad))
|
||||
assert isinstance(getattr(metrics, field_name), (int, float)), \
|
||||
f"{field_name} came back as {getattr(metrics, field_name)!r}"
|
||||
|
||||
# The real proof: arithmetic on the loaded metrics must not explode.
|
||||
metrics.call_count += 1
|
||||
metrics.total_execution_time += 0.5
|
||||
metrics.update_average_execution_time()
|
||||
|
||||
|
||||
def test_a_numeric_string_is_accepted_rather_than_discarded():
|
||||
"""JSON round-trips can widen an int to a string; that is recoverable."""
|
||||
metrics = _monitor({"call_count": "7"}).get_metrics("plugin-g")
|
||||
assert metrics.call_count == 7
|
||||
@@ -0,0 +1,241 @@
|
||||
"""
|
||||
Getting Started checklist: what the server decides, and what it must not.
|
||||
|
||||
The timezone step used to tick server-side when the saved timezone differed
|
||||
from the shipped default, OR-ed with the saved city. That made the step
|
||||
unsatisfiable for anyone genuinely in the default zone (the card nagged
|
||||
forever), and let a saved city tick it off while the timezone was still wrong.
|
||||
The step is now verified in the browser against its own zone, so the server's
|
||||
only job is to hand over the configured value and stay out of the decision.
|
||||
|
||||
These tests pin that contract: the panel-size step still reflects config, the
|
||||
timezone step never pre-ticks, it carries the configured zone, and the city
|
||||
has no influence on it.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
BASE_CONFIG = {
|
||||
"timezone": "America/New_York",
|
||||
"location": {"city": "Tampa", "state": "Florida", "country": "US"},
|
||||
"display": {
|
||||
"hardware": {"rows": 32, "cols": 64, "chain_length": 2, "parallel": 1},
|
||||
"runtime": {},
|
||||
"double_sided": {"enabled": False},
|
||||
"vegas_scroll": {"plugin_order": [], "excluded_plugins": []},
|
||||
"plugin_rotation_order": [],
|
||||
},
|
||||
"plugin_system": {},
|
||||
"schedule": {},
|
||||
"dim_schedule": {},
|
||||
"sync": {},
|
||||
}
|
||||
|
||||
|
||||
def render(config):
|
||||
"""Render the overview partial against one config, as app.py would."""
|
||||
base = PROJECT_ROOT / "web_interface"
|
||||
app = Flask(
|
||||
__name__,
|
||||
template_folder=str(base / "templates"),
|
||||
static_folder=str(base / "static"),
|
||||
)
|
||||
app.config["TESTING"] = True
|
||||
|
||||
from web_interface.blueprints import pages_v3 as pv
|
||||
|
||||
# pages_v3 is a module-level singleton shared across the test process;
|
||||
# restore whatever the previous test left on it.
|
||||
original_cm = getattr(pv.pages_v3, "config_manager", None)
|
||||
original_pm = getattr(pv.pages_v3, "plugin_manager", None)
|
||||
|
||||
mock_cm = MagicMock()
|
||||
mock_cm.load_config.return_value = config
|
||||
mock_cm.get_raw_file_content.return_value = config
|
||||
pv.pages_v3.config_manager = mock_cm
|
||||
|
||||
mock_pm = MagicMock()
|
||||
mock_pm.plugins = {}
|
||||
mock_pm.get_all_plugin_info.return_value = []
|
||||
mock_pm.get_plugin_display_modes.side_effect = lambda pid: []
|
||||
pv.pages_v3.plugin_manager = mock_pm
|
||||
|
||||
app.register_blueprint(pv.pages_v3, url_prefix="")
|
||||
try:
|
||||
resp = app.test_client().get("/partials/overview")
|
||||
assert resp.status_code == 200, resp.status_code
|
||||
return resp.get_data(as_text=True)
|
||||
finally:
|
||||
pv.pages_v3.config_manager = original_cm
|
||||
pv.pages_v3.plugin_manager = original_pm
|
||||
|
||||
|
||||
def timezone_step(body):
|
||||
"""The checklist <button> for the timezone step."""
|
||||
match = re.search(r"<button[^>]*data-check=\"timezone\"[^>]*>", body)
|
||||
assert match, "timezone step not found in the rendered checklist"
|
||||
return match.group(0)
|
||||
|
||||
|
||||
def config_with(**overrides):
|
||||
config = copy.deepcopy(BASE_CONFIG)
|
||||
for key, value in overrides.items():
|
||||
config[key] = value
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"timezone",
|
||||
["America/New_York", "America/Los_Angeles", "Europe/Madrid", "Asia/Kolkata"],
|
||||
)
|
||||
def test_timezone_step_never_pre_ticks_server_side(timezone):
|
||||
"""The browser owns this decision; the server must not pre-empt it.
|
||||
|
||||
The default zone is in the list deliberately: that is the case the old
|
||||
default-comparison could never tick.
|
||||
"""
|
||||
step = timezone_step(render(config_with(timezone=timezone)))
|
||||
assert 'data-done="0"' in step, step
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"timezone",
|
||||
["America/New_York", "Europe/Madrid", "Pacific/Auckland"],
|
||||
)
|
||||
def test_timezone_step_carries_the_configured_zone(timezone):
|
||||
"""JS compares data-tz against the browser, so it has to be the real value."""
|
||||
assert f'data-tz="{timezone}"' in timezone_step(render(config_with(timezone=timezone)))
|
||||
|
||||
|
||||
def test_city_does_not_influence_the_timezone_step():
|
||||
"""The coupling this change removes: city said nothing about the timezone,
|
||||
and OR-ing it let a saved city tick the step off with the zone still wrong.
|
||||
|
||||
timezone_step() returns the opening tag only, so this compares the state
|
||||
the step is in -- data-done and data-tz -- and not the label, which does
|
||||
still show the configured city as context and so differs between the two.
|
||||
"""
|
||||
tampa = timezone_step(render(config_with(
|
||||
location={"city": "Tampa", "state": "Florida", "country": "US"})))
|
||||
seattle = timezone_step(render(config_with(
|
||||
location={"city": "Seattle", "state": "Washington", "country": "US"})))
|
||||
assert tampa == seattle
|
||||
|
||||
|
||||
def test_missing_timezone_leaves_the_step_open():
|
||||
"""Nothing saved means nothing to verify: the step stays unticked and the
|
||||
JS bails on the empty value rather than comparing against ''."""
|
||||
step = timezone_step(render(config_with(timezone="")))
|
||||
assert 'data-tz=""' in step
|
||||
assert 'data-done="0"' in step
|
||||
|
||||
|
||||
def test_zone_comparison_asks_for_the_time_of_day():
|
||||
"""Guard on the Intl options, which look like a stylistic choice.
|
||||
|
||||
dateStyle/timeStyle are late additions (Firefox shipped them in 91). An
|
||||
implementation that does not know them ignores them and formats the date
|
||||
alone -- which compares New York, Chicago and Madrid as equal and ticks
|
||||
the step for a timezone that is plainly wrong. Explicit numeric fields
|
||||
have been in Intl since ECMA-402 v1.
|
||||
"""
|
||||
template = (PROJECT_ROOT / "web_interface" / "templates" / "v3"
|
||||
/ "partials" / "overview.html").read_text()
|
||||
body = template[template.index("function sameZone"):]
|
||||
body = body[:body.index("}())")]
|
||||
# The comment above the options names dateStyle/timeStyle to explain why
|
||||
# they are not used, so match on code only.
|
||||
body = "\n".join(line for line in body.splitlines()
|
||||
if not line.lstrip().startswith("//"))
|
||||
assert "dateStyle" not in body and "timeStyle" not in body, (
|
||||
"zone comparison must not depend on dateStyle/timeStyle")
|
||||
for field in ("hour:", "minute:", "year:", "month:", "day:"):
|
||||
assert field in body, f"zone comparison dropped {field!r}"
|
||||
|
||||
|
||||
def test_zone_comparison_samples_both_sides_of_dst():
|
||||
"""One instant is not enough, and the shortfall is invisible for months.
|
||||
|
||||
America/New_York and America/Lima hold the same offset all winter, so a
|
||||
check against now alone ticks the step in January for a panel that runs an
|
||||
hour off from March. The comparison has to sample instants either side of
|
||||
DST -- mid-January and mid-July, which covers both hemispheres.
|
||||
"""
|
||||
template = (PROJECT_ROOT / "web_interface" / "templates" / "v3"
|
||||
/ "partials" / "overview.html").read_text()
|
||||
body = template[template.index("function sameZone"):]
|
||||
body = body[:body.index("}())")]
|
||||
code = "\n".join(line for line in body.splitlines()
|
||||
if not line.lstrip().startswith("//"))
|
||||
assert "Date.UTC" in code, (
|
||||
"zone comparison samples only the current instant, so zones that "
|
||||
"coincide seasonally would read as equal")
|
||||
assert code.count("Date.UTC") >= 2, "expected an instant either side of DST"
|
||||
|
||||
|
||||
def _stamp(zone, instant):
|
||||
"""The JS comparison's algorithm, for pinning what it must decide.
|
||||
|
||||
There is no JS runtime here (and the repo has no JS test infra), so this
|
||||
mirrors sameZone rather than executing it: same instants, same wall-clock
|
||||
equality. It records the verdicts the shipped code has to reach.
|
||||
"""
|
||||
from zoneinfo import ZoneInfo
|
||||
return instant.astimezone(ZoneInfo(zone)).strftime("%m/%d/%Y %H:%M")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"left,right,equivalent",
|
||||
[
|
||||
# Aliases: one zone under two names.
|
||||
("Asia/Calcutta", "Asia/Kolkata", True),
|
||||
("Europe/Kiev", "Europe/Kyiv", True),
|
||||
# Same rules year-round: either renders the same times, so a panel set
|
||||
# to one and browsed from the other is correctly configured.
|
||||
("America/New_York", "America/Toronto", True),
|
||||
# Coincide in winter only -- the case a single-instant check gets wrong.
|
||||
("America/New_York", "America/Lima", False),
|
||||
("America/Phoenix", "America/Los_Angeles", False),
|
||||
("Australia/Sydney", "Pacific/Guadalcanal", False),
|
||||
# Plainly different.
|
||||
("America/New_York", "America/Chicago", False),
|
||||
("America/New_York", "Europe/Madrid", False),
|
||||
],
|
||||
)
|
||||
def test_which_zone_pairs_must_count_as_the_same(left, right, equivalent):
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
year = 2026
|
||||
instants = [datetime(year, 1, 15, 12, tzinfo=ZoneInfo("UTC")),
|
||||
datetime(year, 7, 15, 12, tzinfo=ZoneInfo("UTC"))]
|
||||
matched = all(_stamp(left, at) == _stamp(right, at) for at in instants)
|
||||
assert matched is equivalent, (
|
||||
f"{left} vs {right}: sampling both seasons gave {matched}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hardware,expected",
|
||||
[
|
||||
({"rows": 32, "cols": 64, "chain_length": 2, "parallel": 1}, "1"),
|
||||
({"rows": 0, "cols": 0, "chain_length": 0, "parallel": 1}, "0"),
|
||||
],
|
||||
)
|
||||
def test_panel_size_step_still_reflects_config(hardware, expected):
|
||||
"""Regression guard: the hardware step is still decided server-side."""
|
||||
config = config_with()
|
||||
config["display"]["hardware"] = hardware
|
||||
body = render(config)
|
||||
match = re.search(r"<button[^>]*data-tab=\"display\"[^>]*>", body)
|
||||
assert match, "panel-size step not found"
|
||||
assert f'data-done="{expected}"' in match.group(0), match.group(0)
|
||||
@@ -0,0 +1,180 @@
|
||||
"""
|
||||
Tests for scripts/download_pixlet.sh -- release-tag resolution and download guards.
|
||||
|
||||
Background: Starlark apps render through the pixlet binary, and the installer
|
||||
that fetches it failed silently. It resolved the release tag by grepping the
|
||||
GitHub API response for '"tag_name"' and taking the last quoted token on the
|
||||
match with a greedy sed. When the response arrives on one line that token is
|
||||
"mentions_count", not the tag, so the script built a URL for a release that
|
||||
cannot exist -- and `curl -L -o` without -f wrote the 404 body to the file and
|
||||
exited 0, so the first sign of trouble was tar reporting "not in gzip format"
|
||||
about a page of HTML.
|
||||
|
||||
The API is pretty-printed by default, which is exactly why this needs a test:
|
||||
by hand the old command looks correct, and the failure only appears when the
|
||||
formatting changes. These drive the real script with a stubbed curl on PATH, so
|
||||
both response shapes are covered without touching the network.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "download_pixlet.sh"
|
||||
|
||||
PRETTY = """{
|
||||
"url": "https://api.github.com/repos/tronbyt/pixlet/releases/12345",
|
||||
"id": 12345,
|
||||
"tag_name": "v0.53.1",
|
||||
"name": "v0.53.1",
|
||||
"draft": false,
|
||||
"prerelease": false,
|
||||
"mentions_count": 3
|
||||
}
|
||||
"""
|
||||
|
||||
# The shape that broke it: one line, and the last quoted token is not the tag.
|
||||
MINIFIED = (
|
||||
'{"url":"https://api.github.com/repos/tronbyt/pixlet/releases/12345",'
|
||||
'"id":12345,"tag_name":"v0.53.1","name":"v0.53.1","draft":false,'
|
||||
'"prerelease":false,"mentions_count":3}'
|
||||
)
|
||||
|
||||
|
||||
def run_script(tmp_path, api_body, download=None):
|
||||
"""Run the real script against a stubbed curl.
|
||||
|
||||
Args:
|
||||
api_body: what the stub returns for the api.github.com request.
|
||||
download: bytes to write for a release-asset request, or None to make
|
||||
that request fail the way `curl -f` does on an HTTP error.
|
||||
"""
|
||||
root = tmp_path / "project"
|
||||
(root / "scripts").mkdir(parents=True)
|
||||
shutil.copy(SCRIPT, root / "scripts" / "download_pixlet.sh")
|
||||
|
||||
api_file = tmp_path / "api.json"
|
||||
api_file.write_text(api_body)
|
||||
|
||||
stub_dir = tmp_path / "stub"
|
||||
stub_dir.mkdir()
|
||||
asset_file = tmp_path / "asset.bin"
|
||||
if download is not None:
|
||||
asset_file.write_bytes(download)
|
||||
|
||||
# Stands in for curl, including the -f semantics the fix turns on: without
|
||||
# -f, real curl writes the error body to the output file and exits 0, which
|
||||
# is what let a 404 masquerade as a successful download. The stub has to
|
||||
# honour that or a test of the fix would pass against the old script too.
|
||||
(stub_dir / "curl").write_text(f"""#!/bin/bash
|
||||
out=""
|
||||
url=""
|
||||
fail_on_error=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-o) out="$2"; shift 2 ;;
|
||||
-*f*) fail_on_error=1; shift ;;
|
||||
-*) shift ;;
|
||||
*) url="$1"; shift ;;
|
||||
esac
|
||||
done
|
||||
if [[ "$url" == *api.github.com* ]]; then
|
||||
cat {api_file}
|
||||
exit 0
|
||||
fi
|
||||
if [ -f "{asset_file}" ]; then
|
||||
cp "{asset_file}" "$out"
|
||||
exit 0
|
||||
fi
|
||||
# No asset: stand in for an HTTP 404.
|
||||
if [ "$fail_on_error" = "1" ]; then
|
||||
exit 22
|
||||
fi
|
||||
printf '<!DOCTYPE html><html>404 Not Found</html>' > "$out"
|
||||
exit 0
|
||||
""")
|
||||
(stub_dir / "curl").chmod(0o755)
|
||||
|
||||
return subprocess.run(
|
||||
["bash", str(root / "scripts" / "download_pixlet.sh")],
|
||||
capture_output=True, text=True,
|
||||
env={"PATH": f"{stub_dir}:/usr/bin:/bin:/usr/sbin:/sbin",
|
||||
"PIXLET_VERSION": "latest"},
|
||||
)
|
||||
|
||||
|
||||
def resolved_version(result):
|
||||
match = re.search(r"^Version: (.+)$", result.stdout, re.M)
|
||||
assert match, f"no version line in output:\n{result.stdout}"
|
||||
return match.group(1).strip()
|
||||
|
||||
|
||||
def test_script_is_syntactically_valid():
|
||||
result = subprocess.run(["bash", "-n", str(SCRIPT)], capture_output=True, text=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body,label", [(PRETTY, "pretty"), (MINIFIED, "minified")])
|
||||
def test_tag_is_resolved_from_either_response_shape(tmp_path, body, label):
|
||||
"""The minified case is the regression: the last quoted token there is
|
||||
"mentions_count", which is what the old greedy sed captured."""
|
||||
result = run_script(tmp_path, body)
|
||||
assert resolved_version(result) == "v0.53.1", f"{label}: {result.stdout}"
|
||||
assert "mentions_count" not in result.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tag",
|
||||
["mentions_count", "v0.53garbage", "0.53", "v0.5", "", "v0.53.1 ; echo pwned"],
|
||||
)
|
||||
def test_a_tag_that_is_not_a_release_falls_back(tmp_path, tag):
|
||||
"""A wrong-but-non-empty value is what made the original bug silent, so the
|
||||
check is on the shape. Partial matches must not pass: "v0.53garbage" and
|
||||
"0.53" would build a URL for a release that cannot exist."""
|
||||
result = run_script(tmp_path, '{"tag_name": "%s"}' % tag)
|
||||
assert resolved_version(result) == "v0.50.2", result.stdout
|
||||
assert "using fallback" in result.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tag", ["v0.53.1", "v1.0.0", "v0.54.0-rc.1", "v1.2.3+build.4"])
|
||||
def test_real_release_tag_shapes_are_accepted(tmp_path, tag):
|
||||
assert resolved_version(run_script(tmp_path, '{"tag_name": "%s"}' % tag)) == tag
|
||||
|
||||
|
||||
def test_an_http_error_is_reported_as_a_failed_download(tmp_path):
|
||||
"""Without curl -f the 404 body lands in the file and curl exits 0, so the
|
||||
failure surfaced two steps later as tar complaining about gzip -- about
|
||||
what was really a page of HTML. It has to be reported where it happened.
|
||||
|
||||
Both versions end at 0/1, so asserting only on the count would pass against
|
||||
the old script; the discriminating part is which layer reports it.
|
||||
"""
|
||||
result = run_script(tmp_path, PRETTY, download=None)
|
||||
assert "Download complete: 0/1 succeeded" in result.stdout
|
||||
assert "✓ Downloaded" not in result.stdout
|
||||
assert "Failed to download" in result.stdout
|
||||
assert "Failed to extract" not in result.stdout, (
|
||||
"an HTTP error should not surface as an extraction failure")
|
||||
|
||||
|
||||
def test_a_non_archive_response_is_rejected_before_extraction(tmp_path):
|
||||
result = run_script(tmp_path, PRETTY, download=b"<!DOCTYPE html><html>502 Bad Gateway")
|
||||
assert "not a gzip archive" in result.stdout
|
||||
assert "Download complete: 0/1 succeeded" in result.stdout
|
||||
|
||||
|
||||
def test_the_diagnostic_cannot_smuggle_terminal_escapes(tmp_path):
|
||||
"""Those bytes come from whatever answered the request. An error page
|
||||
carrying escapes must not be able to rewrite the output or bury it."""
|
||||
hostile = b"<!DOCTYPE html>\x1b[2J\x1b[31mgone\x1b[0m\rHTTP 200 OK\x08\x08"
|
||||
result = run_script(tmp_path, PRETTY, download=hostile)
|
||||
assert "not a gzip archive" in result.stdout
|
||||
printed = re.search(r"^\s*\(first bytes: (.*)\)$", result.stdout, re.M)
|
||||
assert printed, f"no diagnostic line:\n{result.stdout}"
|
||||
assert "DOCTYPE" in printed.group(1), "the useful part of the page was dropped"
|
||||
for forbidden in ("\x1b", "\r", "\x08", "\x00"):
|
||||
assert forbidden not in printed.group(1), (
|
||||
f"control byte {forbidden!r} reached the terminal")
|
||||
@@ -19,6 +19,7 @@ The rules being pinned here, in priority order:
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -146,9 +147,18 @@ def store(tmp_path, monkeypatch):
|
||||
|
||||
|
||||
class TestInstallGate:
|
||||
"""`install_plugin` is the chokepoint: `_reinstall_with_rollback` calls it,
|
||||
so gating there covers updates too, and a refused update restores the
|
||||
version the user already had."""
|
||||
"""`install_plugin` is the chokepoint for every route that re-downloads:
|
||||
`_reinstall_with_rollback` calls it, so a refused update restores the
|
||||
version the user already had.
|
||||
|
||||
It is not the *only* route in, and saying so here once is cheaper than
|
||||
rediscovering it. `update_plugin` also has a git branch that pulls in
|
||||
place and never re-downloads; that one is gated separately by
|
||||
`_gate_pulled_commit` and pinned in `TestGitPullGate` below. A third
|
||||
route, `install_from_url` (sideloading from a URL), is gated in that
|
||||
function and pinned in `TestSideloadGate`. All three refuse on the same
|
||||
rule.
|
||||
"""
|
||||
|
||||
def _install_with_manifest(self, store, manifest, core_version, monkeypatch):
|
||||
mgr, plugins_dir = store
|
||||
@@ -208,6 +218,291 @@ class TestInstallGate:
|
||||
assert (path / "manifest.json").exists()
|
||||
|
||||
|
||||
class TestSideloadGate:
|
||||
"""`install_from_url` never looked at the core version.
|
||||
|
||||
Sideloading is an explicit act rather than an automatic store update, so
|
||||
the argument for gating it is different: not "the user did not choose
|
||||
this", but that the floor states the plugin *cannot run here*. Letting it
|
||||
through produces the same silent PluginState.ERROR at load that the store
|
||||
gate exists to prevent, and the user who typed the URL is no better placed
|
||||
to diagnose it than one who pressed Update.
|
||||
"""
|
||||
|
||||
def _sideload(self, store, manifest, core_version, monkeypatch):
|
||||
mgr, plugins_dir = store
|
||||
plugin_id = manifest["id"]
|
||||
|
||||
def fake_clone(repo_url, target_path, branches=None):
|
||||
target_path.mkdir(parents=True, exist_ok=True)
|
||||
(target_path / "manifest.json").write_text(
|
||||
json.dumps(manifest), encoding="utf-8")
|
||||
(target_path / "manager.py").write_text(
|
||||
"class P: pass\n", encoding="utf-8")
|
||||
return "main"
|
||||
|
||||
monkeypatch.setattr(mgr, "_install_via_git", fake_clone)
|
||||
monkeypatch.setattr(mgr, "_install_dependencies", lambda *a, **k: True)
|
||||
|
||||
import src
|
||||
monkeypatch.setattr(src, "__version__", core_version)
|
||||
return mgr.install_from_url("https://example.invalid/plugin"), \
|
||||
plugins_dir / plugin_id
|
||||
|
||||
def test_refuses_a_plugin_that_needs_a_newer_core(self, store, monkeypatch):
|
||||
manifest = {
|
||||
"id": "sideload-newer", "name": "Sideload", "class_name": "P",
|
||||
"display_modes": ["a"], "min_ledmatrix_version": "9.9.9",
|
||||
}
|
||||
result, path = self._sideload(store, manifest, "3.2.0", monkeypatch)
|
||||
|
||||
assert result["success"] is False
|
||||
assert "9.9.9" in result["error"], result["error"]
|
||||
assert not path.exists(), (
|
||||
"a refused sideload must not leave the plugin installed")
|
||||
|
||||
def test_allows_a_compatible_plugin(self, store, monkeypatch):
|
||||
"""The guard against over-refusing: a gate that blocks everything
|
||||
passes the test above and breaks sideloading entirely."""
|
||||
manifest = {
|
||||
"id": "sideload-fine", "name": "Sideload", "class_name": "P",
|
||||
"display_modes": ["a"], "min_ledmatrix_version": "3.0.0",
|
||||
}
|
||||
result, path = self._sideload(store, manifest, "3.2.0", monkeypatch)
|
||||
|
||||
assert result["success"] is True, result.get("error")
|
||||
assert (path / "manifest.json").exists()
|
||||
|
||||
def test_untrustworthy_core_does_not_block_a_2_0_0_floor(
|
||||
self, store, monkeypatch):
|
||||
"""Same rule as the other two routes: a v3.1.0 release reports 1.0.0,
|
||||
and nearly every published manifest floors at 2.0.0."""
|
||||
manifest = {
|
||||
"id": "sideload-floored", "name": "Sideload", "class_name": "P",
|
||||
"display_modes": ["a"], "versions": [{"ledmatrix_min": "2.0.0"}],
|
||||
}
|
||||
result, _ = self._sideload(store, manifest, "1.0.0", monkeypatch)
|
||||
|
||||
assert result["success"] is True, result.get("error")
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The git-pull update path — the one route that does not re-download
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _git_available() -> bool:
|
||||
# OSError, not just a non-zero exit: with no git on PATH subprocess raises
|
||||
# FileNotFoundError, and this runs at import time -- before skipif can act,
|
||||
# so the whole module would error out instead of skipping.
|
||||
try:
|
||||
return subprocess.run(
|
||||
['git', '--version'], capture_output=True).returncode == 0
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
_HAS_GIT = _git_available()
|
||||
|
||||
|
||||
def _git(*args, cwd):
|
||||
return subprocess.run(['git', *args], cwd=str(cwd),
|
||||
capture_output=True, text=True, check=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _HAS_GIT, reason='git not available')
|
||||
class TestGitPullGate:
|
||||
"""A plugin installed as a git checkout updates by pulling in place, so it
|
||||
never passes through `install_plugin` and was never gated.
|
||||
|
||||
Real git repositories rather than mocks, because the claim under test is
|
||||
that `git reset --hard` puts the checkout back — a mock of git would only
|
||||
prove the call was made, which is the easy half.
|
||||
|
||||
Only the handful of registry entries with no `plugin_path` reach this path
|
||||
in practice; monorepo plugins install as archives and update through
|
||||
`_reinstall_with_rollback`. It is gated anyway because the sunset rule in
|
||||
`docs/plugin-development/08-shared-sports-code.md` states the core enforces
|
||||
the floor "at install/update time", and a precondition that is documented
|
||||
but not true is worse than one that is merely missing.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def checkout(self, tmp_path, monkeypatch):
|
||||
"""An origin repo holding a plugin, and a clone of it installed as
|
||||
`plugin-repos/gitplug`, with the store pointed at it."""
|
||||
from src.plugin_system.store_manager import PluginStoreManager
|
||||
|
||||
origin = tmp_path / 'origin'
|
||||
origin.mkdir()
|
||||
_git('init', '--initial-branch=main', '--bare', cwd=origin)
|
||||
|
||||
seed = tmp_path / 'seed'
|
||||
_git('clone', str(origin), str(seed), cwd=tmp_path)
|
||||
_git('config', 'user.email', 'test@example.com', cwd=seed)
|
||||
_git('config', 'user.name', 'Test', cwd=seed)
|
||||
(seed / 'manifest.json').write_text(json.dumps({
|
||||
"id": "gitplug", "name": "Git Plug", "class_name": "P",
|
||||
"display_modes": ["a"], "min_ledmatrix_version": "2.0.0",
|
||||
}), encoding='utf-8')
|
||||
(seed / 'manager.py').write_text("class P: pass\n", encoding='utf-8')
|
||||
_git('add', '-A', cwd=seed)
|
||||
_git('commit', '-m', 'initial', cwd=seed)
|
||||
_git('push', '-u', 'origin', 'main', cwd=seed)
|
||||
|
||||
plugins_dir = tmp_path / 'plugin-repos'
|
||||
plugins_dir.mkdir()
|
||||
work = plugins_dir / 'gitplug'
|
||||
_git('clone', str(origin), str(work), cwd=tmp_path)
|
||||
_git('config', 'user.email', 'test@example.com', cwd=work)
|
||||
_git('config', 'user.name', 'Test', cwd=work)
|
||||
|
||||
mgr = PluginStoreManager(plugins_dir=str(plugins_dir))
|
||||
mgr.logger = MagicMock()
|
||||
# No registry entry, so update_plugin takes the plain-pull branch
|
||||
# rather than the remote-mismatch or already-current shortcuts.
|
||||
monkeypatch.setattr(mgr, 'fetch_registry', lambda *a, **k: None)
|
||||
monkeypatch.setattr(mgr, 'get_plugin_info', lambda *a, **k: None)
|
||||
monkeypatch.setattr(mgr, '_install_dependencies', lambda *a, **k: True)
|
||||
return mgr, seed, work
|
||||
|
||||
def _push(self, seed, manifest_text):
|
||||
(seed / 'manifest.json').write_text(manifest_text, encoding='utf-8')
|
||||
_git('add', '-A', cwd=seed)
|
||||
_git('commit', '-m', 'update', cwd=seed)
|
||||
_git('push', 'origin', 'main', cwd=seed)
|
||||
|
||||
def _head(self, work):
|
||||
return _git('rev-parse', 'HEAD', cwd=work).stdout.strip()
|
||||
|
||||
def test_refuses_a_pulled_commit_that_needs_a_newer_core(
|
||||
self, checkout, monkeypatch):
|
||||
mgr, seed, work = checkout
|
||||
before = self._head(work)
|
||||
self._push(seed, json.dumps({
|
||||
"id": "gitplug", "name": "Git Plug", "class_name": "P",
|
||||
"display_modes": ["a"], "min_ledmatrix_version": "9.9.9",
|
||||
}))
|
||||
|
||||
import src
|
||||
monkeypatch.setattr(src, '__version__', '3.2.0')
|
||||
assert mgr.update_plugin('gitplug') is False
|
||||
|
||||
assert self._head(work) == before, (
|
||||
"a refused update must leave the checkout on the commit it was "
|
||||
"already running, not on the one it cannot load")
|
||||
floor = json.loads((work / 'manifest.json').read_text())
|
||||
assert floor['min_ledmatrix_version'] == '2.0.0', (
|
||||
"the reset must restore the working tree, not just the ref")
|
||||
|
||||
def test_allows_a_pulled_commit_the_core_can_run(
|
||||
self, checkout, monkeypatch):
|
||||
"""The guard against over-refusing. A gate that refuses everything
|
||||
passes the test above and breaks every update."""
|
||||
mgr, seed, work = checkout
|
||||
before = self._head(work)
|
||||
self._push(seed, json.dumps({
|
||||
"id": "gitplug", "name": "Git Plug", "class_name": "P",
|
||||
"display_modes": ["a"], "min_ledmatrix_version": "3.0.0",
|
||||
}))
|
||||
|
||||
import src
|
||||
monkeypatch.setattr(src, '__version__', '3.2.0')
|
||||
assert mgr.update_plugin('gitplug') is True
|
||||
assert self._head(work) != before
|
||||
|
||||
def test_an_unreadable_manifest_after_pull_is_not_a_refusal(
|
||||
self, checkout, monkeypatch):
|
||||
"""Rule 1 of this file — refuse only on evidence. A manifest that
|
||||
will not parse declares no floor, so it is not evidence of anything."""
|
||||
mgr, seed, work = checkout
|
||||
self._push(seed, '{ this is not json')
|
||||
|
||||
import src
|
||||
monkeypatch.setattr(src, '__version__', '3.2.0')
|
||||
assert mgr.update_plugin('gitplug') is True
|
||||
|
||||
def test_an_untrustworthy_core_does_not_block_a_2_0_0_floor_on_pull(
|
||||
self, checkout, monkeypatch):
|
||||
"""The same regression guard as
|
||||
`test_untrustworthy_core_does_not_block_installs`, because this path
|
||||
now shares that rule: a v3.1.0 release reports 1.0.0, and nearly every
|
||||
published manifest floors at 2.0.0."""
|
||||
mgr, seed, work = checkout
|
||||
before = self._head(work)
|
||||
self._push(seed, json.dumps({
|
||||
"id": "gitplug", "name": "Git Plug", "class_name": "P",
|
||||
"display_modes": ["a"], "min_ledmatrix_version": "2.0.0",
|
||||
"description": "changed",
|
||||
}))
|
||||
|
||||
import src
|
||||
monkeypatch.setattr(src, '__version__', '1.0.0')
|
||||
assert mgr.update_plugin('gitplug') is True
|
||||
assert self._head(work) != before
|
||||
|
||||
def test_a_failed_stash_stops_the_update_before_pulling(
|
||||
self, checkout, monkeypatch):
|
||||
"""Uncommitted work is not collateral for the gate.
|
||||
|
||||
The gate's only rollback is `git reset --hard`, which discards
|
||||
uncommitted tracked edits. update_plugin stashes them first -- but when
|
||||
that stash fails it used to pull anyway, and a pull touching different
|
||||
files succeeds on a dirty tree. Refuse, refuse the rollback's rollback,
|
||||
and the user's edits go with it.
|
||||
"""
|
||||
mgr, seed, work = checkout
|
||||
before = self._head(work)
|
||||
(work / 'manager.py').write_text(
|
||||
"class P:\n MINE = 'do not lose this'\n", encoding='utf-8')
|
||||
self._push(seed, json.dumps({
|
||||
"id": "gitplug", "name": "Git Plug", "class_name": "P",
|
||||
"display_modes": ["a"], "min_ledmatrix_version": "9.9.9",
|
||||
}))
|
||||
|
||||
real_run = subprocess.run
|
||||
|
||||
def fail_stash(cmd, *a, **k):
|
||||
if isinstance(cmd, list) and 'stash' in cmd:
|
||||
return subprocess.CompletedProcess(cmd, 1, '', 'stash boom')
|
||||
return real_run(cmd, *a, **k)
|
||||
|
||||
import src
|
||||
monkeypatch.setattr(src, '__version__', '3.2.0')
|
||||
monkeypatch.setattr(
|
||||
'src.plugin_system.store_manager.subprocess.run', fail_stash)
|
||||
assert mgr.update_plugin('gitplug') is False
|
||||
|
||||
assert self._head(work) == before, "must not pull what it cannot undo"
|
||||
assert 'do not lose this' in (work / 'manager.py').read_text(), (
|
||||
"the local edit the stash failed to save must still be there")
|
||||
|
||||
def test_rollback_reports_the_recovery_command_when_git_fails(
|
||||
self, checkout, monkeypatch):
|
||||
"""If the reset itself fails the user is left on a version that cannot
|
||||
load, so the log line has to carry the command that fixes it —
|
||||
it is the only thing standing between them and a manual reinstall."""
|
||||
mgr, seed, work = checkout
|
||||
self._push(seed, json.dumps({
|
||||
"id": "gitplug", "name": "Git Plug", "class_name": "P",
|
||||
"display_modes": ["a"], "min_ledmatrix_version": "9.9.9",
|
||||
}))
|
||||
|
||||
real_run = subprocess.run
|
||||
|
||||
def fail_reset(cmd, *a, **k):
|
||||
if isinstance(cmd, list) and 'reset' in cmd:
|
||||
return subprocess.CompletedProcess(cmd, 1, '', 'reset boom')
|
||||
return real_run(cmd, *a, **k)
|
||||
|
||||
import src
|
||||
monkeypatch.setattr(src, '__version__', '3.2.0')
|
||||
monkeypatch.setattr(
|
||||
'src.plugin_system.store_manager.subprocess.run', fail_reset)
|
||||
assert mgr.update_plugin('gitplug') is False
|
||||
|
||||
logged = ' '.join(str(c) for c in mgr.logger.error.call_args_list)
|
||||
assert 'reset --hard' in logged, logged
|
||||
|
||||
|
||||
class TestLoaderAndStoreAgree:
|
||||
"""Both read the same manifests; a disagreement means one of them is
|
||||
lying to the user."""
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discovery must say when it skips a directory.
|
||||
|
||||
A plugin can be enabled in config, enabled in plugin state, present on disk
|
||||
with a valid entry point -- and simply absent from the running process, with
|
||||
nothing in the journal to say why. Working that out afterwards meant comparing
|
||||
cache-file mtimes to find when it had last run.
|
||||
|
||||
Two paths were silent. A directory with no manifest.json was ignored, and --
|
||||
quieter still -- a manifest that parsed but carried no "id" was read
|
||||
successfully and then dropped on the floor.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from src.plugin_system.plugin_manager import PluginManager # noqa: E402
|
||||
|
||||
|
||||
def _manager(tmp_path):
|
||||
pm = PluginManager.__new__(PluginManager)
|
||||
pm.plugins_dir = tmp_path
|
||||
pm.logger = logging.getLogger("test.discovery")
|
||||
pm.plugin_manifests = {}
|
||||
pm.plugin_directories = {}
|
||||
pm._discovery_lock = __import__("threading").RLock()
|
||||
pm._skip_reported = set()
|
||||
pm.schema_manager = MagicMock()
|
||||
return pm
|
||||
|
||||
|
||||
def test_a_directory_without_a_manifest_is_reported(tmp_path, caplog):
|
||||
(tmp_path / "not-a-plugin").mkdir()
|
||||
pm = _manager(tmp_path)
|
||||
with caplog.at_level(logging.WARNING, logger="test.discovery"):
|
||||
pm._scan_directory_for_plugins(tmp_path)
|
||||
joined = " ".join(r.message for r in caplog.records)
|
||||
assert "not-a-plugin" in joined and "manifest" in joined, (
|
||||
f"skip was silent; log said: {joined!r}")
|
||||
|
||||
|
||||
def test_a_manifest_without_an_id_is_reported(tmp_path, caplog):
|
||||
d = tmp_path / "idless"
|
||||
d.mkdir()
|
||||
(d / "manifest.json").write_text(json.dumps({"name": "No Id", "version": "1.0.0"}))
|
||||
pm = _manager(tmp_path)
|
||||
with caplog.at_level(logging.WARNING, logger="test.discovery"):
|
||||
pm._scan_directory_for_plugins(tmp_path)
|
||||
joined = " ".join(r.message for r in caplog.records)
|
||||
assert "idless" in joined and "id" in joined, (
|
||||
f"a parsed-but-unusable manifest vanished silently; log said: {joined!r}")
|
||||
|
||||
|
||||
def test_a_good_plugin_still_registers(tmp_path, caplog):
|
||||
d = tmp_path / "real-plugin"
|
||||
d.mkdir()
|
||||
(d / "manifest.json").write_text(json.dumps(
|
||||
{"id": "real-plugin", "name": "Real", "version": "1.0.0"}))
|
||||
pm = _manager(tmp_path)
|
||||
pm._scan_directory_for_plugins(tmp_path)
|
||||
assert "real-plugin" in pm.plugin_manifests, "a valid plugin was not registered"
|
||||
|
||||
|
||||
def test_the_warning_does_not_repeat_on_every_scan(tmp_path, caplog):
|
||||
"""Discovery runs on every web UI page load and every config reconcile.
|
||||
|
||||
Warning unconditionally would put a line in the journal each time someone
|
||||
opened a page -- the same log-volume problem this is meant to help
|
||||
diagnose.
|
||||
"""
|
||||
(tmp_path / "not-a-plugin").mkdir()
|
||||
pm = _manager(tmp_path)
|
||||
with caplog.at_level(logging.WARNING, logger="test.discovery"):
|
||||
for _ in range(5):
|
||||
pm._scan_directory_for_plugins(tmp_path)
|
||||
hits = [r for r in caplog.records if "not-a-plugin" in r.message]
|
||||
assert len(hits) == 1, f"warned {len(hits)} times across 5 scans"
|
||||
|
||||
|
||||
def _plugin(tmp_path, name, body):
|
||||
d = tmp_path / name
|
||||
d.mkdir()
|
||||
(d / "manifest.json").write_text(json.dumps(body))
|
||||
return d
|
||||
|
||||
|
||||
VALID = {"name": "V", "version": "1.0.0", "class_name": "X", "display_modes": ["m"]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [None, [1, 2], "not an object", 42, True])
|
||||
def test_a_manifest_that_is_not_an_object_is_skipped_not_fatal(tmp_path, caplog, body):
|
||||
"""json.load accepts any JSON value, not just objects.
|
||||
|
||||
manifest.get('id') then raised AttributeError, which nothing here caught --
|
||||
the outer handler takes OSError/PermissionError only. A single malformed
|
||||
manifest aborted the entire scan, so every other plugin on disk, however
|
||||
healthy, silently failed to register.
|
||||
"""
|
||||
_plugin(tmp_path, "aaa-good", dict(VALID, id="aaa-good"))
|
||||
_plugin(tmp_path, "mmm-bad", body)
|
||||
_plugin(tmp_path, "zzz-good", dict(VALID, id="zzz-good"))
|
||||
|
||||
pm = _manager(tmp_path)
|
||||
with caplog.at_level(logging.WARNING, logger="test.discovery"):
|
||||
found = pm._scan_directory_for_plugins(tmp_path)
|
||||
|
||||
assert sorted(found) == ["aaa-good", "zzz-good"], (
|
||||
"one unusable manifest took the healthy plugins down with it")
|
||||
joined = " ".join(r.message for r in caplog.records)
|
||||
assert "mmm-bad" in joined, f"the skip was silent; log said: {joined!r}"
|
||||
|
||||
|
||||
def test_the_bad_manifest_is_named_with_what_it_actually_was(tmp_path, caplog):
|
||||
_plugin(tmp_path, "listy", [1, 2])
|
||||
pm = _manager(tmp_path)
|
||||
with caplog.at_level(logging.WARNING, logger="test.discovery"):
|
||||
pm._scan_directory_for_plugins(tmp_path)
|
||||
joined = " ".join(r.message for r in caplog.records)
|
||||
assert "listy" in joined and "list" in joined, (
|
||||
f"the warning does not say what the manifest was: {joined!r}")
|
||||
@@ -91,3 +91,101 @@ def test_force_reload_refreshes_stale_in_memory_snapshot():
|
||||
|
||||
# and it asked the cache to bypass the in-memory tier (memory_ttl=0).
|
||||
assert any(c.kwargs.get("memory_ttl") == 0 for c in cache.get.call_args_list)
|
||||
|
||||
|
||||
# --- persisted state that does not match the current schema -------------------
|
||||
#
|
||||
# A record on disk can be missing fields the callers index directly: a partial
|
||||
# write, a restored backup, or a state written by an older schema. Returning it
|
||||
# verbatim raises KeyError inside record_success / record_failure, which takes
|
||||
# the display down in a restart loop that survives reboots, because the bad
|
||||
# entry is on disk and gets read again on the way back up. Observed in the wild
|
||||
# as `plugin clock-simple operation failed: 'circuit_state'`, repeating ~50x a
|
||||
# minute with the panel frozen.
|
||||
|
||||
_INDEXED_FIELDS = (
|
||||
"consecutive_failures", "total_failures", "total_successes",
|
||||
"last_success_time", "last_failure_time", "circuit_state",
|
||||
"circuit_opened_time", "half_open_start_time", "last_error",
|
||||
)
|
||||
|
||||
|
||||
def _tracker_reading(persisted):
|
||||
cache = _cache()
|
||||
cache.get.return_value = persisted
|
||||
return PluginHealthTracker(cache)
|
||||
|
||||
|
||||
def test_partial_state_is_completed_not_returned_raw():
|
||||
"""The shape seen in the wild: one field, everything else absent."""
|
||||
state = _tracker_reading({"circuit_state": "closed"}).get_health_state("p")
|
||||
for field in _INDEXED_FIELDS:
|
||||
assert field in state, f"{field} missing; callers index it directly"
|
||||
|
||||
|
||||
def test_repair_keeps_real_failure_history():
|
||||
"""A record with genuine counts must not be reset to healthy just because
|
||||
an optional field is absent -- that would clear a tripped breaker."""
|
||||
state = _tracker_reading({
|
||||
"consecutive_failures": 5,
|
||||
"total_failures": 5,
|
||||
"circuit_state": "open",
|
||||
}).get_health_state("p")
|
||||
assert state["consecutive_failures"] == 5
|
||||
assert state["total_failures"] == 5
|
||||
assert state["circuit_state"] == "open"
|
||||
|
||||
|
||||
def test_wrong_types_fall_back_per_field():
|
||||
"""A counter persisted as a string would pass a membership check and then
|
||||
fail on the first += 1; an unknown circuit_state would take a branch the
|
||||
breaker has no handling for."""
|
||||
state = _tracker_reading({
|
||||
"consecutive_failures": "3",
|
||||
"circuit_state": "melted",
|
||||
"total_failures": 7,
|
||||
}).get_health_state("p")
|
||||
assert state["consecutive_failures"] == 0
|
||||
assert state["circuit_state"] == CircuitState.CLOSED.value
|
||||
assert state["total_failures"] == 7, "valid neighbours must survive"
|
||||
|
||||
|
||||
def test_newer_fields_are_carried_through():
|
||||
"""degraded/degraded_reason are read with .get() and are not part of the
|
||||
indexed set; repairing must not drop them."""
|
||||
state = _tracker_reading({
|
||||
"circuit_state": "closed", "degraded": True, "degraded_reason": "x",
|
||||
}).get_health_state("p")
|
||||
assert state["degraded"] is True
|
||||
assert state["degraded_reason"] == "x"
|
||||
|
||||
|
||||
def test_recording_against_a_repaired_state_does_not_raise():
|
||||
"""The actual failure: record_failure indexing a field that was not there.
|
||||
|
||||
The seed deliberately omits circuit_state. Seeding a record that *has* it
|
||||
would pass against the old raw-return behaviour too -- the counters are
|
||||
read with .get(), so circuit_state is the only field whose absence used to
|
||||
raise.
|
||||
"""
|
||||
tracker = _tracker_reading({"total_failures": 2, "total_successes": 1})
|
||||
tracker.record_failure("p", Exception("boom"))
|
||||
tracker.record_success("p")
|
||||
|
||||
|
||||
def test_unhashable_or_boolean_values_are_repaired():
|
||||
"""Values that break the repair itself rather than a later caller.
|
||||
|
||||
An unhashable circuit_state raises TypeError inside a set membership test,
|
||||
and bool is a subclass of int, so True would pass as a timestamp and then
|
||||
compare as 1.0 -- expiring a cooldown the moment it opens.
|
||||
"""
|
||||
for bad_state in ({"circuit_state": []}, {"circuit_state": {}}):
|
||||
state = _tracker_reading(bad_state).get_health_state("p")
|
||||
assert state["circuit_state"] == CircuitState.CLOSED.value
|
||||
|
||||
state = _tracker_reading({
|
||||
"circuit_opened_time": True, "last_success_time": False,
|
||||
}).get_health_state("p")
|
||||
assert state["circuit_opened_time"] is None
|
||||
assert state["last_success_time"] is None
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Tests for PluginManager.discovered_plugin_ids().
|
||||
|
||||
The config-watcher thread needs the set of discovered plugin ids while the
|
||||
render thread may be rebuilding plugin_manifests. Iterating that dict directly
|
||||
can observe a half-populated mapping or raise "dictionary changed size during
|
||||
iteration", so the accessor snapshots it under the discovery lock.
|
||||
"""
|
||||
|
||||
import tempfile
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from src.plugin_system.plugin_manager import PluginManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pm():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
yield PluginManager(plugins_dir=str(Path(tmp) / "plugins"))
|
||||
|
||||
|
||||
def test_returns_the_discovered_ids(pm):
|
||||
pm.plugin_manifests = {"clock-simple": {}, "hockey-scoreboard": {}}
|
||||
assert pm.discovered_plugin_ids() == {"clock-simple", "hockey-scoreboard"}
|
||||
|
||||
|
||||
def test_empty_when_nothing_discovered(pm):
|
||||
pm.plugin_manifests = {}
|
||||
assert pm.discovered_plugin_ids() == set()
|
||||
|
||||
|
||||
def test_is_a_snapshot_not_a_live_view(pm):
|
||||
"""The caller iterates the result on another thread; it must not alias
|
||||
the mapping discovery is still writing to."""
|
||||
pm.plugin_manifests = {"clock-simple": {}}
|
||||
snapshot = pm.discovered_plugin_ids()
|
||||
pm.plugin_manifests["hockey-scoreboard"] = {}
|
||||
assert snapshot == {"clock-simple"}
|
||||
|
||||
|
||||
def test_takes_the_discovery_lock(pm):
|
||||
"""Guards against the lock being dropped in a later refactor: with the
|
||||
lock held by another thread the call must block rather than read."""
|
||||
pm.plugin_manifests = {"clock-simple": {}}
|
||||
finished = threading.Event()
|
||||
|
||||
def call():
|
||||
pm.discovered_plugin_ids()
|
||||
finished.set()
|
||||
|
||||
pm._discovery_lock.acquire()
|
||||
try:
|
||||
# RLock is reentrant per-thread, so use a *different* thread to prove
|
||||
# the accessor actually waits on it.
|
||||
t = threading.Thread(target=call, daemon=True)
|
||||
t.start()
|
||||
assert not finished.wait(timeout=0.3), "accessor did not take the discovery lock"
|
||||
finally:
|
||||
pm._discovery_lock.release()
|
||||
t.join(timeout=2)
|
||||
assert finished.is_set()
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Plugin state history must not grow without bound.
|
||||
|
||||
`PluginStateManager` recorded every state transition in a per-plugin list and
|
||||
never trimmed it. The only code that removed entries was `clear_state()`, called
|
||||
solely from `PluginManager.unload_plugin()`, so a plugin that stays loaded --
|
||||
i.e. normal operation -- never released a single entry.
|
||||
|
||||
The list is written on the hot scheduling path. Every update cycle appends
|
||||
twice: `_reserve_for_update()` sets RUNNING and `_finish()` sets ENABLED back
|
||||
again. At the default 60-second update interval that is 2,880 entries per
|
||||
plugin per day, and nothing ever reads the entries -- `get_state_info()` only
|
||||
takes their `len()`. It is pure dead weight.
|
||||
|
||||
Measured against the unpatched class, ten plugins on a 60s interval retain
|
||||
864,010 transitions after thirty simulated days, for 231 MB of heap. On a 1 GB
|
||||
Pi that is fatal on its own, and the failure is not a clean OOM: once
|
||||
MemAvailable falls far enough, fork() starts returning ENOMEM, so sshd accepts
|
||||
connections and closes them before its banner while the kernel still answers
|
||||
pings. The board looks like a hardware fault and needs a power cycle.
|
||||
|
||||
These tests pin the cap, the retention order, and the one piece of behaviour the
|
||||
cap must not change: `state_history_count` is surfaced through the web API, so
|
||||
it has to keep reporting the lifetime total rather than plateauing at the cap.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from src.plugin_system.plugin_state import ( # noqa: E402
|
||||
MAX_STATE_HISTORY_PER_PLUGIN,
|
||||
PluginState,
|
||||
PluginStateManager,
|
||||
)
|
||||
|
||||
|
||||
def _cycle_updates(manager, plugin_id, cycles):
|
||||
"""Drive the real scheduling path: RUNNING on reserve, ENABLED on finish."""
|
||||
for _ in range(cycles):
|
||||
manager.set_state(plugin_id, PluginState.RUNNING)
|
||||
manager.set_state(plugin_id, PluginState.ENABLED)
|
||||
|
||||
|
||||
def test_state_history_is_capped():
|
||||
"""A day of updates must not retain a day of transitions."""
|
||||
manager = PluginStateManager()
|
||||
manager.set_state("clock", PluginState.ENABLED)
|
||||
|
||||
# One simulated day at the default 60s update interval.
|
||||
_cycle_updates(manager, "clock", 1440)
|
||||
|
||||
history = manager.get_state_history("clock")
|
||||
assert len(history) <= MAX_STATE_HISTORY_PER_PLUGIN, (
|
||||
f"history grew to {len(history)} entries; it is never trimmed"
|
||||
)
|
||||
|
||||
|
||||
def test_state_history_keeps_the_most_recent_transitions():
|
||||
"""Trimming drops the oldest entries, not the newest."""
|
||||
manager = PluginStateManager()
|
||||
manager.set_state("clock", PluginState.ENABLED)
|
||||
_cycle_updates(manager, "clock", MAX_STATE_HISTORY_PER_PLUGIN)
|
||||
|
||||
history = manager.get_state_history("clock")
|
||||
|
||||
# The scheduling cycle ends on ENABLED, so the newest entry is the
|
||||
# RUNNING -> ENABLED half of the last cycle.
|
||||
assert history[-1]["from"] == PluginState.RUNNING.value
|
||||
assert history[-1]["to"] == PluginState.ENABLED.value
|
||||
|
||||
# And the very first ENABLED transition has aged out.
|
||||
assert history[0]["from"] != PluginState.UNLOADED.value
|
||||
|
||||
|
||||
def test_state_history_count_reports_lifetime_total():
|
||||
"""The count exposed through the API must not plateau at the cap.
|
||||
|
||||
`get_state_info()['state_history_count']` is surfaced by the web UI. Capping
|
||||
the retained list must not turn it into "entries we happen to still hold".
|
||||
"""
|
||||
manager = PluginStateManager()
|
||||
manager.set_state("clock", PluginState.ENABLED)
|
||||
total = 1
|
||||
|
||||
cycles = MAX_STATE_HISTORY_PER_PLUGIN * 2
|
||||
_cycle_updates(manager, "clock", cycles)
|
||||
total += cycles * 2
|
||||
|
||||
info = manager.get_state_info("clock")
|
||||
assert info["state_history_count"] == total
|
||||
assert len(manager.get_state_history("clock")) <= MAX_STATE_HISTORY_PER_PLUGIN
|
||||
|
||||
|
||||
def test_error_transitions_are_capped_too():
|
||||
"""set_state_with_error() appends to the same list and needs the same cap."""
|
||||
manager = PluginStateManager()
|
||||
manager.set_state("clock", PluginState.ENABLED)
|
||||
|
||||
for _ in range(MAX_STATE_HISTORY_PER_PLUGIN * 2):
|
||||
manager.set_state_with_error(
|
||||
"clock",
|
||||
PluginState.ENABLED,
|
||||
{"reason": "update timeout"},
|
||||
error=RuntimeError("boom"),
|
||||
)
|
||||
|
||||
assert len(manager.get_state_history("clock")) <= MAX_STATE_HISTORY_PER_PLUGIN
|
||||
|
||||
|
||||
def test_history_is_isolated_per_plugin():
|
||||
"""The cap is per plugin, not shared across the manager."""
|
||||
manager = PluginStateManager()
|
||||
for plugin_id in ("clock", "weather"):
|
||||
manager.set_state(plugin_id, PluginState.ENABLED)
|
||||
_cycle_updates(manager, plugin_id, 50)
|
||||
|
||||
assert len(manager.get_state_history("clock")) == 101
|
||||
assert len(manager.get_state_history("weather")) == 101
|
||||
|
||||
|
||||
def test_get_state_history_returns_a_copy():
|
||||
"""Callers must not be able to mutate the manager's internal history."""
|
||||
manager = PluginStateManager()
|
||||
manager.set_state("clock", PluginState.ENABLED)
|
||||
|
||||
history = manager.get_state_history("clock")
|
||||
history.clear()
|
||||
|
||||
assert len(manager.get_state_history("clock")) == 1
|
||||
|
||||
|
||||
def test_get_state_history_entries_are_copies():
|
||||
"""Copying the outer list is not enough -- the entries are handed out too.
|
||||
|
||||
A caller holding a returned transition must not be able to rewrite the
|
||||
manager's record of what happened.
|
||||
"""
|
||||
manager = PluginStateManager()
|
||||
manager.set_state("clock", PluginState.ENABLED)
|
||||
|
||||
entry = manager.get_state_history("clock")[0]
|
||||
entry["to"] = "tampered"
|
||||
entry["error"] = "injected"
|
||||
|
||||
stored = manager.get_state_history("clock")[0]
|
||||
assert stored["to"] == PluginState.ENABLED.value
|
||||
assert stored["error"] is None
|
||||
|
||||
|
||||
def test_clear_state_drops_history():
|
||||
"""Unloading a plugin still releases everything it accumulated."""
|
||||
manager = PluginStateManager()
|
||||
manager.set_state("clock", PluginState.ENABLED)
|
||||
_cycle_updates(manager, "clock", 10)
|
||||
|
||||
manager.clear_state("clock")
|
||||
|
||||
assert manager.get_state_history("clock") == []
|
||||
assert manager.get_state_info("clock")["state_history_count"] == 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Retention is bounded by age first and by count second.
|
||||
|
||||
The cap added in the parent change is a flat entry count, and an entry count
|
||||
answers the wrong question. What a reader wants from this history is "the last
|
||||
couple of hours"; how many transitions that is depends entirely on the
|
||||
plugin's update interval, which on a real board spans 2s to 3600s. A flat 200
|
||||
entries is 4.2 days of history for the slowest plugin and 3.3 minutes for the
|
||||
fastest -- so the plugin churning hardest, the one actually worth looking at,
|
||||
keeps the least.
|
||||
|
||||
Trimming by age makes the retained window comparable whatever the cadence, and
|
||||
the count then serves only as a memory ceiling for pollers fast enough to
|
||||
produce thousands of transitions inside that window.
|
||||
"""
|
||||
|
||||
import time
|
||||
import pytest
|
||||
|
||||
from src.plugin_system.plugin_state import (
|
||||
PluginState,
|
||||
PluginStateManager,
|
||||
MAX_STATE_HISTORY_PER_PLUGIN,
|
||||
STATE_HISTORY_MAX_AGE_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
class FakeClock:
|
||||
"""A monotonic clock the test drives, so no test has to sleep."""
|
||||
|
||||
def __init__(self):
|
||||
self.t = 1000.0
|
||||
|
||||
def __call__(self):
|
||||
return self.t
|
||||
|
||||
def advance(self, seconds):
|
||||
self.t += seconds
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clock(monkeypatch):
|
||||
c = FakeClock()
|
||||
monkeypatch.setattr("src.plugin_system.plugin_state.time.monotonic", c)
|
||||
return c
|
||||
|
||||
|
||||
def _cycle(manager, plugin_id, clock, interval, cycles):
|
||||
"""One update cycle: RUNNING on reserve, ENABLED on finish."""
|
||||
for _ in range(cycles):
|
||||
manager.set_state(plugin_id, PluginState.RUNNING)
|
||||
manager.set_state(plugin_id, PluginState.ENABLED)
|
||||
clock.advance(interval)
|
||||
|
||||
|
||||
def test_transitions_older_than_the_window_are_dropped(clock):
|
||||
m = PluginStateManager()
|
||||
_cycle(m, "clock", clock, interval=60, cycles=10)
|
||||
assert len(m.get_state_history("clock")) == 20
|
||||
|
||||
# Nothing happens for longer than the window, then one more cycle.
|
||||
clock.advance(STATE_HISTORY_MAX_AGE_SECONDS + 1)
|
||||
_cycle(m, "clock", clock, interval=60, cycles=1)
|
||||
|
||||
assert len(m.get_state_history("clock")) == 2, (
|
||||
"only the transitions inside the window should survive")
|
||||
|
||||
|
||||
def test_every_plugin_keeps_the_same_WINDOW_not_the_same_COUNT(clock):
|
||||
"""The point of the age policy, stated as the property that distinguishes it.
|
||||
|
||||
Run both plugins for three times the retention window. Under a flat count
|
||||
cap the slow one would still be holding transitions from hours before the
|
||||
window, because it never produces enough entries to evict them. Under the
|
||||
age policy each plugin retains its own last two hours and no more --
|
||||
different entry counts, same span of time.
|
||||
"""
|
||||
window = STATE_HISTORY_MAX_AGE_SECONDS
|
||||
m = PluginStateManager()
|
||||
|
||||
_cycle(m, "slow", clock, interval=60, cycles=(3 * window) // 60)
|
||||
slow = len(m.get_state_history("slow"))
|
||||
|
||||
# Assert the property directly rather than a derived count. The guarantee
|
||||
# is about the SPAN of retained history, not its age against the current
|
||||
# clock: trimming happens on append, so a plugin that has gone quiet keeps
|
||||
# its last window until it writes again. That is intentional -- it is
|
||||
# bounded either way, and a lazy trim costs nothing on the hot path.
|
||||
stamps = [stamp for stamp, _ in m._state_history["slow"]]
|
||||
assert stamps[-1] - stamps[0] <= window, (
|
||||
f"retained history spans {stamps[-1] - stamps[0]:.0f}s, "
|
||||
f"window is {window}s")
|
||||
assert slow < 2 * ((3 * window) // 60), (
|
||||
f"slow plugin kept {slow} entries -- three windows' worth was retained")
|
||||
|
||||
clock.t = 1000.0
|
||||
_cycle(m, "fast", clock, interval=2, cycles=(3 * window) // 2)
|
||||
fast = len(m.get_state_history("fast"))
|
||||
|
||||
# Different counts, and the fast poller keeps more of them -- under a flat
|
||||
# count cap these would be equal and the fast one would cover minutes.
|
||||
assert fast > slow, f"fast={fast} slow={slow}"
|
||||
|
||||
|
||||
def test_the_count_ceiling_still_bounds_a_fast_poller(clock):
|
||||
"""Age alone would let a 2s plugin hold 7,200 entries."""
|
||||
m = PluginStateManager()
|
||||
_cycle(m, "flights", clock, interval=2, cycles=STATE_HISTORY_MAX_AGE_SECONDS)
|
||||
assert len(m.get_state_history("flights")) <= MAX_STATE_HISTORY_PER_PLUGIN
|
||||
|
||||
|
||||
def test_a_burst_inside_the_window_is_capped_not_kept(clock):
|
||||
"""Transitions with no time between them still cannot grow without bound."""
|
||||
m = PluginStateManager()
|
||||
for _ in range(MAX_STATE_HISTORY_PER_PLUGIN * 3):
|
||||
m.set_state("flapping", PluginState.RUNNING) # clock never advances
|
||||
assert len(m.get_state_history("flapping")) <= MAX_STATE_HISTORY_PER_PLUGIN
|
||||
|
||||
|
||||
def test_ageing_out_does_not_disturb_the_lifetime_count(clock):
|
||||
m = PluginStateManager()
|
||||
_cycle(m, "clock", clock, interval=60, cycles=10)
|
||||
clock.advance(STATE_HISTORY_MAX_AGE_SECONDS + 1)
|
||||
_cycle(m, "clock", clock, interval=60, cycles=1)
|
||||
|
||||
assert len(m.get_state_history("clock")) == 2
|
||||
assert m.get_state_info("clock")["state_history_count"] == 22, (
|
||||
"the lifetime total must survive trimming, it is the flap signal")
|
||||
|
||||
|
||||
def test_the_surviving_entries_are_the_recent_ones(clock):
|
||||
m = PluginStateManager()
|
||||
_cycle(m, "clock", clock, interval=60, cycles=5)
|
||||
clock.advance(STATE_HISTORY_MAX_AGE_SECONDS + 1)
|
||||
m.set_state("clock", PluginState.ERROR)
|
||||
|
||||
history = m.get_state_history("clock")
|
||||
assert [h["to"] for h in history] == ["error"]
|
||||
|
||||
|
||||
def test_a_monotonic_clock_is_used_not_the_wall_clock(clock):
|
||||
"""A DST shift or NTP step must not flush the history.
|
||||
|
||||
The trim reads time.monotonic(); the human-readable datetime inside each
|
||||
transition is for display only.
|
||||
"""
|
||||
m = PluginStateManager()
|
||||
_cycle(m, "clock", clock, interval=60, cycles=3)
|
||||
before = len(m.get_state_history("clock"))
|
||||
|
||||
import datetime as real_datetime
|
||||
|
||||
class ShiftedDatetime(real_datetime.datetime):
|
||||
@classmethod
|
||||
def now(cls, tz=None):
|
||||
return real_datetime.datetime(1999, 1, 1) # clock jumps backwards
|
||||
|
||||
import src.plugin_system.plugin_state as ps
|
||||
original = ps.datetime
|
||||
ps.datetime = ShiftedDatetime
|
||||
try:
|
||||
m.set_state("clock", PluginState.ENABLED)
|
||||
finally:
|
||||
ps.datetime = original
|
||||
|
||||
assert len(m.get_state_history("clock")) == before + 1, (
|
||||
"a wall-clock jump must not trim anything")
|
||||
|
||||
|
||||
def test_get_state_info_is_a_consistent_snapshot():
|
||||
"""An unload running concurrently must not be observed half-done.
|
||||
|
||||
Each field used to be read under its own lock, so clear_state() could
|
||||
interleave: 'state' read before the removal, 'state_history_count' after,
|
||||
handing a caller a plugin that is ENABLED with zero transitions. The whole
|
||||
payload is now built in one critical section.
|
||||
"""
|
||||
import threading
|
||||
|
||||
m = PluginStateManager()
|
||||
for _ in range(50):
|
||||
m.set_state("clock", PluginState.RUNNING)
|
||||
m.set_state("clock", PluginState.ENABLED)
|
||||
|
||||
inconsistent = []
|
||||
stop = threading.Event()
|
||||
|
||||
def reader():
|
||||
while not stop.is_set():
|
||||
info = m.get_state_info("clock")
|
||||
# Either fully present or fully cleared -- never a live state with
|
||||
# a wiped count.
|
||||
if info["state"] != PluginState.UNLOADED.value and \
|
||||
info["state_history_count"] == 0:
|
||||
inconsistent.append(info)
|
||||
return
|
||||
|
||||
def clearer():
|
||||
for _ in range(200):
|
||||
for _ in range(20):
|
||||
m.set_state("clock", PluginState.ENABLED)
|
||||
m.clear_state("clock")
|
||||
|
||||
t = threading.Thread(target=reader, daemon=True)
|
||||
t.start()
|
||||
clearer()
|
||||
stop.set()
|
||||
t.join(timeout=5)
|
||||
|
||||
assert not inconsistent, f"observed a torn snapshot: {inconsistent[:1]}"
|
||||
@@ -11,7 +11,7 @@ Focus areas:
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from src.plugin_system.resource_monitor import (
|
||||
PluginResourceMonitor,
|
||||
@@ -127,3 +127,87 @@ class TestForceReload:
|
||||
fresh = mon.get_metrics_summary("p", force_reload=True)
|
||||
assert fresh["call_count"] == 7
|
||||
assert any(c.kwargs.get("memory_ttl") == 0 for c in cache.get.call_args_list)
|
||||
|
||||
|
||||
class TestMetricsPersistenceChurn:
|
||||
"""Metrics are telemetry; writing them on every call wore the SD card.
|
||||
|
||||
Each write is a ~350-byte file, which on ext4 costs a 4KB block plus a
|
||||
journal entry. At roughly nine calls a minute per plugin across fourteen
|
||||
plugins it dominated the device's write volume.
|
||||
"""
|
||||
|
||||
def test_the_first_snapshot_is_written_even_seconds_after_boot(self):
|
||||
"""The throttle must key off "have we written?", not process uptime.
|
||||
|
||||
time.monotonic() is time since boot on Linux, and systemd starts this
|
||||
service at boot. With 0.0 as the missing-timestamp default,
|
||||
`now - 0.0 < 30` was true for the first half-minute of every run, so
|
||||
the very first metrics write -- the one that matters most after a
|
||||
restart -- was silently skipped.
|
||||
"""
|
||||
import src.plugin_system.resource_monitor as rm
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
# 12 seconds after boot: inside the interval, but nothing written yet.
|
||||
with patch.object(rm.time, "monotonic", return_value=12.0):
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if "plugin_metrics:" in str(c)]
|
||||
assert writes, \
|
||||
"the first snapshot was dropped because the process was young"
|
||||
|
||||
def test_repeated_calls_persist_once_per_interval(self):
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
for _ in range(50):
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 1, (
|
||||
f"50 calls produced {len(writes)} metric writes; expected 1")
|
||||
|
||||
def test_the_interval_elapsing_allows_the_next_write(self, monkeypatch):
|
||||
import src.plugin_system.resource_monitor as rm
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
mon.monitor_call("p", lambda: None)
|
||||
# pretend the interval has passed
|
||||
mon._metrics_persisted_at["p"] -= rm._METRICS_PERSIST_INTERVAL + 1
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 2
|
||||
|
||||
def test_in_memory_metrics_stay_exact_while_writes_are_skipped(self):
|
||||
mon = PluginResourceMonitor(_cache(), enable_monitoring=False)
|
||||
for _ in range(20):
|
||||
mon.monitor_call("p", lambda: None)
|
||||
assert mon.get_metrics("p").call_count == 20
|
||||
|
||||
def test_reset_lets_the_next_call_persist_immediately(self):
|
||||
cache = _cache()
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
mon.monitor_call("p", lambda: None)
|
||||
mon.reset_metrics("p")
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 2, "reset should clear the throttle timestamp"
|
||||
|
||||
def test_a_failed_write_does_not_buy_the_next_interval_of_silence(self):
|
||||
"""A set() that raises must not count as having persisted.
|
||||
|
||||
Marking the timestamp before the write would leave no snapshot in the
|
||||
cache and still suppress the next 30 seconds of attempts.
|
||||
"""
|
||||
cache = _cache()
|
||||
cache.set.side_effect = [OSError("disk full"), None]
|
||||
mon = PluginResourceMonitor(cache, enable_monitoring=False)
|
||||
with pytest.raises(OSError):
|
||||
mon.monitor_call("p", lambda: None)
|
||||
# the very next call must try again rather than skip the interval
|
||||
mon.monitor_call("p", lambda: None)
|
||||
writes = [c for c in cache.set.call_args_list
|
||||
if c.args and str(c.args[0]).startswith("plugin_metrics:")]
|
||||
assert len(writes) == 2, "a failed write should be retried, not skipped"
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
Tests for the device-location default: a plugin that ships a location field in
|
||||
its schema must default to the device's configured City/State/Country, not to
|
||||
whatever place the plugin author hard-coded.
|
||||
|
||||
The bug this pins: ledmatrix-weather ships ``"location_city": "Dallas"`` as a
|
||||
schema default, so a user who set Kansas City under General settings but never
|
||||
opened the weather plugin's own config form got Dallas weather — and a radar
|
||||
centred on Dallas — with nothing in config.json to explain it.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from src.plugin_system.schema_manager import SchemaManager
|
||||
|
||||
|
||||
class FakeConfigManager:
|
||||
"""Minimal stand-in exposing the load_config() SchemaManager relies on."""
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.load_count = 0
|
||||
|
||||
def load_config(self):
|
||||
self.load_count += 1
|
||||
return self.config
|
||||
|
||||
|
||||
class ExplodingConfigManager:
|
||||
def load_config(self):
|
||||
raise OSError("config.json is unreadable")
|
||||
|
||||
|
||||
WEATHER_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location_city": {"type": "string", "default": "Dallas"},
|
||||
"location_state": {"type": "string", "default": "Texas"},
|
||||
"location_country": {"type": "string", "default": "US"},
|
||||
"units": {"type": "string", "default": "imperial"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_plugin(plugins_dir, plugin_id, schema):
|
||||
plugin_dir = plugins_dir / plugin_id
|
||||
plugin_dir.mkdir(parents=True, exist_ok=True)
|
||||
(plugin_dir / "config_schema.json").write_text(json.dumps(schema))
|
||||
return plugin_dir
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plugins_dir(tmp_path):
|
||||
d = tmp_path / "plugin-repos"
|
||||
d.mkdir()
|
||||
return d
|
||||
|
||||
|
||||
def make_sm(plugins_dir, tmp_path, location):
|
||||
config = {} if location is None else {"location": location}
|
||||
cm = FakeConfigManager(config)
|
||||
sm = SchemaManager(plugins_dir=plugins_dir, project_root=tmp_path,
|
||||
config_manager=cm)
|
||||
return sm, cm
|
||||
|
||||
|
||||
class TestDeviceLocationDefaults:
|
||||
def test_device_location_replaces_plugin_default(self, plugins_dir, tmp_path):
|
||||
write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA)
|
||||
sm, _ = make_sm(plugins_dir, tmp_path,
|
||||
{"city": "Kansas City", "state": "Missouri", "country": "US"})
|
||||
|
||||
defaults = sm.generate_default_config("ledmatrix-weather")
|
||||
|
||||
assert defaults["location_city"] == "Kansas City"
|
||||
assert defaults["location_state"] == "Missouri"
|
||||
assert defaults["location_country"] == "US"
|
||||
# Non-location defaults are untouched.
|
||||
assert defaults["units"] == "imperial"
|
||||
|
||||
def test_user_set_plugin_value_still_wins(self, plugins_dir, tmp_path):
|
||||
write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA)
|
||||
sm, _ = make_sm(plugins_dir, tmp_path,
|
||||
{"city": "Kansas City", "state": "Missouri", "country": "US"})
|
||||
|
||||
defaults = sm.generate_default_config("ledmatrix-weather")
|
||||
merged = sm.merge_with_defaults({"location_city": "Denver"}, defaults)
|
||||
|
||||
assert merged["location_city"] == "Denver"
|
||||
# Fields the user did not override still follow the device.
|
||||
assert merged["location_state"] == "Missouri"
|
||||
|
||||
def test_blank_and_missing_device_fields_leave_schema_default(self, plugins_dir, tmp_path):
|
||||
write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA)
|
||||
sm, _ = make_sm(plugins_dir, tmp_path, {"city": "Kansas City", "state": " "})
|
||||
|
||||
defaults = sm.generate_default_config("ledmatrix-weather")
|
||||
|
||||
assert defaults["location_city"] == "Kansas City"
|
||||
assert defaults["location_state"] == "Texas" # blank -> not configured
|
||||
assert defaults["location_country"] == "US" # absent -> schema default
|
||||
|
||||
def test_no_device_location_configured_is_a_no_op(self, plugins_dir, tmp_path):
|
||||
write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA)
|
||||
sm, _ = make_sm(plugins_dir, tmp_path, None)
|
||||
|
||||
defaults = sm.generate_default_config("ledmatrix-weather")
|
||||
|
||||
assert defaults["location_city"] == "Dallas"
|
||||
|
||||
def test_no_config_manager_is_a_no_op(self, plugins_dir, tmp_path):
|
||||
write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA)
|
||||
sm = SchemaManager(plugins_dir=plugins_dir, project_root=tmp_path)
|
||||
|
||||
assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Dallas"
|
||||
|
||||
def test_unreadable_config_falls_back_to_schema_defaults(self, plugins_dir, tmp_path):
|
||||
write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA)
|
||||
sm = SchemaManager(plugins_dir=plugins_dir, project_root=tmp_path,
|
||||
config_manager=ExplodingConfigManager())
|
||||
|
||||
assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Dallas"
|
||||
|
||||
|
||||
class TestScopedToNamespacedKeys:
|
||||
def test_bare_state_key_is_not_rewritten(self, plugins_dir, tmp_path):
|
||||
"""ledmatrix-elections' ``state`` is a two-letter code, not a place name."""
|
||||
write_plugin(plugins_dir, "ledmatrix-elections", {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"state": {"type": "string", "default": "CA"},
|
||||
"city": {"type": "string", "default": "Springfield"},
|
||||
},
|
||||
})
|
||||
sm, _ = make_sm(plugins_dir, tmp_path,
|
||||
{"city": "Kansas City", "state": "Missouri", "country": "US"})
|
||||
|
||||
defaults = sm.generate_default_config("ledmatrix-elections")
|
||||
|
||||
assert defaults["state"] == "CA"
|
||||
assert defaults["city"] == "Springfield"
|
||||
|
||||
def test_plugin_without_location_fields_never_reads_config(self, plugins_dir, tmp_path):
|
||||
write_plugin(plugins_dir, "clock-simple", {
|
||||
"type": "object",
|
||||
"properties": {"format": {"type": "string", "default": "12h"}},
|
||||
})
|
||||
sm, cm = make_sm(plugins_dir, tmp_path, {"city": "Kansas City"})
|
||||
|
||||
defaults = sm.generate_default_config("clock-simple")
|
||||
|
||||
assert defaults["format"] == "12h"
|
||||
assert cm.load_count == 0
|
||||
|
||||
|
||||
class TestCachingStaysFresh:
|
||||
def test_location_change_is_picked_up_through_the_defaults_cache(self, plugins_dir, tmp_path):
|
||||
write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA)
|
||||
sm, cm = make_sm(plugins_dir, tmp_path, {"city": "Kansas City"})
|
||||
|
||||
assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Kansas City"
|
||||
|
||||
cm.config["location"]["city"] = "Omaha"
|
||||
|
||||
# Second call is served from the defaults cache, but must not serve a
|
||||
# stale location.
|
||||
assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Omaha"
|
||||
|
||||
def test_cached_defaults_are_not_mutated_by_the_overlay(self, plugins_dir, tmp_path):
|
||||
write_plugin(plugins_dir, "ledmatrix-weather", WEATHER_SCHEMA)
|
||||
sm, cm = make_sm(plugins_dir, tmp_path, {"city": "Kansas City"})
|
||||
|
||||
sm.generate_default_config("ledmatrix-weather")
|
||||
assert sm._defaults_cache["ledmatrix-weather"]["location_city"] == "Dallas"
|
||||
|
||||
cm.config.pop("location")
|
||||
assert sm.generate_default_config("ledmatrix-weather")["location_city"] == "Dallas"
|
||||
@@ -0,0 +1,195 @@
|
||||
"""The card helpers the eight scoreboards now share.
|
||||
|
||||
These bodies lived in eight byte-identical copies. Moving them here means one
|
||||
fix reaches every scoreboard — and that a mistake does too, which is what this
|
||||
file guards. Each case below is one the plugins' own code already handled; the
|
||||
point is that it keeps handling it.
|
||||
|
||||
The functions take ``config``/``logger``/``fonts`` as arguments rather than
|
||||
reading them off an instance, so a plugin keeps its method and delegates the
|
||||
body. That is what let all eight adopt this with byte-identical renders.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from src.common import sports_card as C
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def log():
|
||||
return logging.getLogger("test_sports_card")
|
||||
|
||||
|
||||
class TestSettingsLookup:
|
||||
def test_reads_the_scroll_card_block(self):
|
||||
cfg = {"scroll_card": {"vs_text": "@"}}
|
||||
assert C.scroll_card_option(cfg, "vs_text", "VS") == "@"
|
||||
|
||||
@pytest.mark.parametrize("cfg", [None, {}, {"scroll_card": None},
|
||||
{"scroll_card": {"vs_text": None}}])
|
||||
def test_missing_or_null_falls_back(self, cfg):
|
||||
"""A null in config means "unset", not "empty string"."""
|
||||
assert C.scroll_card_option(cfg, "vs_text", "VS") == "VS"
|
||||
|
||||
def test_upcoming_center_rejects_unknown_modes(self):
|
||||
for bad in ("sideways", "", None, 7):
|
||||
assert C.upcoming_center_mode({"scroll_card": {"upcoming_center": bad}}) == "vs"
|
||||
assert C.upcoming_center_mode({"scroll_card": {"upcoming_center": "DATE_TIME"}}) == "date_time"
|
||||
|
||||
|
||||
class TestColour:
|
||||
def test_rgb_list_and_hex_both_work(self):
|
||||
assert C.element_color({"customization": {"score_text": {"text_color": [1, 2, 3]}}},
|
||||
"score_text") == (1, 2, 3)
|
||||
assert C.element_color({"customization": {"score_text": {"text_color": "#ff8000"}}},
|
||||
"score_text") == (255, 128, 0)
|
||||
|
||||
@pytest.mark.parametrize("value", ["nope", "#fff", [1, 2], None, ["a", "b", "c"]])
|
||||
def test_unusable_colour_falls_back(self, value):
|
||||
cfg = {"customization": {"score_text": {"text_color": value}}}
|
||||
assert C.element_color(cfg, "score_text", (9, 9, 9)) == (9, 9, 9)
|
||||
|
||||
def test_coerce_rgb_clamps_rather_than_rejecting(self):
|
||||
assert C.coerce_rgb([300, -5, 20], (0, 0, 0)) == (255, 0, 20)
|
||||
|
||||
def test_coerce_rgb_refuses_a_three_character_string(self):
|
||||
""""123" would otherwise iterate into three digits and yield a colour."""
|
||||
assert C.coerce_rgb("123", (7, 7, 7)) == (7, 7, 7)
|
||||
|
||||
def test_font_colour_is_resolved_by_identity(self):
|
||||
a, b = object(), object()
|
||||
cfg = {"customization": {"score_text": {"text_color": [4, 5, 6]}}}
|
||||
assert C.font_color(cfg, {"score": a, "team": b}, a) == (4, 5, 6)
|
||||
|
||||
def test_a_shared_face_gives_up_rather_than_guessing(self):
|
||||
"""One object used for two elements has no single right colour."""
|
||||
shared = object()
|
||||
cfg = {"customization": {"score_text": {"text_color": [4, 5, 6]}}}
|
||||
assert C.font_color(cfg, {"score": shared, "team": shared}, shared) == (255, 255, 255)
|
||||
|
||||
|
||||
class TestFavourites:
|
||||
GAME = {"home_abbr": "TB", "away_abbr": "NO", "home_score": "21", "away_score": "17"}
|
||||
|
||||
def test_win_loss_and_tie(self):
|
||||
cfg = {"favorite_teams": ["TB"]}
|
||||
assert C.favorite_result(cfg, self.GAME) == "win"
|
||||
assert C.favorite_result({"favorite_teams": ["NO"]}, self.GAME) == "loss"
|
||||
tied = dict(self.GAME, home_score="3", away_score="3")
|
||||
assert C.favorite_result(cfg, tied) == "tie"
|
||||
|
||||
def test_no_verdict_without_exactly_one_favourite_side(self):
|
||||
assert C.favorite_result({}, self.GAME) is None
|
||||
assert C.favorite_result({"favorite_teams": ["TB", "NO"]}, self.GAME) is None
|
||||
assert C.favorite_result({"favorite_teams": ["SEA"]}, self.GAME) is None
|
||||
|
||||
def test_unusable_scores_give_no_verdict(self):
|
||||
bad = dict(self.GAME, home_score="x")
|
||||
assert C.favorite_result({"favorite_teams": ["TB"]}, bad) is None
|
||||
|
||||
def test_nested_payload_shape_is_read_too(self):
|
||||
game = {"home_team": {"abbrev": "TB", "score": 9},
|
||||
"away_team": {"abbrev": "NO", "score": 2}}
|
||||
assert C.side_score(game, "home") == 9
|
||||
assert C.side_is_favorite(game, "home", {"TB"}) is True
|
||||
|
||||
def test_matches_on_id_where_abbreviations_collide(self):
|
||||
"""NRL keys favourites by ESPN id; abbreviations are not unique there."""
|
||||
game = {"home_abbr": "SYD", "home_id": "4321"}
|
||||
assert C.side_is_favorite(game, "home", {"4321"}) is True
|
||||
|
||||
def test_game_and_config_favourites_are_both_used(self):
|
||||
"""Games carry resolved dynamic groups; config catches later edits."""
|
||||
game = dict(self.GAME, favorite_teams=["NO"], league="nfl")
|
||||
assert set(C.favorite_teams_for({"nfl": {"favorite_teams": ["TB"]}}, game)) == {"NO", "TB"}
|
||||
|
||||
|
||||
class TestDateAndTime:
|
||||
def test_date_formats(self, log):
|
||||
for fmt, want in [("abbrev", "Sep 5"), ("numeric", "9/5"),
|
||||
("day_first", "5 Sep"), ("numeric_day_first", "5/9")]:
|
||||
cfg = {"scroll_card": {"date_format": fmt}}
|
||||
assert C.format_game_date(cfg, log, "9/5") == want
|
||||
|
||||
@pytest.mark.parametrize("raw", ["", "garbage", "13/40", "no/slash/here"])
|
||||
def test_unparseable_dates_pass_through(self, log, raw):
|
||||
assert C.format_game_date({}, log, raw) == raw.strip()
|
||||
|
||||
def test_24h_conversion(self):
|
||||
cfg = {"scroll_card": {"time_format": "24h"}}
|
||||
assert C.format_game_time(cfg, "7:30 PM") == "19:30"
|
||||
assert C.format_game_time(cfg, "12:00 AM") == "00:00"
|
||||
assert C.format_game_time(cfg, "12:15 PM") == "12:15"
|
||||
|
||||
def test_12h_is_left_alone_and_junk_survives(self):
|
||||
assert C.format_game_time({}, "7:30 PM") == "7:30 PM"
|
||||
assert C.format_game_time({"scroll_card": {"time_format": "24h"}}, "soon") == "soon"
|
||||
|
||||
def test_a_bad_timezone_falls_back_to_utc(self, log):
|
||||
"""A typo in config should not blank the card."""
|
||||
from datetime import timezone
|
||||
assert C.card_tzinfo({"timezone": "Not/AZone"}, log) is timezone.utc
|
||||
|
||||
|
||||
class TestFontSizing:
|
||||
def test_snaps_to_the_faces_pixel_grid(self):
|
||||
assert C.crisp_size("4x6-font.ttf", 6) == 7 # 7px grid
|
||||
assert C.crisp_size("PressStart2P-Regular.ttf", 10) == 8
|
||||
assert C.crisp_size("PressStart2P-Regular.ttf", 13) == 16
|
||||
|
||||
def test_an_unknown_face_is_never_second_guessed(self):
|
||||
assert C.crisp_size("SomeUserFont.ttf", 11) == 11
|
||||
|
||||
def test_aliases_resolve_before_the_grid_lookup(self):
|
||||
assert C.crisp_size("four_by_six", 6) == C.crisp_size("4x6-font.ttf", 6)
|
||||
|
||||
@pytest.mark.parametrize("desired", [0, -3, None])
|
||||
def test_unusable_sizes_pass_through_without_raising(self, desired):
|
||||
"""None reached this in the field; football's variant raised TypeError."""
|
||||
assert C.crisp_size("4x6-font.ttf", desired) == desired
|
||||
|
||||
def test_schema_cache_is_keyed_per_schema_not_globally(self, tmp_path):
|
||||
"""Two plugins declaring different defaults must not share an answer."""
|
||||
a, b = tmp_path / "a.json", tmp_path / "b.json"
|
||||
for path, size in ((a, 11), (b, 22)):
|
||||
path.write_text(json.dumps({"properties": {"customization": {"properties": {
|
||||
"score_text": {"properties": {"font_size": {"default": size}}}}}}}))
|
||||
assert C.schema_font_size(str(a), "score_text") == 11
|
||||
assert C.schema_font_size(str(b), "score_text") == 22
|
||||
|
||||
def test_a_missing_schema_is_not_an_error(self, tmp_path):
|
||||
assert C.schema_font_size(str(tmp_path / "nope.json"), "score_text") is None
|
||||
|
||||
def test_a_configured_size_matching_the_schema_default_is_not_a_choice(self, tmp_path):
|
||||
"""The web UI writes the whole default block on every save, so
|
||||
font_size == schema default carries no intent and must not pin the
|
||||
install to an off-grid size forever."""
|
||||
schema = tmp_path / "s.json"
|
||||
schema.write_text(json.dumps({"properties": {"customization": {"properties": {
|
||||
"score_text": {"properties": {"font_size": {"default": 10}}}}}}}))
|
||||
got = C.resolve_font_size(str(schema), {"font_size": 10}, "score_text", 10,
|
||||
"PressStart2P-Regular.ttf")
|
||||
assert got == 8, "a default-valued size should snap to the grid"
|
||||
|
||||
def test_a_real_choice_wins(self, tmp_path):
|
||||
schema = tmp_path / "s.json"
|
||||
schema.write_text(json.dumps({"properties": {"customization": {"properties": {
|
||||
"score_text": {"properties": {"font_size": {"default": 10}}}}}}}))
|
||||
got = C.resolve_font_size(str(schema), {"font_size": 13}, "score_text", 10,
|
||||
"PressStart2P-Regular.ttf")
|
||||
assert got == 13, "an explicit size the user chose is not second-guessed"
|
||||
|
||||
|
||||
class TestTables:
|
||||
def test_every_font_key_maps_to_an_element(self):
|
||||
assert set(C.ELEMENT_FOR_FONT) == {"score", "time", "team", "status", "detail", "rank"}
|
||||
|
||||
def test_result_colours_cover_every_verdict(self):
|
||||
assert set(C.FAVORITE_RESULT_COLOR_DEFAULTS) == {"win", "loss", "tie"}
|
||||
|
||||
def test_month_and_weekday_tables_are_complete(self):
|
||||
assert len(C.MONTH_ABBR) == 12 and len(C.WEEKDAY_ABBR) == 7
|
||||
@@ -0,0 +1,258 @@
|
||||
"""The shared card geometry, exercised against a host that supplies only what
|
||||
the mixin's contract names.
|
||||
|
||||
The point of these is the contract, not the arithmetic. The mixin reaches for
|
||||
``display_width``, ``fonts``, ``config`` and six ``sports_card`` delegations
|
||||
through ``self``, and the eight plugins are what actually provide them. A
|
||||
stub host that provides exactly the documented surface and nothing else is
|
||||
what catches the mixin quietly growing a dependency the plugins do not have.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
from src.common.sports_game_renderer import SportsGameRendererMixin
|
||||
|
||||
|
||||
class Host(SportsGameRendererMixin):
|
||||
"""The documented contract, and not one attribute more."""
|
||||
|
||||
def __init__(self, width=128, height=32, config=None, scroll=None):
|
||||
self.display_width = width
|
||||
self.display_height = height
|
||||
self.config = config or {}
|
||||
self.logger = _Logger()
|
||||
self._team_rankings_cache = {}
|
||||
self._scroll = scroll or {}
|
||||
font = ImageFont.load_default()
|
||||
self.fonts = {'score': font, 'time': font, 'detail': font}
|
||||
self.drawn = []
|
||||
|
||||
# -- the six sports_card delegations the mixin calls --
|
||||
def _scroll_card_option(self, key, default=None):
|
||||
return self._scroll.get(key, default)
|
||||
|
||||
def _upcoming_center_mode(self):
|
||||
return self._scroll.get('upcoming_center', 'vs')
|
||||
|
||||
def _vs_text(self):
|
||||
return self._scroll.get('vs_text', 'VS')
|
||||
|
||||
def _element_color(self, element):
|
||||
return (255, 255, 255)
|
||||
|
||||
def _format_game_date(self, raw, game):
|
||||
return raw
|
||||
|
||||
def _format_game_time(self, raw):
|
||||
return raw
|
||||
|
||||
# -- the one hook whose body genuinely varies per plugin --
|
||||
def _draw_text_with_outline(self, draw, text, position, font,
|
||||
fill=None, outline_color=(0, 0, 0)):
|
||||
self.drawn.append((text, position))
|
||||
|
||||
|
||||
class _Logger:
|
||||
def debug(self, *a, **k):
|
||||
pass
|
||||
|
||||
|
||||
def _draw():
|
||||
return ImageDraw.Draw(Image.new("RGB", (256, 64)))
|
||||
|
||||
|
||||
class TestCenterGap:
|
||||
def test_explicit_center_gap_wins_outright(self):
|
||||
assert Host(scroll={'center_gap': 31})._center_gap_width() == 31
|
||||
|
||||
def test_explicit_zero_restores_edge_to_edge_logos(self):
|
||||
# 0 is a real setting, not a falsy miss -- the guard is `>= 0`.
|
||||
assert Host(scroll={'center_gap': 0})._center_gap_width() == 0
|
||||
|
||||
def test_otherwise_it_scales_with_card_width_within_the_clamp(self):
|
||||
h = Host(width=512)
|
||||
# 512 * 0.28 = 143, clamped to the 40px ceiling.
|
||||
assert h._center_gap_width() >= h.CENTER_GAP_MIN_PX
|
||||
|
||||
def test_the_gap_never_ends_up_narrower_than_the_score(self):
|
||||
# This is the bug the measurement exists to prevent: a derived gap
|
||||
# smaller than the rendered score drew the score over the logos.
|
||||
h = Host(width=64)
|
||||
assert h._center_gap_width() >= h._score_reserve_width()
|
||||
|
||||
def test_a_junk_ratio_falls_back_to_the_floor(self):
|
||||
h = Host(scroll={'center_gap_ratio': 'wide'})
|
||||
assert h._center_gap_width() == h.CENTER_GAP_MIN_PX
|
||||
|
||||
|
||||
class TestNonFiniteSettings:
|
||||
"""inf reaches int() and raises OverflowError, which the old
|
||||
`except (TypeError, ValueError)` did not catch -- so one bad config value
|
||||
aborted the entire card render rather than falling back."""
|
||||
|
||||
@pytest.mark.parametrize("bad", [float("inf"), float("-inf")])
|
||||
def test_a_non_finite_center_gap_falls_back(self, bad):
|
||||
h = Host(scroll={'center_gap': bad})
|
||||
assert h._center_gap_width() >= h.CENTER_GAP_MIN_PX
|
||||
|
||||
@pytest.mark.parametrize("bad", [float("inf"), float("-inf"), float("nan")])
|
||||
def test_a_non_finite_ratio_falls_back_to_the_floor(self, bad):
|
||||
h = Host(scroll={'center_gap_ratio': bad})
|
||||
assert h._center_gap_width() == h.CENTER_GAP_MIN_PX
|
||||
|
||||
@pytest.mark.parametrize("bad", [float("inf"), float("-inf")])
|
||||
def test_non_finite_clamp_bounds_fall_back(self, bad):
|
||||
h = Host(scroll={'center_gap_min': bad, 'center_gap_max': bad})
|
||||
assert h._center_gap_width() == h.CENTER_GAP_MIN_PX
|
||||
|
||||
@pytest.mark.parametrize("bad", [float("inf"), float("-inf"), "inf", "-inf", "nan"])
|
||||
def test_a_non_finite_layout_offset_gives_the_default(self, bad):
|
||||
cfg = {'customization': {'layout': {'score': {'x_offset': bad}}}}
|
||||
assert Host(config=cfg)._layout_offset('score', 'x_offset', 7) == 7
|
||||
|
||||
def test_a_finite_value_is_still_honoured(self):
|
||||
# The guard must not swallow ordinary settings.
|
||||
assert Host(scroll={'center_gap': 31})._center_gap_width() == 31
|
||||
cfg = {'customization': {'layout': {'score': {'x_offset': -3}}}}
|
||||
assert Host(config=cfg)._layout_offset('score', 'x_offset', 7) == -3
|
||||
|
||||
|
||||
class TestScoreReserve:
|
||||
def test_it_measures_the_probe_plus_both_gutters(self):
|
||||
h = Host()
|
||||
assert h._score_reserve_width() > 2 * h._SCORE_LOGO_GUTTER_PX
|
||||
|
||||
def test_a_wider_probe_reserves_more(self):
|
||||
class Wide(Host):
|
||||
_SCORE_PROBE = "000-000"
|
||||
assert Wide()._score_reserve_width() > Host()._score_reserve_width()
|
||||
|
||||
def test_an_unmeasurable_font_reserves_nothing_rather_than_raising(self):
|
||||
h = Host()
|
||||
h.fonts = {'score': object()}
|
||||
assert h._score_reserve_width() == 0
|
||||
|
||||
|
||||
class TestLogoSlot:
|
||||
def test_the_slot_is_what_is_left_after_the_gap(self):
|
||||
h = Host(width=128, scroll={'center_gap': 40})
|
||||
assert h._logo_slot_width() == 44
|
||||
|
||||
def test_it_is_not_capped_at_the_card_height(self):
|
||||
# The height cap is what froze logos at 46px on a 128px card.
|
||||
h = Host(width=512, height=32, scroll={'center_gap': 40})
|
||||
assert h._logo_slot_width() > h.display_height
|
||||
|
||||
def test_a_gap_wider_than_the_card_still_leaves_a_usable_slot(self):
|
||||
assert Host(width=64, scroll={'center_gap': 200})._logo_slot_width() == 8
|
||||
|
||||
def test_the_cache_key_is_scoped_to_the_slot_not_just_the_name(self):
|
||||
# One cache dict is shared by renderers of different card widths.
|
||||
narrow = Host(width=64, scroll={'center_gap': 20})._logo_cache_key("NYY")
|
||||
wide = Host(width=256, scroll={'center_gap': 20})._logo_cache_key("NYY")
|
||||
assert narrow != wide
|
||||
|
||||
|
||||
class TestLayoutOffset:
|
||||
def _host(self, value):
|
||||
return Host(config={'customization': {'layout': {'score': {'x_offset': value}}}})
|
||||
|
||||
def test_it_reads_the_same_block_as_the_full_screen_scorebug(self):
|
||||
assert self._host(5)._layout_offset('score', 'x_offset') == 5
|
||||
|
||||
def test_a_string_offset_from_the_web_ui_is_coerced(self):
|
||||
assert self._host("-3")._layout_offset('score', 'x_offset') == -3
|
||||
|
||||
def test_a_bool_is_not_silently_an_offset_of_one(self):
|
||||
assert self._host(True)._layout_offset('score', 'x_offset', 9) == 9
|
||||
|
||||
@pytest.mark.parametrize("cfg", [{}, {'customization': {}},
|
||||
{'customization': {'layout': {}}}])
|
||||
def test_a_missing_block_gives_the_default(self, cfg):
|
||||
assert Host(config=cfg)._layout_offset('score', 'x_offset', 7) == 7
|
||||
|
||||
def test_an_unparseable_offset_gives_the_default(self):
|
||||
assert self._host("left")._layout_offset('score', 'x_offset', 4) == 4
|
||||
|
||||
|
||||
class TestUpcomingCenter:
|
||||
def test_none_draws_nothing(self):
|
||||
h = Host(scroll={'upcoming_center': 'none'})
|
||||
h._draw_upcoming_center(_draw(), {})
|
||||
assert h.drawn == []
|
||||
|
||||
def test_vs_is_the_default_and_never_a_score(self):
|
||||
# An upcoming game has not started; the extractor's 0-0 is noise.
|
||||
h = Host()
|
||||
h._draw_upcoming_center(_draw(), {'home_score': 0, 'away_score': 0})
|
||||
assert [t for t, _ in h.drawn] == ['VS']
|
||||
|
||||
def test_an_empty_vs_string_draws_nothing(self):
|
||||
h = Host(scroll={'vs_text': ''})
|
||||
h._draw_upcoming_center(_draw(), {})
|
||||
assert h.drawn == []
|
||||
|
||||
def test_date_time_stacks_both_lines(self):
|
||||
h = Host(scroll={'upcoming_center': 'date_time'})
|
||||
h._draw_upcoming_center(_draw(), {'game_date': 'Sep 19', 'game_time': '7:00 PM'})
|
||||
assert [t for t, _ in h.drawn] == ['Sep 19', '7:00 PM']
|
||||
|
||||
def test_hiding_both_lines_draws_nothing(self):
|
||||
h = Host(scroll={'upcoming_center': 'date_time',
|
||||
'show_date': False, 'show_time': False})
|
||||
h._draw_upcoming_center(_draw(), {'game_date': 'Sep 19', 'game_time': '7:00 PM'})
|
||||
assert h.drawn == []
|
||||
|
||||
|
||||
class TestUpcomingStatus:
|
||||
def test_time_on_top_and_date_below_by_default(self):
|
||||
h = Host()
|
||||
h._draw_upcoming_game_status(_draw(), {'game_date': 'Sep 19', 'game_time': '7:00 PM'})
|
||||
assert [t for t, _ in h.drawn] == ['7:00 PM', 'Sep 19']
|
||||
|
||||
def test_swap_date_time_reverses_them(self):
|
||||
h = Host(scroll={'swap_date_time': True})
|
||||
h._draw_upcoming_game_status(_draw(), {'game_date': 'Sep 19', 'game_time': '7:00 PM'})
|
||||
assert [t for t, _ in h.drawn] == ['Sep 19', '7:00 PM']
|
||||
|
||||
def test_it_stays_out_of_the_way_when_the_centre_already_has_them(self):
|
||||
# Otherwise the date and time print twice on the same card.
|
||||
h = Host(scroll={'upcoming_center': 'date_time'})
|
||||
h._draw_upcoming_game_status(_draw(), {'game_date': 'Sep 19', 'game_time': '7:00 PM'})
|
||||
assert h.drawn == []
|
||||
|
||||
def test_the_bottom_line_is_measured_not_a_fixed_offset(self):
|
||||
# A fixed -7 ran "Sep 19" past the card wherever the detail font is
|
||||
# 10px rather than 6px.
|
||||
h = Host(height=64)
|
||||
h._draw_upcoming_game_status(_draw(), {'game_date': 'Sep 19', 'game_time': '7:00 PM'})
|
||||
bottom_y = h.drawn[1][1][1]
|
||||
assert 0 <= bottom_y < h.display_height
|
||||
|
||||
|
||||
class TestRankings:
|
||||
def test_set_rankings_cache_replaces_the_cache(self):
|
||||
h = Host()
|
||||
h.set_rankings_cache({'UGA': 1})
|
||||
assert h._team_rankings_cache == {'UGA': 1}
|
||||
|
||||
|
||||
class TestContract:
|
||||
def test_the_mixin_carries_no_state_of_its_own(self):
|
||||
# Adoption must be one line on the class statement; a mixin with an
|
||||
# __init__ would force eight constructors to cooperate.
|
||||
assert '__init__' not in SportsGameRendererMixin.__dict__
|
||||
|
||||
def test_a_host_providing_the_documented_surface_needs_nothing_more(self):
|
||||
# Host defines exactly what the module docstring names. If the mixin
|
||||
# grows a new self.* dependency, this is what fails.
|
||||
h = Host()
|
||||
h._center_gap_width()
|
||||
h._logo_slot_width()
|
||||
h._logo_cache_key("X")
|
||||
h._layout_offset('score', 'x_offset')
|
||||
h._upcoming_date_and_time({})
|
||||
h._draw_upcoming_center(_draw(), {})
|
||||
h._draw_upcoming_game_status(_draw(), {})
|
||||
h.set_rankings_cache({})
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Odds must be fetched for the games shown, not every game in the window.
|
||||
|
||||
SportsUpcoming.update() collected every upcoming game in the schedule window
|
||||
and called _fetch_odds() on each one *inside* that collection loop, narrowing
|
||||
to upcoming_games_to_show only afterwards. The comment there said odds were
|
||||
fetched "only for games that will be displayed", but the sole narrowing it
|
||||
applied was show_favorite_teams_only, which is not the default -- so in the
|
||||
usual configuration nothing narrowed it at all.
|
||||
|
||||
Measured on a live rig: a college league produced 946 upcoming games in one
|
||||
cycle and displayed 1 of them. The same shape on the football plugin produced
|
||||
a burst of 467 sequential ESPN requests that ran for 35s and blew that
|
||||
plugin's 30s update budget, and it repeats every time the 1h odds TTL expires.
|
||||
|
||||
SportsLive is deliberately different: it walks the raw event list because it
|
||||
has to find which games are live, but only fetches odds for a game that has
|
||||
already passed the is_live/is_halftime test, so the fan-out is bounded by how
|
||||
many games are actually in progress.
|
||||
"""
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
MODES = (Path(__file__).resolve().parent.parent
|
||||
/ "src" / "base_classes" / "sports" / "modes.py")
|
||||
TREE = ast.parse(MODES.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _fetch_sites():
|
||||
"""(class name, method name, lineno) for every self._fetch_odds(...) call."""
|
||||
calls = [n.lineno for n in ast.walk(TREE)
|
||||
if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
|
||||
and n.func.attr == "_fetch_odds"]
|
||||
sites = []
|
||||
for cls in [n for n in ast.walk(TREE) if isinstance(n, ast.ClassDef)]:
|
||||
for fn in [n for n in cls.body if isinstance(n, ast.FunctionDef)]:
|
||||
for lineno in calls:
|
||||
if fn.lineno <= lineno <= (fn.end_lineno or fn.lineno):
|
||||
sites.append((cls.name, fn.name, lineno))
|
||||
assert len(sites) == len(calls), "a _fetch_odds call sits outside any method"
|
||||
return sites
|
||||
|
||||
|
||||
def _innermost_loop_iterable(lineno):
|
||||
best = None
|
||||
for node in ast.walk(TREE):
|
||||
if isinstance(node, ast.For) and \
|
||||
node.lineno <= lineno <= (node.end_lineno or node.lineno):
|
||||
if best is None or node.lineno > best.lineno:
|
||||
best = node
|
||||
return None if best is None else ast.unparse(best.iter)
|
||||
|
||||
|
||||
def _spans(body, lineno):
|
||||
"""True when `lineno` falls inside this list of statements."""
|
||||
return any(n.lineno <= lineno <= (n.end_lineno or n.lineno) for n in body)
|
||||
|
||||
|
||||
def _parents(tree):
|
||||
table = {}
|
||||
for node in ast.walk(tree):
|
||||
for child in ast.iter_child_nodes(node):
|
||||
table[child] = node
|
||||
return table
|
||||
|
||||
|
||||
PARENTS = _parents(TREE)
|
||||
|
||||
|
||||
def _mentions_positively(test, names):
|
||||
"""True when `test` references every name, none of them under a `not`.
|
||||
|
||||
Structural, not textual. Matching the unparsed source would accept
|
||||
`not (details["is_live"] or details["is_halftime"])` -- which selects
|
||||
exactly the non-live games this guard exists to exclude -- because the
|
||||
names still appear in the text.
|
||||
"""
|
||||
found = set()
|
||||
for node in ast.walk(test):
|
||||
if not (isinstance(node, ast.Constant) and node.value in names):
|
||||
continue
|
||||
negated = False
|
||||
cursor = node
|
||||
while cursor is not test and cursor in PARENTS:
|
||||
cursor = PARENTS[cursor]
|
||||
if isinstance(cursor, ast.UnaryOp) and isinstance(cursor.op, ast.Not):
|
||||
negated = True
|
||||
break
|
||||
if not negated:
|
||||
found.add(node.value)
|
||||
return found >= set(names)
|
||||
|
||||
|
||||
def _guarded_by_positive(lineno, names):
|
||||
"""True when some enclosing `if` runs this line only if `names` hold.
|
||||
|
||||
Only the TRUE branch counts: an `if` whose `else` contains the call would
|
||||
otherwise look like a guard while doing the opposite.
|
||||
"""
|
||||
for node in ast.walk(TREE):
|
||||
if isinstance(node, ast.If) and _spans(node.body, lineno) \
|
||||
and _mentions_positively(node.test, names):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def test_every_fetch_site_is_accounted_for():
|
||||
"""A new call site must be classified deliberately, not inherited silently."""
|
||||
found = {(cls, fn) for cls, fn, _ in _fetch_sites()}
|
||||
assert found == {("SportsUpcoming", "update"), ("SportsLive", "update")}, (
|
||||
f"unexpected _fetch_odds call sites: {sorted(found)}. Each one is a "
|
||||
"sequential ESPN request per game -- classify it here on purpose.")
|
||||
|
||||
|
||||
def test_upcoming_fetches_only_the_selected_games():
|
||||
for cls, _fn, lineno in _fetch_sites():
|
||||
if cls != "SportsUpcoming":
|
||||
continue
|
||||
iterable = _innermost_loop_iterable(lineno)
|
||||
assert iterable == "team_games", (
|
||||
f"SportsUpcoming._fetch_odds at line {lineno} iterates over "
|
||||
f"{iterable!r}. It must run over team_games -- already narrowed to "
|
||||
"upcoming_games_to_show -- not over every event in the schedule "
|
||||
"window. Each item costs one sequential ESPN request.")
|
||||
|
||||
|
||||
def test_live_only_fetches_for_games_actually_in_progress():
|
||||
for cls, _fn, lineno in _fetch_sites():
|
||||
if cls != "SportsLive":
|
||||
continue
|
||||
assert _guarded_by_positive(lineno, {"is_live", "is_halftime"}), (
|
||||
f"SportsLive._fetch_odds at line {lineno} does not sit in the true "
|
||||
"branch of a test requiring the game to be in progress. Without "
|
||||
"that, it fans out across the whole event list -- one sequential "
|
||||
"ESPN request per game.")
|
||||
@@ -0,0 +1,321 @@
|
||||
"""The shared sports.py mixins: their host contract, and _plugin_dir.
|
||||
|
||||
Two things are worth testing here and the rest is not. The 45 method bodies
|
||||
moved verbatim from the plugins, so they are covered by the plugins' own tests
|
||||
and by 176 byte-identical safety-harness renders. What is genuinely new is:
|
||||
|
||||
1. The contract. Every ``self.<CONSTANT>`` a mixin reads must be defined on the
|
||||
mixin, or a host that does not happen to declare it raises AttributeError at
|
||||
runtime. Two were missed on the first pass (_QUALITY_CHOICES and
|
||||
_RANKING_COVERAGE_SECONDS); the eight plugins all declare them, so nothing
|
||||
failed -- it would only have bitten a ninth. The test derives the list rather
|
||||
than restating it, so the next omission fails here instead of in the field.
|
||||
|
||||
2. ``_plugin_dir``. This is the only line of genuinely new logic in the move. In
|
||||
sports.py these methods found config_schema.json with ``__file__``; here that
|
||||
is src/common/, so the plugin directory has to be recovered from the
|
||||
instance -- and getting it wrong is silent, costing grid-snapped font sizes
|
||||
(measured at 81% anti-aliased edges) rather than raising.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import sys
|
||||
import types
|
||||
from abc import ABC
|
||||
|
||||
import pytest
|
||||
|
||||
from src.common import sports_shared
|
||||
from src.common.sports_shared import (
|
||||
SportsCoreSharedMixin, SportsLiveSharedMixin, SportsRecentSharedMixin)
|
||||
|
||||
MIXINS = (SportsCoreSharedMixin, SportsLiveSharedMixin, SportsRecentSharedMixin)
|
||||
|
||||
|
||||
def _constants_read_by_mixins():
|
||||
"""Every ALL-CAPS ``self.X`` the mixin bodies read, found by parsing them."""
|
||||
tree = ast.parse(open(sports_shared.__file__).read())
|
||||
names = set()
|
||||
for node in ast.walk(tree):
|
||||
if (isinstance(node, ast.Attribute)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "self"
|
||||
and node.attr.upper() == node.attr):
|
||||
names.add(node.attr)
|
||||
return names
|
||||
|
||||
|
||||
class TestHostContract:
|
||||
def test_every_constant_read_is_also_defined(self):
|
||||
# Otherwise a host that does not declare it raises AttributeError the
|
||||
# first time the code path runs -- which for these is mid-render.
|
||||
missing = sorted(
|
||||
name for name in _constants_read_by_mixins()
|
||||
if not any(hasattr(m, name) for m in MIXINS))
|
||||
assert missing == [], (
|
||||
f"read but never defined on a mixin: {missing}. Give each a default "
|
||||
f"on SportsCoreSharedMixin and document it in the module docstring.")
|
||||
|
||||
@pytest.mark.parametrize("name,expected", [
|
||||
("_QUALITY_CHOICES", frozenset({"any", "ranked"})),
|
||||
("_RANKING_COVERAGE_SECONDS", 3600),
|
||||
("_SCORE_PROBE_TEXT", "00-00"),
|
||||
("_FONT_DESIGN_HEIGHT", 32),
|
||||
])
|
||||
def test_defaults_match_what_the_plugins_ship(self, name, expected):
|
||||
# The eight plugins declare their own copies, which shadow these. The
|
||||
# values must still agree, or a ninth plugin inheriting the default
|
||||
# behaves differently from the eight.
|
||||
assert getattr(SportsCoreSharedMixin, name) == expected
|
||||
|
||||
def test_only_the_recent_mixin_carries_a_constructor(self):
|
||||
# SportsCore and SportsLive keep their own __init__ -- those differ per
|
||||
# plugin. SportsRecent.__init__ was one of the 48 byte-identical bodies,
|
||||
# so it moved with the rest; that is deliberate, not an oversight.
|
||||
assert "__init__" not in SportsCoreSharedMixin.__dict__
|
||||
assert "__init__" not in SportsLiveSharedMixin.__dict__
|
||||
assert "__init__" in SportsRecentSharedMixin.__dict__
|
||||
|
||||
def test_the_recent_constructor_still_chains_to_the_host(self):
|
||||
"""Its zero-arg super() binds to where it is DEFINED, not where it is used.
|
||||
|
||||
Moving a body containing bare ``super()`` is the one move that can
|
||||
change meaning: the compiler closes over __class__ = the defining class,
|
||||
so after the move that is SportsRecentSharedMixin rather than the
|
||||
plugin's SportsRecent. It still works only because the mixin is listed
|
||||
first, leaving the host class next in the MRO -- adopt it in the other
|
||||
order and the chain silently skips the host's __init__.
|
||||
"""
|
||||
calls = []
|
||||
|
||||
class Host:
|
||||
def __init__(self, config, display_manager, cache_manager, logger, sport_key):
|
||||
calls.append(sport_key)
|
||||
self.mode_config = {}
|
||||
|
||||
class Recent(SportsRecentSharedMixin, Host):
|
||||
pass
|
||||
|
||||
inst = Recent({}, None, None, None, "nhl")
|
||||
assert calls == ["nhl"], "the host constructor must still run"
|
||||
assert inst.current_game_index == 0
|
||||
assert inst.update_interval == 3600
|
||||
assert inst._zero_clock_timestamps == {}
|
||||
|
||||
def test_adopting_the_recent_mixin_second_would_skip_the_host(self):
|
||||
# The failure mode the ordering above prevents, pinned so nobody
|
||||
# "tidies" the base list.
|
||||
calls = []
|
||||
|
||||
class Host:
|
||||
def __init__(self, *a):
|
||||
calls.append(a)
|
||||
self.mode_config = {}
|
||||
|
||||
class Wrong(Host, SportsRecentSharedMixin):
|
||||
pass
|
||||
|
||||
Wrong({}, None, None, None, "nhl")
|
||||
# Host.__init__ wins and the mixin's setup never runs at all.
|
||||
assert not hasattr(Wrong({}, None, None, None, "nhl"), "current_game_index")
|
||||
|
||||
|
||||
class _Host(SportsCoreSharedMixin):
|
||||
pass
|
||||
|
||||
|
||||
def _write_plugin(tmp_path, name="fakeplug", schema=True):
|
||||
"""A throwaway package on sys.path, with or without a config_schema.json."""
|
||||
d = tmp_path / name
|
||||
d.mkdir()
|
||||
(d / "__init__.py").write_text("")
|
||||
(d / "mod.py").write_text("class Leaf:\n pass\n")
|
||||
if schema:
|
||||
(d / "config_schema.json").write_text(
|
||||
'{"properties": {"customization": {"properties": '
|
||||
'{"score": {"properties": {"font_size": {"default": 16}}}}}}}')
|
||||
return d
|
||||
|
||||
|
||||
class TestPluginDir:
|
||||
def test_it_finds_the_directory_holding_config_schema_json(self, tmp_path, monkeypatch):
|
||||
d = _write_plugin(tmp_path)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
mod = __import__("fakeplug.mod", fromlist=["Leaf"])
|
||||
host = type("H", (mod.Leaf, SportsCoreSharedMixin), {})()
|
||||
assert host._plugin_dir() == str(d)
|
||||
|
||||
def test_a_class_built_by_type_still_resolves(self, tmp_path, monkeypatch):
|
||||
# SportsCore is an ABC, so type(name, bases, ns) reports __module__ as
|
||||
# "abc" rather than the plugin -- which is exactly what the plugins'
|
||||
# own tests build. Walking the MRO is what steps past it.
|
||||
d = _write_plugin(tmp_path, "abcplug")
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
mod = __import__("abcplug.mod", fromlist=["Leaf"])
|
||||
|
||||
class Base(SportsCoreSharedMixin, mod.Leaf, ABC):
|
||||
pass
|
||||
|
||||
synthetic = type("Probe", (Base,), {})
|
||||
assert synthetic.__module__ == "abc", "precondition: the trap this guards"
|
||||
assert synthetic.__new__(synthetic)._plugin_dir() == str(d)
|
||||
|
||||
def test_it_returns_none_when_no_schema_is_anywhere_on_the_mro(self, tmp_path, monkeypatch):
|
||||
d = _write_plugin(tmp_path, "noschema", schema=False)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
mod = __import__("noschema.mod", fromlist=["Leaf"])
|
||||
host = type("H", (mod.Leaf, SportsCoreSharedMixin), {})()
|
||||
# None rather than a wrong guess: _schema_font_size then caches empty
|
||||
# and every element keeps its own default.
|
||||
assert host._plugin_dir() is None
|
||||
|
||||
def test_it_never_returns_the_core_module_directory(self):
|
||||
# The bug this replaced: __file__ pointed at src/common/, so the schema
|
||||
# was never found and font sizes silently stopped snapping to the grid.
|
||||
host = _Host()
|
||||
core_common = os.path.dirname(os.path.abspath(sports_shared.__file__))
|
||||
assert host._plugin_dir() != core_common
|
||||
|
||||
def test_a_module_with_no_file_is_skipped_not_crashed_on(self, monkeypatch):
|
||||
# Namespace packages and some frozen/dynamic modules have no __file__.
|
||||
ghost = types.ModuleType("ghost_no_file")
|
||||
if hasattr(ghost, "__file__"):
|
||||
del ghost.__file__
|
||||
monkeypatch.setitem(sys.modules, "ghost_no_file", ghost)
|
||||
cls = type("H", (SportsCoreSharedMixin,), {"__module__": "ghost_no_file"})
|
||||
assert cls.__new__(cls)._plugin_dir() is None
|
||||
|
||||
|
||||
class TestSchemaFontSize:
|
||||
def test_it_reads_the_plugin_schema_not_the_cores(self, tmp_path, monkeypatch):
|
||||
d = _write_plugin(tmp_path, "sizeplug")
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
mod = __import__("sizeplug.mod", fromlist=["Leaf"])
|
||||
host = type("H", (mod.Leaf, SportsCoreSharedMixin), {})()
|
||||
assert host._schema_font_size("score") == 16
|
||||
|
||||
def test_an_unknown_element_is_none(self, tmp_path, monkeypatch):
|
||||
_write_plugin(tmp_path, "unkplug")
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
mod = __import__("unkplug.mod", fromlist=["Leaf"])
|
||||
host = type("H", (mod.Leaf, SportsCoreSharedMixin), {})()
|
||||
assert host._schema_font_size("nonesuch") is None
|
||||
|
||||
def test_an_empty_key_is_none_without_touching_the_disk(self):
|
||||
assert _Host()._schema_font_size("") is None
|
||||
|
||||
def test_a_missing_schema_degrades_to_none_rather_than_raising(self, tmp_path, monkeypatch):
|
||||
_write_plugin(tmp_path, "bareplug", schema=False)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
mod = __import__("bareplug.mod", fromlist=["Leaf"])
|
||||
host = type("H", (mod.Leaf, SportsCoreSharedMixin), {})()
|
||||
assert host._schema_font_size("score") is None
|
||||
|
||||
|
||||
class _LiveHost(SportsLiveSharedMixin):
|
||||
"""The documented contract for the live mixin, and nothing else."""
|
||||
|
||||
def __init__(self, no_data_interval=300, stale_game_timeout=600, over=()):
|
||||
self.no_data_interval = no_data_interval
|
||||
self.stale_game_timeout = stale_game_timeout
|
||||
self.game_update_timestamps = {}
|
||||
self._over = set(over)
|
||||
|
||||
class _L:
|
||||
def __getattr__(self, _n):
|
||||
return lambda *a, **k: None
|
||||
self.logger = _L()
|
||||
|
||||
def _is_game_really_over(self, game):
|
||||
return game.get("id") in self._over
|
||||
|
||||
|
||||
class TestLiveMixin:
|
||||
"""These three moved to the core, so they are tested here.
|
||||
|
||||
They were already covered by hockey's and lacrosse's own tests, but those
|
||||
two plugins disable live mode in their safety-harness fixtures, so the 176
|
||||
renders never exercise this path. Testing the mixin directly means the
|
||||
coverage no longer depends on which plugin happens to have a unit test.
|
||||
"""
|
||||
|
||||
def test_a_stale_game_is_dropped_and_forgotten(self):
|
||||
h = _LiveHost(stale_game_timeout=600)
|
||||
import time as _t
|
||||
h.game_update_timestamps["g1"] = {"last_seen": _t.time() - 5000}
|
||||
games = [{"id": "g1", "home_abbr": "H", "away_abbr": "A"}]
|
||||
h._detect_stale_games(games)
|
||||
assert games == []
|
||||
assert "g1" not in h.game_update_timestamps, "its timestamp must go too"
|
||||
|
||||
def test_a_fresh_game_survives(self):
|
||||
h = _LiveHost(stale_game_timeout=600)
|
||||
import time as _t
|
||||
h.game_update_timestamps["g1"] = {"last_seen": _t.time() - 5}
|
||||
games = [{"id": "g1"}]
|
||||
h._detect_stale_games(games)
|
||||
assert len(games) == 1
|
||||
|
||||
def test_a_game_never_seen_is_not_treated_as_stale(self):
|
||||
# last_seen 0 means "no reading", not "seen at the epoch".
|
||||
h = _LiveHost()
|
||||
games = [{"id": "g1"}]
|
||||
h._detect_stale_games(games)
|
||||
assert len(games) == 1
|
||||
|
||||
def test_a_game_with_no_id_is_left_alone(self):
|
||||
h = _LiveHost()
|
||||
games = [{"home_abbr": "H"}]
|
||||
h._detect_stale_games(games)
|
||||
assert len(games) == 1
|
||||
|
||||
def test_a_finished_game_is_dropped_even_when_fresh(self):
|
||||
h = _LiveHost(over=("g2",))
|
||||
games = [{"id": "g1"}, {"id": "g2"}]
|
||||
h._detect_stale_games(games)
|
||||
assert [g["id"] for g in games] == ["g1"]
|
||||
|
||||
def test_removing_several_does_not_skip_any(self):
|
||||
# It iterates a copy for exactly this reason; mutating the live list
|
||||
# while looping would step over the element after each removal.
|
||||
h = _LiveHost(over=("g1", "g2", "g3"))
|
||||
games = [{"id": "g1"}, {"id": "g2"}, {"id": "g3"}]
|
||||
h._detect_stale_games(games)
|
||||
assert games == []
|
||||
|
||||
def test_the_idle_interval_escalates_with_the_empty_streak(self):
|
||||
h = _LiveHost(no_data_interval=60)
|
||||
h.live_idle_max_interval = 100000
|
||||
base = h._idle_live_interval()
|
||||
h._empty_live_streak = 6
|
||||
short = h._idle_live_interval()
|
||||
h._empty_live_streak = 24
|
||||
long = h._idle_live_interval()
|
||||
assert base < short < long
|
||||
|
||||
def test_the_ceiling_bounds_even_the_unescalated_interval(self):
|
||||
# base > ceiling is a reachable config: the two settings are
|
||||
# independent integers with no cross-validation. Returning base
|
||||
# unclamped made the wait SHRINK as the streak grew.
|
||||
h = _LiveHost(no_data_interval=3600)
|
||||
h.live_idle_max_interval = 900
|
||||
h._empty_live_streak = 0
|
||||
assert h._idle_live_interval() == 900
|
||||
h._empty_live_streak = 24
|
||||
assert h._idle_live_interval() == 900
|
||||
|
||||
def test_finding_a_live_game_resets_the_streak(self):
|
||||
h = _LiveHost()
|
||||
h._note_live_fetch(False)
|
||||
h._note_live_fetch(False)
|
||||
assert h._empty_live_streak == 2
|
||||
h._note_live_fetch(True)
|
||||
assert h._empty_live_streak == 0
|
||||
|
||||
def test_the_streak_starts_from_absent_state(self):
|
||||
# The host is not required to pre-declare _empty_live_streak.
|
||||
h = _LiveHost()
|
||||
assert not hasattr(h, "_empty_live_streak")
|
||||
h._note_live_fetch(False)
|
||||
assert h._empty_live_streak == 1
|
||||
@@ -0,0 +1,310 @@
|
||||
"""What happens to an adopted sports plugin on each core it can meet.
|
||||
|
||||
B5 moved the eight scoreboards onto `src.common.sports_scroll` behind a guarded
|
||||
import, keeping a bundled copy as the fallback. B6 deletes those copies. The
|
||||
two phases make *different* promises, and only the second one is obvious:
|
||||
|
||||
| | bundled copy present | bundled copy removed |
|
||||
|--------------------|---------------------------------|-----------------------------|
|
||||
| **pinned old core**| loads -- B5's whole guarantee | ERROR, naming the module |
|
||||
| **current core** | loads, using core code | loads, using core code |
|
||||
|
||||
The top-left cell is the one worth having: nothing else in this suite proves an
|
||||
adopted plugin still runs on a core that predates the module, and that claim is
|
||||
the entire basis for having shipped B5 ahead of B6's gate.
|
||||
|
||||
The bottom-left cell is the B6 failure mode, and it is asserted through
|
||||
`PluginManager.load_plugin` rather than a bare import on purpose. The manager
|
||||
catches the `ModuleNotFoundError`, so nothing propagates to a caller: a test
|
||||
that expected `pytest.raises` would pass against a core where the module is
|
||||
merely *broken* rather than absent, and would say nothing about what the user
|
||||
actually experiences. What they get is a plugin parked in ERROR and one log
|
||||
line -- which is precisely why B6 needs the install gate rather than trusting
|
||||
the failure to be noticed.
|
||||
|
||||
This covers the load path. The install/update gate -- which is what should stop
|
||||
a sunset plugin reaching an old core in the first place -- is the other half of
|
||||
the guarantee and is tested in test_plugin_compatibility_gate.py.
|
||||
|
||||
See docs/SPORTS_UNIFICATION.md, "B6 -- why the sunset needs more than a version
|
||||
floor".
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
project_root = Path(__file__).parent.parent
|
||||
if str(project_root) not in sys.path:
|
||||
sys.path.insert(0, str(project_root))
|
||||
|
||||
from src.plugin_system.plugin_manager import PluginManager
|
||||
from src.plugin_system.plugin_state import PluginState
|
||||
|
||||
|
||||
_PLUGIN_IDS = itertools.count()
|
||||
|
||||
CORE_MODULE = "src.common.sports_scroll"
|
||||
|
||||
# Only the leaf. A pre-3.2.0 core still ships `src/common/` -- scroll_helper
|
||||
# and friends live there -- and it is `sports_scroll.py` alone that is absent.
|
||||
# Hiding the whole package would be a different, harsher core than any that
|
||||
# shipped, and it would make the load failure name the package rather than the
|
||||
# module, which is the thing a reader needs to see.
|
||||
HIDDEN = (CORE_MODULE,)
|
||||
|
||||
|
||||
class _PinnedOldCore:
|
||||
"""Make `src.common.sports_scroll` un-importable for the duration.
|
||||
|
||||
A meta_path finder rather than a monkeypatched `__import__`: the plugin is
|
||||
executed by the real loader through `exec_module`, so the block has to live
|
||||
in the import system itself to be reached.
|
||||
"""
|
||||
|
||||
def __init__(self, *names):
|
||||
self.names = set(names)
|
||||
self._saved = {}
|
||||
|
||||
def find_spec(self, fullname, path=None, target=None):
|
||||
if fullname in self.names:
|
||||
raise ModuleNotFoundError(f"No module named {fullname!r}", name=fullname)
|
||||
return None
|
||||
|
||||
def __enter__(self):
|
||||
for name in list(sys.modules):
|
||||
if name in self.names:
|
||||
self._saved[name] = sys.modules.pop(name)
|
||||
sys.meta_path.insert(0, self)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
sys.meta_path.remove(self)
|
||||
sys.modules.update(self._saved)
|
||||
return False
|
||||
|
||||
|
||||
# The two source shapes, kept as literals rather than copied from a plugin at
|
||||
# runtime: this file lives in the core repo and must not depend on a plugin
|
||||
# checkout being present, and pinning the shapes here means a plugin that
|
||||
# drifts away from one is a visible edit, not a silently weakened test.
|
||||
|
||||
# B5, as the eight scoreboards ship today: prefer the core module, fall back to
|
||||
# the bundled copy. The except clause is narrow on purpose -- a bare
|
||||
# `except ImportError` would also swallow a failure raised *inside* a core
|
||||
# module that is present, quietly loading the legacy copy and hiding a broken
|
||||
# core install.
|
||||
ADOPTED_WITH_FALLBACK = '''
|
||||
_USING_CORE_SCROLL = False
|
||||
try:
|
||||
from src.common.sports_scroll import SportsScrollDisplay as _Base
|
||||
_USING_CORE_SCROLL = True
|
||||
except ModuleNotFoundError as exc:
|
||||
if exc.name not in {"src", "src.common", "src.common.sports_scroll"}:
|
||||
raise
|
||||
_Base = None
|
||||
|
||||
if not _USING_CORE_SCROLL:
|
||||
from scroll_display_legacy import LegacyScrollDisplay as _Base
|
||||
'''
|
||||
|
||||
# B6, once the copies are deleted: there is nothing to fall back to, so the
|
||||
# guard goes with them. Keeping the try/except while removing the file it
|
||||
# falls back to would only mislabel the failure -- the plugin would report a
|
||||
# missing `scroll_display_legacy` and say nothing about the core module that
|
||||
# is actually absent.
|
||||
SUNSET_NO_FALLBACK = '''
|
||||
from src.common.sports_scroll import SportsScrollDisplay as _Base
|
||||
|
||||
_USING_CORE_SCROLL = True
|
||||
'''
|
||||
|
||||
PLUGIN_BODY = '''
|
||||
from src.plugin_system.base_plugin import BasePlugin
|
||||
|
||||
|
||||
class SunsetProbe(BasePlugin):
|
||||
"""Records which scroll implementation the guarded import selected."""
|
||||
|
||||
using_core_scroll = _USING_CORE_SCROLL
|
||||
scroll_base = _Base
|
||||
|
||||
def update(self):
|
||||
pass
|
||||
|
||||
def display(self, force_clear=False):
|
||||
pass
|
||||
'''
|
||||
|
||||
LEGACY_COPY = '''
|
||||
class LegacyScrollDisplay:
|
||||
"""Stands in for the bundled pre-3.2.0 implementation."""
|
||||
'''
|
||||
|
||||
|
||||
def _write_plugin(plugins_dir: Path, plugin_id: str, *, bundled_copy: bool) -> Path:
|
||||
path = plugins_dir / plugin_id
|
||||
path.mkdir(parents=True)
|
||||
(path / "manifest.json").write_text(
|
||||
json.dumps({
|
||||
"id": plugin_id,
|
||||
"name": "Sunset Probe",
|
||||
"version": "1.0.0",
|
||||
"entry_point": "manager.py",
|
||||
"class_name": "SunsetProbe",
|
||||
"display_modes": ["sunset_probe"],
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
shape = ADOPTED_WITH_FALLBACK if bundled_copy else SUNSET_NO_FALLBACK
|
||||
(path / "manager.py").write_text(shape + PLUGIN_BODY, encoding="utf-8")
|
||||
if bundled_copy:
|
||||
(path / "scroll_display_legacy.py").write_text(LEGACY_COPY, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def load(tmp_path):
|
||||
"""Load a synthetic adopted plugin through the real PluginManager.
|
||||
|
||||
Returns a callable taking the two axes of the matrix and handing back the
|
||||
manager, so the caller can ask it for state and recorded error.
|
||||
"""
|
||||
plugins_dir = tmp_path / "plugin-repos"
|
||||
plugins_dir.mkdir()
|
||||
|
||||
def _load(*, core_has_module: bool, bundled_copy: bool, hide=HIDDEN):
|
||||
# A distinct id per cell, from a counter that spans the whole session.
|
||||
# The loader names plugin modules after the plugin id and sys.modules
|
||||
# is process-global, so a per-test counter would hand the second test
|
||||
# the first test's already-imported module -- which passes or fails on
|
||||
# the wrong plugin's import.
|
||||
plugin_id = f"sunset-probe-{next(_PLUGIN_IDS)}"
|
||||
_write_plugin(plugins_dir, plugin_id, bundled_copy=bundled_copy)
|
||||
|
||||
with patch('src.common.permission_utils.ensure_directory_permissions'):
|
||||
manager = PluginManager(
|
||||
plugins_dir=str(plugins_dir),
|
||||
config_manager=MagicMock(),
|
||||
display_manager=MagicMock(),
|
||||
cache_manager=MagicMock(),
|
||||
font_manager=MagicMock(),
|
||||
)
|
||||
manager.discover_plugins()
|
||||
if core_has_module:
|
||||
ok = manager.load_plugin(plugin_id)
|
||||
else:
|
||||
with _PinnedOldCore(*hide):
|
||||
ok = manager.load_plugin(plugin_id)
|
||||
return manager, plugin_id, ok
|
||||
|
||||
return _load
|
||||
|
||||
|
||||
def _assert_loaded(manager, plugin_id, ok):
|
||||
assert ok is True, (
|
||||
f"load_plugin returned False; state is "
|
||||
f"{manager.state_manager.get_state(plugin_id)}, error "
|
||||
f"{manager.state_manager.get_error_info(plugin_id)}"
|
||||
)
|
||||
assert manager.state_manager.get_state(plugin_id) is not PluginState.ERROR
|
||||
|
||||
|
||||
class TestBundledCopyPresent:
|
||||
"""B5's shape: the guarded import with the fallback still shipped."""
|
||||
|
||||
def test_old_core_falls_back_and_still_loads(self, load):
|
||||
"""The claim that made it safe to ship B5 before B6's gate."""
|
||||
manager, plugin_id, ok = load(core_has_module=False, bundled_copy=True)
|
||||
|
||||
_assert_loaded(manager, plugin_id, ok)
|
||||
plugin = manager.plugins[plugin_id]
|
||||
assert plugin.using_core_scroll is False, (
|
||||
"the core module was hidden, so the plugin must be on its bundled copy"
|
||||
)
|
||||
assert plugin.scroll_base.__name__ == "LegacyScrollDisplay"
|
||||
|
||||
def test_current_core_prefers_the_core_module(self, load):
|
||||
manager, plugin_id, ok = load(core_has_module=True, bundled_copy=True)
|
||||
|
||||
_assert_loaded(manager, plugin_id, ok)
|
||||
plugin = manager.plugins[plugin_id]
|
||||
assert plugin.using_core_scroll is True, (
|
||||
"the bundled copy must not win while the core module is importable"
|
||||
)
|
||||
assert plugin.scroll_base.__name__ == "SportsScrollDisplay"
|
||||
|
||||
|
||||
def test_a_core_without_the_package_at_all_still_falls_back(self, load):
|
||||
"""The guard's other accepted names.
|
||||
|
||||
Its except clause accepts `src` and `src.common` as well as the module
|
||||
itself, so those branches exist in all eight shipped plugins. No core
|
||||
that old is likely still running, but the code claiming to handle it is
|
||||
real and nothing else exercises it -- an untested branch in a fallback
|
||||
is exactly the kind that rots unnoticed until the fallback is needed.
|
||||
"""
|
||||
manager, plugin_id, ok = load(
|
||||
core_has_module=False, bundled_copy=True, hide=(CORE_MODULE, "src.common")
|
||||
)
|
||||
|
||||
_assert_loaded(manager, plugin_id, ok)
|
||||
assert manager.plugins[plugin_id].using_core_scroll is False
|
||||
|
||||
|
||||
class TestBundledCopyRemoved:
|
||||
"""B6's shape: the copies are gone and only the core module remains."""
|
||||
|
||||
def test_current_core_still_loads(self, load):
|
||||
manager, plugin_id, ok = load(core_has_module=True, bundled_copy=False)
|
||||
|
||||
_assert_loaded(manager, plugin_id, ok)
|
||||
assert manager.plugins[plugin_id].using_core_scroll is True
|
||||
|
||||
def test_old_core_errors_and_records_the_missing_module(self, load):
|
||||
"""The B6 failure mode, as the user meets it.
|
||||
|
||||
Not `pytest.raises`: load_plugin catches it, so nothing reaches a
|
||||
caller. The observable consequences are the ERROR state and the
|
||||
recorded error -- and the error has to name the module, or whoever
|
||||
reads the log cannot tell a missing core module from any other
|
||||
import failure.
|
||||
"""
|
||||
manager, plugin_id, ok = load(core_has_module=False, bundled_copy=False)
|
||||
|
||||
assert ok is False, "a plugin with no scroll implementation must not load"
|
||||
assert manager.state_manager.get_state(plugin_id) is PluginState.ERROR
|
||||
assert plugin_id not in manager.plugins, (
|
||||
"a plugin that failed to load must not be left registered"
|
||||
)
|
||||
|
||||
info = manager.state_manager.get_error_info(plugin_id)
|
||||
assert info is not None, "ERROR state recorded no error to explain it"
|
||||
assert info['error_type'] == 'ModuleNotFoundError', info
|
||||
assert CORE_MODULE in info['error'], (
|
||||
f"the recorded error must name the module that was missing, got {info['error']!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_the_matrix_has_one_failing_cell(load):
|
||||
"""Guards the shape of the table itself.
|
||||
|
||||
Each cell above is asserted on its own, so a change that broke two of them
|
||||
in compensating ways could leave every individual test passing. This says
|
||||
the outcome depends on both axes and fails in exactly one combination.
|
||||
"""
|
||||
outcomes = {
|
||||
(core, bundled): load(core_has_module=core, bundled_copy=bundled)[2]
|
||||
for core in (True, False)
|
||||
for bundled in (True, False)
|
||||
}
|
||||
assert outcomes == {
|
||||
(True, True): True,
|
||||
(True, False): True,
|
||||
(False, True): True,
|
||||
(False, False): False,
|
||||
}, outcomes
|
||||
@@ -0,0 +1,147 @@
|
||||
"""The captive portal's fixed-argument sudo calls must be granted.
|
||||
|
||||
The installers write two allow-lists, /etc/sudoers.d/ledmatrix_web and
|
||||
ledmatrix_wifi. A sudo call absent from both needs a password, which a service
|
||||
cannot supply, so it fails.
|
||||
|
||||
Four such calls were ungranted, all of them captive-portal teardown/setup:
|
||||
|
||||
sysctl -w net.ipv4.ip_forward=0|1 wifi_manager.py:788, 883
|
||||
nft add|delete table ip ledmatrix wifi_manager.py:835, 895
|
||||
rfkill unblock wifi wifi_manager.py:1811
|
||||
mkdir -p .../dnsmasq-shared.d wifi_manager.py:922
|
||||
|
||||
It goes unnoticed because a stock Raspberry Pi image ships
|
||||
/etc/sudoers.d/010_pi-nopasswd granting the default user
|
||||
`ALL=(ALL) NOPASSWD: ALL`, which satisfies every gap in both files. It only
|
||||
bites once that blanket rule is removed or the service runs as another user.
|
||||
|
||||
Scope, deliberately narrow: this pins the four commands above, each of which
|
||||
can be written out literally. The portal makes further sudo calls whose
|
||||
arguments are built at runtime -- iptables and nft rules carrying an interface
|
||||
name and a port, `ip addr`, `ip link` -- and those cannot be granted safely
|
||||
here. A rule covering them needs a trailing wildcard, and
|
||||
`iptables --modprobe=/path/to/anything` runs that path as root, so
|
||||
`NOPASSWD: iptables *` is a root shell for the web user by another name.
|
||||
Closing that half needs a privileged helper that builds the rules itself and
|
||||
takes only an interface and a port, granted the way safe_plugin_rm.sh already
|
||||
is. That is a design decision, not a one-line grant, and belongs in its own
|
||||
change.
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
INSTALLERS = (
|
||||
ROOT / "first_time_install.sh",
|
||||
ROOT / "scripts" / "install" / "configure_wifi_permissions.sh",
|
||||
)
|
||||
|
||||
#: Commands this change grants, each fully literal in the source.
|
||||
REQUIRED = (
|
||||
("sysctl", "-w", "net.ipv4.ip_forward=0"),
|
||||
("sysctl", "-w", "net.ipv4.ip_forward=1"),
|
||||
("nft", "add", "table", "ip", "ledmatrix"),
|
||||
("nft", "delete", "table", "ip", "ledmatrix"),
|
||||
("rfkill", "unblock", "wifi"),
|
||||
("mkdir", "-p", "/etc/NetworkManager/dnsmasq-shared.d"),
|
||||
)
|
||||
|
||||
#: Tools with an option that executes a program of the caller's choosing.
|
||||
#: A trailing wildcard on any of these is a privilege escalation.
|
||||
EXEC_CAPABLE = ("iptables", "ip6tables", "nft", "tcpdump", "find", "awk",
|
||||
"sed", "perl", "python", "python3", "env")
|
||||
|
||||
|
||||
def _grant_lines():
|
||||
lines = []
|
||||
for installer in INSTALLERS:
|
||||
if not installer.is_file():
|
||||
continue
|
||||
for line in installer.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if "NOPASSWD:" in line:
|
||||
lines.append(line.split("NOPASSWD:", 1)[1])
|
||||
return lines
|
||||
|
||||
|
||||
def _normalise(rule):
|
||||
"""One rule with binary-path variables reduced to bare tool names.
|
||||
|
||||
Rules are written as `$SYSCTL_PATH -w ...` or `${NFT_PATH} ...`, so
|
||||
matching the literal "sysctl" finds nothing and every rule looks absent --
|
||||
which is exactly how an earlier version of this test reported six gaps
|
||||
that did not exist. Both spellings are handled: shell expands them
|
||||
identically, and a check that understood only one silently skipped the
|
||||
other.
|
||||
"""
|
||||
rule = re.sub(r"\$\{?([A-Z][A-Z0-9_]*)_PATH\}?",
|
||||
lambda m: m.group(1).lower(), rule)
|
||||
return re.sub(r"/usr/(?:s?bin)/", "", rule)
|
||||
|
||||
|
||||
def _granted_commands():
|
||||
"""The command each NOPASSWD rule actually grants, normalised.
|
||||
|
||||
_grant_lines() already returns everything after "NOPASSWD:", so what
|
||||
arrives here is the command, possibly preceded by the NOEXEC tag and
|
||||
possibly still carrying the closing quote of an `echo "..."` that wrote
|
||||
it. Both are stripped so the result is comparable to a plain command.
|
||||
"""
|
||||
commands = []
|
||||
for rule in _grant_lines():
|
||||
command = _normalise(rule).strip()
|
||||
command = re.sub(r"^NOEXEC:\s*", "", command)
|
||||
command = command.rstrip('"').rstrip("'").strip()
|
||||
if command:
|
||||
commands.append(" ".join(command.split()))
|
||||
return commands
|
||||
|
||||
|
||||
def test_the_installers_are_present():
|
||||
missing = [str(p.relative_to(ROOT)) for p in INSTALLERS if not p.is_file()]
|
||||
assert not missing, f"installer(s) missing: {missing}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("command", REQUIRED, ids=lambda c: " ".join(c))
|
||||
def test_the_command_is_granted(command):
|
||||
"""Whole command, not just the binary.
|
||||
|
||||
Checking only the binary made this far weaker than it looked: with
|
||||
`sysctl` present anywhere, deleting the ip_forward=0 grant still passed,
|
||||
and the portal would then be unable to restore forwarding on teardown.
|
||||
"""
|
||||
wanted = " ".join(command)
|
||||
granted = _granted_commands()
|
||||
# Exact match, not a prefix. A substring search was satisfied by
|
||||
# `sysctl -w net.ipv4.ip_forward=0 *`, and that trailing wildcard lets the
|
||||
# caller append whatever they like to a command running as root -- a far
|
||||
# wider grant than the one this test is meant to be confirming.
|
||||
assert wanted in granted, (
|
||||
f"no installer grants exactly `{wanted}`; closest matches: "
|
||||
+ str([g for g in granted if g.startswith(command[0])])[:200])
|
||||
|
||||
|
||||
def test_no_wildcard_on_a_tool_that_can_exec():
|
||||
"""`NOPASSWD: iptables *` hands the web user root.
|
||||
|
||||
iptables --modprobe=/path runs that path as root. This caught a grant added
|
||||
in this very change, which is why it is here.
|
||||
"""
|
||||
offenders = []
|
||||
for rule in _grant_lines():
|
||||
rule = rule.strip()
|
||||
if not rule.endswith("*"):
|
||||
continue
|
||||
# Normalised the same way as everything else: `${NFT_PATH} *` left a
|
||||
# brace before the tool name, and the word-boundary check below does
|
||||
# not treat "{" as a boundary, so that spelling slipped through.
|
||||
haystack = _normalise(rule).lower()
|
||||
for tool in EXEC_CAPABLE:
|
||||
if re.search(rf"(^|/|\s|\$){tool}(\s|$)", haystack):
|
||||
offenders.append(rule)
|
||||
break
|
||||
assert not offenders, (
|
||||
"wildcard grant on a tool that can execute another program:\n "
|
||||
+ "\n ".join(offenders))
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Wildcard grants to commands that start a pager must carry NOEXEC.
|
||||
|
||||
`journalctl` runs a pager when its output is a terminal, and from `less` a
|
||||
`!sh` is a shell with the privileges journalctl was given. That is the standard
|
||||
journalctl privilege escalation, and the installer's rules end in a wildcard:
|
||||
|
||||
<user> ALL=(ALL) NOPASSWD: /usr/bin/journalctl -u ledmatrix *
|
||||
|
||||
The web interface always passes --no-pager -- both call sites do, in app.py and
|
||||
api_v3.py -- so nothing the project runs needs the pager. But a sudoers rule
|
||||
cannot require a flag that sits in the middle of the command line, and reasoning
|
||||
about what a trailing `*` does or does not admit is exactly the kind of
|
||||
subtlety that produces a hole.
|
||||
|
||||
sudo's NOEXEC tag stops the command executing another program at all, which
|
||||
closes it without depending on that reasoning. It works by LD_PRELOAD, so it
|
||||
applies to dynamically linked binaries; journalctl is one.
|
||||
|
||||
On a stock Raspberry Pi image none of this is reachable, because
|
||||
/etc/sudoers.d/010_pi-nopasswd already grants the default user
|
||||
`ALL=(ALL) NOPASSWD: ALL`. It matters on a hardened install, or where the
|
||||
service runs as a user without that blanket rule.
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
INSTALLERS = (
|
||||
ROOT / "first_time_install.sh",
|
||||
ROOT / "scripts" / "install" / "configure_wifi_permissions.sh",
|
||||
# Writes the same journalctl grants as first_time_install.sh. It was
|
||||
# missing here, and because of that this suite passed while three
|
||||
# ungranted wildcard rules sat in it.
|
||||
ROOT / "scripts" / "install" / "configure_web_sudo.sh",
|
||||
)
|
||||
|
||||
#: Commands that will start another program of their own accord -- a pager, an
|
||||
#: editor, a shell -- and so must not be granted the ability to do so.
|
||||
SPAWNS_A_PROGRAM = ("journalctl", "systemctl", "less", "more", "man", "git")
|
||||
|
||||
|
||||
def _grant_lines():
|
||||
lines = []
|
||||
for installer in INSTALLERS:
|
||||
if not installer.is_file():
|
||||
continue
|
||||
for line in installer.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
stripped = line.strip()
|
||||
if "NOPASSWD" not in stripped or stripped.startswith("#"):
|
||||
continue
|
||||
# Installers emit rules two ways: written literally into a heredoc,
|
||||
# or echoed into a file. An echoed rule ends in a quote, so the
|
||||
# trailing-wildcard check below would skip it and the rule would
|
||||
# never be examined at all.
|
||||
echoed = re.fullmatch(r"""echo\s+(['"])(.*)\1""", stripped)
|
||||
lines.append(echoed.group(2) if echoed else stripped)
|
||||
return lines
|
||||
|
||||
|
||||
def test_the_installers_are_present():
|
||||
missing = [str(p.relative_to(ROOT)) for p in INSTALLERS if not p.is_file()]
|
||||
assert not missing, f"installer(s) missing: {missing}"
|
||||
|
||||
|
||||
def test_wildcard_pager_grants_carry_noexec():
|
||||
offenders = []
|
||||
for rule in _grant_lines():
|
||||
command = rule.split("NOPASSWD", 1)[1]
|
||||
if not command.rstrip().endswith("*"):
|
||||
continue
|
||||
tool = command.replace("_PATH", "").replace("$", "").lower()
|
||||
for name in SPAWNS_A_PROGRAM:
|
||||
if re.search(rf"(^|/|\s){name}(\s|$)", tool):
|
||||
if "NOEXEC" not in rule:
|
||||
offenders.append(rule)
|
||||
break
|
||||
assert not offenders, (
|
||||
"wildcard grant to a command that can start a pager or shell, without "
|
||||
"NOEXEC:\n " + "\n ".join(offenders))
|
||||
|
||||
|
||||
def test_journalctl_is_granted_at_all():
|
||||
"""Guard against 'fixing' the above by deleting the rules."""
|
||||
text = "\n".join(_grant_lines())
|
||||
assert "JOURNALCTL_PATH" in text or "journalctl" in text, (
|
||||
"no journalctl grant remains; the web interface reads logs through it")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("selector", ["-u ledmatrix.service", "-u ledmatrix",
|
||||
"-t ledmatrix"])
|
||||
def test_each_journalctl_rule_is_tagged(selector):
|
||||
"""Every selector, so removing one cannot pass by the others' presence."""
|
||||
matching = [r for r in _grant_lines()
|
||||
if "JOURNALCTL_PATH" in r and f"{selector} " in r]
|
||||
assert matching, f"no journalctl rule for {selector}"
|
||||
untagged = [r for r in matching if "NOEXEC" not in r]
|
||||
assert not untagged, f"untagged journalctl rule(s): {untagged}"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,92 @@
|
||||
"""/api/v3/system/status must report MemAvailable, not just used/total.
|
||||
|
||||
"Memory used %" cannot tell a healthy board from one about to fail. Page cache
|
||||
counts as used and is reclaimable on demand, so a Pi can read 70% used and be
|
||||
perfectly fine, or read the same and be minutes from trouble. MemAvailable is
|
||||
the kernel's own estimate of what a new allocation can actually obtain, and it
|
||||
is the number that tracked the failure on a 1GB Pi 3B+: healthy running sat at
|
||||
500MB+, the crash happened at 73MB, and by then fork() was failing -- sshd
|
||||
could not spawn a session and systemd could not respawn the display, while the
|
||||
kernel carried on answering pings.
|
||||
|
||||
psutil.virtual_memory().available is MemAvailable on Linux. total - used is not
|
||||
a substitute: they diverge exactly when unreclaimable memory (shmem, tmpfs) is
|
||||
in play, which is when the distinction matters.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
MB = 1024 * 1024
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
pytest.importorskip("psutil")
|
||||
app = Flask(__name__)
|
||||
app.config["TESTING"] = True
|
||||
from web_interface.blueprints.api_v3 import api_v3
|
||||
for attr in ("config_manager", "plugin_manager", "cache_manager"):
|
||||
setattr(api_v3, attr, MagicMock())
|
||||
if "api_v3" not in app.blueprints:
|
||||
app.register_blueprint(api_v3, url_prefix="/api/v3")
|
||||
return app.test_client()
|
||||
|
||||
|
||||
def _memory(total_mb, used_mb, available_mb):
|
||||
m = MagicMock()
|
||||
m.total = total_mb * MB
|
||||
m.used = used_mb * MB
|
||||
m.available = available_mb * MB
|
||||
m.percent = round(used_mb / total_mb * 100, 1)
|
||||
return m
|
||||
|
||||
|
||||
def _get_status(client, memory):
|
||||
# The endpoint caches for 10s; bypass so each case is measured fresh.
|
||||
with patch("web_interface.cache.get_cached", return_value=None), \
|
||||
patch("psutil.virtual_memory", return_value=memory), \
|
||||
patch("psutil.cpu_percent", return_value=5.0), \
|
||||
patch("psutil.boot_time", return_value=0.0):
|
||||
resp = client.get("/api/v3/system/status")
|
||||
assert resp.status_code == 200, resp.data
|
||||
return json.loads(resp.data)["data"]
|
||||
|
||||
|
||||
def test_available_memory_is_reported(client):
|
||||
data = _get_status(client, _memory(total_mb=905, used_mb=620, available_mb=284))
|
||||
assert "memory_available_mb" in data
|
||||
assert data["memory_available_mb"] == pytest.approx(284, abs=0.5)
|
||||
|
||||
|
||||
def test_available_is_not_total_minus_used(client):
|
||||
# The case the readout exists for: 600MB is "not used", but only 300MB can
|
||||
# actually be allocated. Reporting used% alone would call this healthy.
|
||||
data = _get_status(client, _memory(total_mb=1000, used_mb=400, available_mb=300))
|
||||
|
||||
derived = data["memory_total_mb"] - data["memory_used_mb"]
|
||||
assert derived == pytest.approx(600, abs=1)
|
||||
assert data["memory_available_mb"] == pytest.approx(300, abs=0.5)
|
||||
assert data["memory_available_mb"] != pytest.approx(derived, abs=1), \
|
||||
"available must come from MemAvailable, not be derived from used"
|
||||
|
||||
|
||||
def test_existing_memory_fields_are_unchanged(client):
|
||||
data = _get_status(client, _memory(total_mb=905, used_mb=620, available_mb=284))
|
||||
assert data["memory_total_mb"] == pytest.approx(905, abs=0.5)
|
||||
assert data["memory_used_mb"] == pytest.approx(620, abs=0.5)
|
||||
assert "memory_used_percent" in data
|
||||
|
||||
|
||||
def test_a_nearly_exhausted_board_reports_a_small_number(client):
|
||||
# 73MB available is what the board actually read when it stopped being able
|
||||
# to fork. The readout has to surface that rather than round it away.
|
||||
data = _get_status(client, _memory(total_mb=905, used_mb=800, available_mb=73))
|
||||
assert data["memory_available_mb"] == pytest.approx(73, abs=0.5)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""The display unit must cap glibc's malloc arenas.
|
||||
|
||||
glibc hands each allocating thread its own malloc arena, up to 8 x CPU count,
|
||||
and an arena that has grown is never returned to the OS. This process runs
|
||||
threads for the render loop, the update workers and the background fetchers, so
|
||||
on a 3-core Pi the ceiling is 24 arenas.
|
||||
|
||||
Measured on a live rig, 2.5 hours in:
|
||||
|
||||
RSS 1030 MB
|
||||
Private_Dirty 988 MB
|
||||
anonymous mappings > 10 MB 23 (ceiling is 8 x 3 = 24)
|
||||
largest few 104, 79, 66, 63, 63 MB, on 64 MB-aligned addresses
|
||||
|
||||
against live data that accounts for perhaps 15 MB -- the widest scroll strip
|
||||
observed was 35,746 x 64, about 7 MB as RGB and the same again for its numpy
|
||||
mirror. Repeated sampling showed RSS flat between 990 and 1030 MB rather than
|
||||
climbing, so this is arena bloat rather than a leak: memory Python has freed
|
||||
but glibc is holding per-arena.
|
||||
|
||||
The device had 59 MB free at the time.
|
||||
|
||||
Capping the arena count trades a little allocator concurrency for that resident
|
||||
memory. The render loop is latency-sensitive, so if p99 frame time regresses the
|
||||
right response is to raise this rather than remove it.
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
UNIT = (Path(__file__).resolve().parent.parent / "systemd" / "ledmatrix.service")
|
||||
|
||||
#: The value the unit is expected to carry. 2 is the usual choice for a
|
||||
#: threaded Python process; 1-4 all keep some of the saving, but only one of
|
||||
#: them is what this project ships.
|
||||
EXPECTED_ARENA_MAX = 2
|
||||
|
||||
|
||||
def _environment(unit_text):
|
||||
return dict(
|
||||
line.split("=", 2)[1:3] if line.count("=") >= 2 else (line.split("=", 1)[1], "")
|
||||
for line in unit_text.splitlines()
|
||||
if line.startswith("Environment=")
|
||||
)
|
||||
|
||||
|
||||
def test_the_unit_exists():
|
||||
assert UNIT.is_file(), f"{UNIT} is missing"
|
||||
|
||||
|
||||
def test_malloc_arena_max_is_capped():
|
||||
env = _environment(UNIT.read_text(encoding="utf-8"))
|
||||
assert "MALLOC_ARENA_MAX" in env, (
|
||||
"the display unit does not cap glibc arenas; on a 3-core Pi the default "
|
||||
"ceiling is 24 and a measured rig held 23 of them, 920 MB"
|
||||
)
|
||||
value = int(env["MALLOC_ARENA_MAX"])
|
||||
# Pinned, not a range. A range let a change to 4 -- which hands most of the
|
||||
# saving back -- pass unnoticed, which was the point of the finding that
|
||||
# prompted this. Raising it is a legitimate response to a frame-time
|
||||
# regression, but it should be a visible edit here rather than a silent
|
||||
# drift, so the number lives in one place and changing it shows up in
|
||||
# review.
|
||||
assert value == EXPECTED_ARENA_MAX, (
|
||||
f"MALLOC_ARENA_MAX={value}, expected {EXPECTED_ARENA_MAX}. If this was "
|
||||
"raised deliberately because frame times regressed, update "
|
||||
"EXPECTED_ARENA_MAX here and say so in the commit."
|
||||
)
|
||||
|
||||
|
||||
def test_the_reason_is_recorded_next_to_it():
|
||||
"""A bare tuning knob invites removal by whoever meets it next."""
|
||||
text = UNIT.read_text(encoding="utf-8")
|
||||
index = text.index("Environment=MALLOC_ARENA_MAX")
|
||||
preamble = text[:index].splitlines()[-12:]
|
||||
comment = "\n".join(line for line in preamble if line.startswith("#"))
|
||||
assert "arena" in comment.lower(), "no explanation precedes the setting"
|
||||
assert re.search(r"\d", comment), (
|
||||
"the explanation cites no measurement, so a reader cannot tell whether "
|
||||
"it still applies to their hardware"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("unit", ["ledmatrix.service"])
|
||||
def test_the_unit_still_parses_as_ini(unit):
|
||||
"""systemd will refuse a malformed unit, and the panel stays dark."""
|
||||
import configparser
|
||||
|
||||
path = UNIT.parent / unit
|
||||
parser = configparser.ConfigParser(strict=False)
|
||||
# systemd allows repeated keys; ConfigParser needs them merged, not rejected.
|
||||
parser.read_string(path.read_text(encoding="utf-8"))
|
||||
assert parser.has_section("Service")
|
||||
assert parser.has_option("Service", "ExecStart")
|
||||
@@ -0,0 +1,152 @@
|
||||
"""An installed unit that no longer matches the repo's must be reported.
|
||||
|
||||
Nothing re-applies systemd units after the first install. `git pull` -- what
|
||||
the web UI's update button runs -- brings a new template into the checkout, but
|
||||
no code in web_interface/ or src/ copies it to /etc/systemd/system or runs
|
||||
`systemctl daemon-reload`. The unit that actually runs is whatever
|
||||
first_time_install.sh wrote on day one.
|
||||
|
||||
So every hardening added to a unit is inert on existing installs. Measured on a
|
||||
live rig: the installed unit was dated 2026-08-06 and the repo's 2026-08-19,
|
||||
and they differed -- with the result that a MemoryMax=85% present in the repo's
|
||||
template was not being enforced at all. `systemctl show` reported
|
||||
MemoryMax=infinity.
|
||||
|
||||
This is a warning, not an error, and deliberately not a silent rewrite:
|
||||
editing files under /etc and restarting services is the installer's job, not
|
||||
something a display process should do to a machine while it boots.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from src.startup_validator import StartupValidator
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def validator():
|
||||
v = StartupValidator(config_manager=MagicMock())
|
||||
v.logger = logging.getLogger("test")
|
||||
v.warnings = []
|
||||
v.errors = []
|
||||
return v
|
||||
|
||||
|
||||
def test_a_matching_unit_produces_no_warning(validator, tmp_path):
|
||||
"""The installed unit, substituted exactly as the installer would."""
|
||||
project_root = Path("src/startup_validator.py").resolve().parent.parent
|
||||
template_rel = "systemd/ledmatrix.service"
|
||||
template = project_root / template_rel
|
||||
if not template.is_file():
|
||||
pytest.skip("repo unit template not present")
|
||||
|
||||
installed = tmp_path / "ledmatrix.service"
|
||||
installed.write_text(
|
||||
template.read_text(encoding="utf-8")
|
||||
.replace("__PROJECT_ROOT_DIR__", str(project_root))
|
||||
.replace("__USER__", "root"),
|
||||
encoding="utf-8")
|
||||
|
||||
validator._UNITS = ((template_rel, str(installed)),)
|
||||
validator._validate_systemd_units()
|
||||
assert not validator.warnings, f"a matching unit warned: {validator.warnings}"
|
||||
assert not validator.errors
|
||||
|
||||
|
||||
def test_comments_and_blank_lines_are_not_drift():
|
||||
"""Otherwise every comment the repo adds would look like a changed unit."""
|
||||
a = "[Service]\n# explains a setting\nExecStart=/x\nRestart=always\n"
|
||||
b = "[Service]\nExecStart=/x\n\nRestart=always\n"
|
||||
assert StartupValidator._unit_body(a) == StartupValidator._unit_body(b)
|
||||
|
||||
|
||||
def test_a_changed_directive_is_drift():
|
||||
a = "[Service]\nExecStart=/x\nMemoryMax=85%\n"
|
||||
b = "[Service]\nExecStart=/x\n"
|
||||
assert StartupValidator._unit_body(a) != StartupValidator._unit_body(b)
|
||||
|
||||
|
||||
def test_reordered_directives_are_drift():
|
||||
"""Order is not noise in a systemd unit.
|
||||
|
||||
Repeated directives -- ExecStartPre=, ExecStartPost= -- run in the order
|
||||
they appear, and a directive that moves between [Unit], [Service] and
|
||||
[Install] means something different, or nothing, where it lands. This
|
||||
check used to sort the lines before comparing, which reported no drift for
|
||||
a unit that had genuinely changed.
|
||||
"""
|
||||
a = "[Service]\nExecStartPre=/first\nExecStartPre=/second\n"
|
||||
b = "[Service]\nExecStartPre=/second\nExecStartPre=/first\n"
|
||||
assert StartupValidator._unit_body(a) != StartupValidator._unit_body(b), (
|
||||
"swapping two ExecStartPre= lines changes what runs first, and was "
|
||||
"being normalised away")
|
||||
|
||||
|
||||
def test_a_directive_moved_between_sections_is_drift():
|
||||
a = "[Unit]\nDescription=x\n[Service]\nExecStart=/x\n"
|
||||
b = "[Unit]\nDescription=x\nExecStart=/x\n[Service]\n"
|
||||
assert StartupValidator._unit_body(a) != StartupValidator._unit_body(b), (
|
||||
"ExecStart= in [Unit] is not the same unit, and sorting hid it")
|
||||
|
||||
|
||||
def test_cosmetic_differences_do_not_warn(validator, tmp_path):
|
||||
"""Through the real comparison, not the helper.
|
||||
|
||||
The repo's template carries explanatory comments the installed copy may not
|
||||
have, and the installer does not preserve ordering or blank lines. If those
|
||||
counted as drift, every boot would warn and the warning would be ignored.
|
||||
Asserting this on _unit_body alone would not catch a comparison that stopped
|
||||
calling it -- which is exactly what a careless edit does.
|
||||
"""
|
||||
project_root = Path("src/startup_validator.py").resolve().parent.parent
|
||||
template_rel = "systemd/ledmatrix.service"
|
||||
template = project_root / template_rel
|
||||
if not template.is_file():
|
||||
pytest.skip("repo unit template not present")
|
||||
|
||||
substituted = (template.read_text(encoding="utf-8")
|
||||
.replace("__PROJECT_ROOT_DIR__", str(project_root))
|
||||
.replace("__USER__", "root"))
|
||||
# Cosmetic means comments, blank lines and stray indentation -- the things
|
||||
# the installer really does drop. Not reordering: that changes the unit,
|
||||
# and is asserted as drift above.
|
||||
directives = [line.strip() for line in substituted.splitlines()
|
||||
if line.strip() and not line.strip().startswith("#")]
|
||||
installed = tmp_path / "ledmatrix.service"
|
||||
installed.write_text(
|
||||
"\n\n".join(" " + d for d in directives) + "\n", encoding="utf-8")
|
||||
|
||||
validator._UNITS = ((template_rel, str(installed)),)
|
||||
validator._validate_systemd_units()
|
||||
assert not validator.warnings, (
|
||||
f"cosmetic-only difference reported as drift: {validator.warnings}")
|
||||
|
||||
|
||||
def test_drift_is_reported_as_a_warning(validator, tmp_path):
|
||||
"""The whole point: a real difference must surface, and only as a warning."""
|
||||
installed = tmp_path / "ledmatrix.service"
|
||||
installed.write_text("[Service]\nExecStart=/usr/bin/python3 /x/run.py\n")
|
||||
|
||||
project_root = Path("src/startup_validator.py").resolve().parent.parent
|
||||
template_rel = "systemd/ledmatrix.service"
|
||||
template = project_root / template_rel
|
||||
if not template.is_file():
|
||||
pytest.skip("repo unit template not present")
|
||||
|
||||
validator._UNITS = ((template_rel, str(installed)),)
|
||||
validator._validate_systemd_units()
|
||||
|
||||
assert validator.warnings, "a differing unit produced no warning"
|
||||
assert "install_service.sh" in validator.warnings[0], (
|
||||
"the warning does not tell the user how to fix it")
|
||||
assert not validator.errors, "drift must not be fatal at startup"
|
||||
|
||||
|
||||
def test_a_missing_installed_unit_is_silent(validator, tmp_path):
|
||||
"""Development checkouts have no /etc/systemd unit; that is not drift."""
|
||||
validator._UNITS = (("systemd/ledmatrix.service", str(tmp_path / "absent.service")),)
|
||||
validator._validate_systemd_units()
|
||||
assert not validator.warnings
|
||||
assert not validator.errors
|
||||
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests that "which plugins have fresh data" survives the async update worker.
|
||||
|
||||
Regression under test: run_scheduled_updates_with_changes() snapshotted
|
||||
plugin_last_update, called run_scheduled_updates(), and diffed the two. But
|
||||
run_scheduled_updates() only *enqueues* -- the work runs on the update worker
|
||||
and stamps the timestamp there, after the method has already returned. The
|
||||
snapshots were therefore always identical and the result always empty.
|
||||
|
||||
Vegas depends on that result: it is what calls mark_plugin_updated(), which
|
||||
drops the cached content for a plugin whose data changed. With it always
|
||||
empty, a segment kept scrolling whatever it was first built from -- the
|
||||
"last night's live game still drawn as live the next morning" failure the
|
||||
coordinator comments describe. Observed on a live rig: zero update ticks in
|
||||
twenty minutes, with weather, stocks and news all updating.
|
||||
|
||||
Run: python -m pytest test/test_update_change_reporting.py -v
|
||||
"""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from src.plugin_system.plugin_manager import PluginManager # noqa: E402
|
||||
|
||||
|
||||
def _manager():
|
||||
"""A PluginManager with only the update-reporting state initialised."""
|
||||
manager = PluginManager.__new__(PluginManager)
|
||||
manager._completed_updates = set()
|
||||
manager._completed_updates_lock = threading.Lock()
|
||||
return manager
|
||||
|
||||
|
||||
class DrainCompletedUpdates(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.manager = _manager()
|
||||
|
||||
def test_nothing_completed_reports_nothing(self):
|
||||
self.assertEqual(self.manager.drain_completed_updates(), [])
|
||||
|
||||
def test_a_completed_update_is_reported(self):
|
||||
self.manager._note_update_completed("news")
|
||||
self.assertEqual(self.manager.drain_completed_updates(), ["news"])
|
||||
|
||||
def test_draining_clears_so_the_next_poll_is_empty(self):
|
||||
self.manager._note_update_completed("news")
|
||||
self.manager.drain_completed_updates()
|
||||
self.assertEqual(
|
||||
self.manager.drain_completed_updates(), [],
|
||||
"a plugin must be reported once per update, not on every poll, "
|
||||
"or Vegas would drop its cached content every few seconds")
|
||||
|
||||
def test_repeated_completions_between_polls_collapse(self):
|
||||
for _ in range(5):
|
||||
self.manager._note_update_completed("weather")
|
||||
self.assertEqual(self.manager.drain_completed_updates(), ["weather"])
|
||||
|
||||
def test_multiple_plugins_are_all_reported(self):
|
||||
for plugin_id in ("news", "weather", "ledmatrix-stocks"):
|
||||
self.manager._note_update_completed(plugin_id)
|
||||
self.assertEqual(self.manager.drain_completed_updates(),
|
||||
["ledmatrix-stocks", "news", "weather"])
|
||||
|
||||
|
||||
class CompletionReportingIsAsyncSafe(unittest.TestCase):
|
||||
"""The point of the change: completion may land after the call returns."""
|
||||
|
||||
def setUp(self):
|
||||
self.manager = _manager()
|
||||
|
||||
def test_an_update_completing_after_the_call_is_still_reported(self):
|
||||
"""The exact shape of the bug.
|
||||
|
||||
The enqueueing call sees nothing, because the worker has not run yet.
|
||||
The next poll must report it -- under the old diff it was lost, since
|
||||
the second snapshot was taken before the worker ever stamped.
|
||||
"""
|
||||
first = self.manager.drain_completed_updates()
|
||||
self.assertEqual(first, [], "nothing has finished yet")
|
||||
|
||||
# The worker finishes some time later, on its own thread.
|
||||
worker = threading.Thread(
|
||||
target=self.manager._note_update_completed, args=("news",))
|
||||
worker.start()
|
||||
worker.join()
|
||||
|
||||
self.assertEqual(
|
||||
self.manager.drain_completed_updates(), ["news"],
|
||||
"an update that finishes between polls must still be reported")
|
||||
|
||||
def test_concurrent_completions_are_not_lost(self):
|
||||
ids = ["plugin-%02d" % i for i in range(40)]
|
||||
threads = [threading.Thread(target=self.manager._note_update_completed,
|
||||
args=(pid,)) for pid in ids]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
self.assertEqual(self.manager.drain_completed_updates(), sorted(ids))
|
||||
|
||||
def test_a_completion_during_a_drain_is_not_swallowed(self):
|
||||
"""A drain must not clear an entry it did not report."""
|
||||
self.manager._note_update_completed("news")
|
||||
reported = self.manager.drain_completed_updates()
|
||||
# ...worker finishes another one immediately afterwards
|
||||
self.manager._note_update_completed("weather")
|
||||
self.assertEqual(reported, ["news"])
|
||||
self.assertEqual(self.manager.drain_completed_updates(), ["weather"])
|
||||
|
||||
|
||||
class EveryStampRecordsACompletion(unittest.TestCase):
|
||||
"""The ledger is only correct if the production paths actually fill it.
|
||||
|
||||
Asserting on the mechanics alone passes even when nothing calls
|
||||
_note_update_completed -- verified by deleting the call sites, which the
|
||||
behavioural tests above did not notice. This checks the invariant at the
|
||||
source: wherever a successful update stamps plugin_last_update, it must
|
||||
also record the completion, or Vegas silently stops being told.
|
||||
"""
|
||||
|
||||
def test_success_paths_record_the_completion(self):
|
||||
import src.plugin_system.plugin_manager as pm
|
||||
|
||||
tree = ast.parse(inspect.getsource(pm))
|
||||
stamps = []
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.With):
|
||||
continue
|
||||
# `with self._plugin_last_update_lock:` blocks that stamp a real
|
||||
# time on success. Two stamps are deliberately excluded: the 0.0
|
||||
# written at registration, and the failure path, which backs the
|
||||
# timestamp off to space out retries -- neither means fresh data.
|
||||
assigns_time = any(
|
||||
isinstance(stmt, ast.Assign)
|
||||
and any(isinstance(t, ast.Subscript)
|
||||
and getattr(t.value, "attr", None) == "plugin_last_update"
|
||||
for t in stmt.targets)
|
||||
and not (isinstance(stmt.value, ast.Constant)
|
||||
and stmt.value.value == 0.0)
|
||||
and "failure" not in ast.dump(stmt.value)
|
||||
for stmt in node.body
|
||||
)
|
||||
if assigns_time:
|
||||
stamps.append(node)
|
||||
|
||||
self.assertGreaterEqual(
|
||||
len(stamps), 2,
|
||||
"expected the worker and inline success paths to stamp the time; "
|
||||
"if this drops, the search below is looking at the wrong thing")
|
||||
|
||||
for stamp in stamps:
|
||||
enclosing = self._enclosing_function(tree, stamp)
|
||||
calls = [n for n in ast.walk(enclosing)
|
||||
if isinstance(n, ast.Call)
|
||||
and getattr(n.func, "attr", None) == "_note_update_completed"]
|
||||
self.assertTrue(
|
||||
calls,
|
||||
"%s stamps plugin_last_update on success but never calls "
|
||||
"_note_update_completed, so a plugin's fresh data would never "
|
||||
"be reported and Vegas would keep its stale cached content"
|
||||
% enclosing.name)
|
||||
|
||||
@staticmethod
|
||||
def _enclosing_function(tree, target):
|
||||
best = None
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
if node.lineno <= target.lineno <= (node.end_lineno or node.lineno):
|
||||
if best is None or node.lineno > best.lineno:
|
||||
best = node
|
||||
return best
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""A check that could not run must not be reported as "up to date".
|
||||
|
||||
check-update returned update_available=False whenever git failed. The banner
|
||||
is the only route to the update button, so a checkout git refuses to touch
|
||||
looked exactly like a current one -- permanently, and with nothing for the
|
||||
user to act on. The usual cause is an install performed as root, after which
|
||||
every git command fails with "detected dubious ownership".
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from web_interface.blueprints import api_v3 as mod # noqa: E402
|
||||
from web_interface.blueprints.api_v3 import api_v3 # noqa: E402
|
||||
|
||||
DUBIOUS = ("fatal: detected dubious ownership in repository at "
|
||||
"'/home/pi/LEDMatrix'\nTo add an exception for this directory, call:\n"
|
||||
"\tgit config --global --add safe.directory /home/pi/LEDMatrix\n")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
app.register_blueprint(api_v3, url_prefix='/api/v3')
|
||||
mod._update_check_cache['result'] = None
|
||||
mod._update_check_cache['ts'] = 0
|
||||
return app.test_client()
|
||||
|
||||
|
||||
def _fetch_fails(stderr: bytes):
|
||||
def fake_run(args, **kwargs):
|
||||
if args[:2] == ['git', 'fetch']:
|
||||
return subprocess.CompletedProcess(args, 1, stdout=b'', stderr=stderr)
|
||||
return subprocess.CompletedProcess(args, 0, stdout='', stderr='')
|
||||
return fake_run
|
||||
|
||||
|
||||
class TestFailedCheckIsNotSilence:
|
||||
def test_dubious_ownership_is_reported_not_swallowed(self, client):
|
||||
with patch.object(mod.subprocess, 'run', _fetch_fails(DUBIOUS.encode())):
|
||||
data = client.get('/api/v3/system/check-update').get_json()
|
||||
assert data['check_failed'] is True, (
|
||||
"a git failure was reported as a successful 'no update' check")
|
||||
assert data['update_available'] is False
|
||||
|
||||
def test_the_message_tells_the_user_what_to_do(self, client):
|
||||
with patch.object(mod.subprocess, 'run', _fetch_fails(DUBIOUS.encode())):
|
||||
data = client.get('/api/v3/system/check-update').get_json()
|
||||
assert 'chown' in data['error'], (
|
||||
"dubious ownership is unactionable without the fix command")
|
||||
assert 'root' in data['error']
|
||||
|
||||
def test_an_ordinary_git_failure_still_surfaces(self, client):
|
||||
with patch.object(mod.subprocess, 'run',
|
||||
_fetch_fails(b'fatal: some other git problem\n')):
|
||||
data = client.get('/api/v3/system/check-update').get_json()
|
||||
assert data['check_failed'] is True
|
||||
assert 'some other git problem' in data['error']
|
||||
|
||||
def test_offline_reads_as_offline(self, client):
|
||||
with patch.object(mod.subprocess, 'run',
|
||||
_fetch_fails(b'fatal: could not resolve host: github.com\n')):
|
||||
data = client.get('/api/v3/system/check-update').get_json()
|
||||
assert 'Could not reach GitHub' in data['error']
|
||||
|
||||
|
||||
class TestSuccessPathUnchanged:
|
||||
def test_up_to_date_carries_no_failure_flag(self, client):
|
||||
def fake_run(args, **kwargs):
|
||||
if args[:2] == ['git', 'fetch']:
|
||||
return subprocess.CompletedProcess(args, 0, stdout=b'', stderr=b'')
|
||||
if args[:2] == ['git', 'rev-parse']:
|
||||
return subprocess.CompletedProcess(args, 0, stdout='abc123\n', stderr='')
|
||||
return subprocess.CompletedProcess(args, 0, stdout='0\n', stderr='')
|
||||
with patch.object(mod.subprocess, 'run', fake_run):
|
||||
data = client.get('/api/v3/system/check-update').get_json()
|
||||
assert data['update_available'] is False
|
||||
assert not data.get('check_failed'), "a healthy check must not look like a failure"
|
||||
@@ -0,0 +1,84 @@
|
||||
"""A pull that changed nothing on the running system is not an applied update.
|
||||
|
||||
git_pull replaces files on disk and restarts nothing -- there is no systemctl
|
||||
call anywhere in the handler. The display and web services keep running the
|
||||
code they loaded at boot, so the user is told "Code updated successfully" and
|
||||
sees no change until they happen to reboot. The response now says whether a
|
||||
restart is owed, and the UI raises the existing restart-pending banner.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from web_interface.blueprints import api_v3 as mod # noqa: E402
|
||||
from web_interface.blueprints.api_v3 import api_v3 # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
app = Flask(__name__)
|
||||
app.config['TESTING'] = True
|
||||
app.register_blueprint(api_v3, url_prefix='/api/v3')
|
||||
# The handler consults these after a successful pull; None is the
|
||||
# "not wired up" case it already guards for.
|
||||
api_v3.plugin_store_manager = None
|
||||
api_v3.config_manager = None
|
||||
return app.test_client()
|
||||
|
||||
|
||||
def _git(heads, pull_rc=0, pull_out='Updating a1b2c3..d4e5f6\n'):
|
||||
"""Fake git. `heads` are the successive answers to rev-parse HEAD."""
|
||||
seq = list(heads)
|
||||
|
||||
def run(args, **kwargs):
|
||||
def ok(stdout='', rc=0, b=False):
|
||||
return subprocess.CompletedProcess(
|
||||
args, rc, stdout=(stdout.encode() if b else stdout),
|
||||
stderr=(b'' if b else ''))
|
||||
if args[:2] == ['git', 'rev-parse'] and args[-1] == 'HEAD':
|
||||
return ok(seq.pop(0) + '\n' if seq else 'deadbeef\n')
|
||||
if 'symbolic-full-name' in args or '@{u}' in args:
|
||||
return ok('origin/main\n')
|
||||
if args[:2] == ['git', 'status']:
|
||||
return ok('')
|
||||
if args[:2] == ['git', 'diff']:
|
||||
return ok('')
|
||||
if args[:2] == ['git', 'pull']:
|
||||
return ok(pull_out, pull_rc)
|
||||
return ok('')
|
||||
return run
|
||||
|
||||
|
||||
def _pull(client):
|
||||
return client.post('/api/v3/system/action',
|
||||
json={'action': 'git_pull'}).get_json()
|
||||
|
||||
|
||||
class TestRestartIsRequestedWhenCodeChanged:
|
||||
def test_a_pull_that_moved_head_asks_for_a_restart(self, client):
|
||||
with patch.object(mod.subprocess, 'run', _git(['aaa111', 'bbb222'])):
|
||||
data = _pull(client)
|
||||
assert data['status'] == 'success'
|
||||
assert data['restart_required'] is True, (
|
||||
"new code on disk, services still running the old code, and "
|
||||
"nothing told the user to restart")
|
||||
|
||||
def test_already_up_to_date_does_not(self, client):
|
||||
with patch.object(mod.subprocess, 'run',
|
||||
_git(['aaa111', 'aaa111'], pull_out='Already up to date.\n')):
|
||||
data = _pull(client)
|
||||
assert data['status'] == 'success'
|
||||
assert data['restart_required'] is False, (
|
||||
"prompting after a no-op update trains users to ignore the prompt")
|
||||
|
||||
def test_a_failed_pull_does_not(self, client):
|
||||
with patch.object(mod.subprocess, 'run', _git(['aaa111'], pull_rc=1)):
|
||||
data = _pull(client)
|
||||
assert data['status'] == 'error'
|
||||
assert data['restart_required'] is False
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user