mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-10-19 05:09:14 +00:00
Compare commits
2
Commits
116abb0daa
...
19686ab761
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
19686ab761 | ||
|
|
92f1960d00 |
@@ -40,6 +40,14 @@ Config saves and plugin config preparation:
|
|||||||
the object, posting it back saves, and hot reload hands plugins the same
|
the object, posting it back saves, and hot reload hands plugins the same
|
||||||
shape (schema defaults included) they were constructed with.
|
shape (schema defaults included) they were constructed with.
|
||||||
`schema_manager.prepare_plugin_config` is the one implementation.
|
`schema_manager.prepare_plugin_config` is the one implementation.
|
||||||
|
- A plugin's settings tab shows schema defaults for options its saved config
|
||||||
|
doesn't have yet. A boolean added with `"default": true` in a plugin update
|
||||||
|
(geochron 1.2.0's `show_date` and `show_date_line`) used to render unchecked,
|
||||||
|
and the next save of that tab stored it as `false`. Enum dropdowns likewise
|
||||||
|
showed their first option instead of the default. The partial now runs the
|
||||||
|
stored section through `prepare_plugin_config` like `GET /plugins/config`
|
||||||
|
(secrets are still masked, after the merge), and the form falls back to a
|
||||||
|
field's own `default` inside objects that declare a default of their own.
|
||||||
- `scripts/dev_server.py`, `check_plugin.py`, `render_plugin.py` and the plugin
|
- `scripts/dev_server.py`, `check_plugin.py`, `render_plugin.py` and the plugin
|
||||||
harness build configs the way a device does: nested defaults are included,
|
harness build configs the way a device does: nested defaults are included,
|
||||||
a schema `enabled: false` no longer beats the forced `enabled: true` in the
|
a schema `enabled: false` no longer beats the forced `enabled: true` in the
|
||||||
|
|||||||
+101
-21
@@ -34,7 +34,9 @@ workaround retires itself if ESPN reverts.
|
|||||||
|
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from datetime import date, timedelta
|
from datetime import date, timedelta
|
||||||
|
from functools import partial
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
# Above this, ESPN returns a truncated list instead of an error. See module
|
# Above this, ESPN returns a truncated list instead of an error. See module
|
||||||
@@ -44,11 +46,18 @@ ESPN_MAX_LIMIT = 500
|
|||||||
# How long a rejected range keeps later ranges from being tried as ranges.
|
# How long a rejected range keeps later ranges from being tried as ranges.
|
||||||
RANGE_RETRY_SECONDS = 6 * 60 * 60
|
RANGE_RETRY_SECONDS = 6 * 60 * 60
|
||||||
|
|
||||||
|
# How many chunk requests may be in flight at once. Four busy months of
|
||||||
|
# college baseball are ~130 chunks once each is re-asked day by day: 17.7s one
|
||||||
|
# at a time on a Pi 4, 2.6-3.3s six at a time. Kept under requests' default
|
||||||
|
# pool_maxsize of 10 so the shared Session never has to discard connections.
|
||||||
|
ESPN_CHUNK_WORKERS = 6
|
||||||
|
|
||||||
_range_lock = threading.Lock()
|
_range_lock = threading.Lock()
|
||||||
_ranges_rejected_until = 0.0
|
_ranges_rejected_until = 0.0
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"ESPN_MAX_LIMIT",
|
"ESPN_MAX_LIMIT",
|
||||||
|
"ESPN_CHUNK_WORKERS",
|
||||||
"RANGE_RETRY_SECONDS",
|
"RANGE_RETRY_SECONDS",
|
||||||
"clamp_espn_limit",
|
"clamp_espn_limit",
|
||||||
"parse_espn_date_range",
|
"parse_espn_date_range",
|
||||||
@@ -169,6 +178,55 @@ def merge_scoreboard_payloads(payloads: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|||||||
return merged
|
return merged
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_one_chunk(
|
||||||
|
session, url: str, params: Dict[str, Any], headers, timeout, logger, chunk: str,
|
||||||
|
) -> Optional[Dict[str, Any]]:
|
||||||
|
"""GET a single ``dates=`` chunk, or None when it failed.
|
||||||
|
|
||||||
|
One bad chunk must not sink the rest of the season, so every error is
|
||||||
|
logged and swallowed here rather than raised to the gather below.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
response = session.get(
|
||||||
|
url,
|
||||||
|
params=dict(params, dates=chunk, limit=ESPN_MAX_LIMIT),
|
||||||
|
headers=headers,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()
|
||||||
|
except Exception as exc: # noqa: BLE001 - see docstring
|
||||||
|
if logger:
|
||||||
|
logger.warning("ESPN chunk %s failed, skipping it: %s", chunk, exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_chunks(
|
||||||
|
session, url: str, params: Dict[str, Any], headers, timeout, logger,
|
||||||
|
chunks: List[str],
|
||||||
|
) -> List[Optional[Dict[str, Any]]]:
|
||||||
|
"""Fetch every chunk, returning payloads positionally aligned with ``chunks``.
|
||||||
|
|
||||||
|
Requests go out ``ESPN_CHUNK_WORKERS`` at a time because a cold season is
|
||||||
|
over a hundred of them. The order they come back in is not significant --
|
||||||
|
callers keep ``chunks`` order from the returned list -- but it does mean
|
||||||
|
the session is shared across threads, which is why this only ever issues
|
||||||
|
GETs and never touches session state.
|
||||||
|
"""
|
||||||
|
if not chunks:
|
||||||
|
return []
|
||||||
|
fetch = partial(
|
||||||
|
_fetch_one_chunk, session, url, params, headers, timeout, logger,
|
||||||
|
)
|
||||||
|
if len(chunks) == 1:
|
||||||
|
return [fetch(chunks[0])]
|
||||||
|
workers = min(ESPN_CHUNK_WORKERS, len(chunks))
|
||||||
|
with ThreadPoolExecutor(
|
||||||
|
max_workers=workers, thread_name_prefix="espn-chunk",
|
||||||
|
) as pool:
|
||||||
|
return list(pool.map(fetch, chunks))
|
||||||
|
|
||||||
|
|
||||||
def fetch_espn_date_chunks(
|
def fetch_espn_date_chunks(
|
||||||
session,
|
session,
|
||||||
url: str,
|
url: str,
|
||||||
@@ -188,6 +246,11 @@ def fetch_espn_date_chunks(
|
|||||||
largest value that does not corrupt the answer. A month that comes back
|
largest value that does not corrupt the answer. A month that comes back
|
||||||
with 500 events is assumed truncated and re-asked day by day. A failed
|
with 500 events is assumed truncated and re-asked day by day. A failed
|
||||||
chunk is logged and skipped so one bad day cannot cost a whole season.
|
chunk is logged and skipped so one bad day cannot cost a whole season.
|
||||||
|
|
||||||
|
Chunks go out ``ESPN_CHUNK_WORKERS`` at a time, in two passes: the months
|
||||||
|
and edge days first, then the days of any month that came back capped.
|
||||||
|
Merged events keep ``espn_date_chunks`` order regardless of which request
|
||||||
|
finished first, so the result does not depend on the race.
|
||||||
"""
|
"""
|
||||||
params = dict(params or {})
|
params = dict(params or {})
|
||||||
span = parse_espn_date_range(params.get("dates"))
|
span = parse_espn_date_range(params.get("dates"))
|
||||||
@@ -201,35 +264,52 @@ def fetch_espn_date_chunks(
|
|||||||
params.get("dates"), len(chunks),
|
params.get("dates"), len(chunks),
|
||||||
)
|
)
|
||||||
|
|
||||||
payloads: List[Dict[str, Any]] = []
|
results = _fetch_chunks(
|
||||||
attempted = 0
|
session, url, params, headers, timeout, logger, chunks,
|
||||||
pending = list(chunks)
|
)
|
||||||
while pending:
|
attempted = len(chunks)
|
||||||
chunk = pending.pop(0)
|
|
||||||
attempted += 1
|
# A month that came back at the cap is truncated; its days replace it in
|
||||||
try:
|
# place, so merged events stay in chunk order however the requests raced.
|
||||||
response = session.get(
|
slots: List[Any] = results
|
||||||
url,
|
capped: Dict[int, List[str]] = {}
|
||||||
params=dict(params, dates=chunk, limit=ESPN_MAX_LIMIT),
|
for index, chunk in enumerate(chunks):
|
||||||
headers=headers,
|
payload = slots[index]
|
||||||
timeout=timeout,
|
if payload is None or len(chunk) != 6:
|
||||||
)
|
|
||||||
response.raise_for_status()
|
|
||||||
payload = response.json()
|
|
||||||
except Exception as exc: # noqa: BLE001 - one bad chunk must not sink the rest
|
|
||||||
if logger:
|
|
||||||
logger.warning("ESPN chunk %s failed, skipping it: %s", chunk, exc)
|
|
||||||
continue
|
continue
|
||||||
events = payload.get("events") if isinstance(payload, dict) else None
|
events = payload.get("events") if isinstance(payload, dict) else None
|
||||||
if len(chunk) == 6 and len(events or []) >= ESPN_MAX_LIMIT:
|
if len(events or []) >= ESPN_MAX_LIMIT:
|
||||||
if logger:
|
if logger:
|
||||||
logger.info(
|
logger.info(
|
||||||
"ESPN month %s hit the %d-event cap; re-asking it day by day",
|
"ESPN month %s hit the %d-event cap; re-asking it day by day",
|
||||||
chunk, ESPN_MAX_LIMIT,
|
chunk, ESPN_MAX_LIMIT,
|
||||||
)
|
)
|
||||||
pending[:0] = _days_of_month(chunk)
|
capped[index] = _days_of_month(chunk)
|
||||||
|
# Drop the truncated month now rather than after its days arrive:
|
||||||
|
# a capped college-baseball month is ~2MB of parsed JSON, and
|
||||||
|
# holding four of them through ~120 day requests added ~25MB to
|
||||||
|
# the peak -- more than the concurrency itself. Low-memory boards
|
||||||
|
# (docs/LOW_MEMORY_BOARDS.md) have under 200MB of headroom.
|
||||||
|
slots[index] = None
|
||||||
|
payload = events = None
|
||||||
|
|
||||||
|
if capped:
|
||||||
|
days = [day for index in sorted(capped) for day in capped[index]]
|
||||||
|
attempted += len(days)
|
||||||
|
by_day = dict(zip(days, _fetch_chunks(
|
||||||
|
session, url, params, headers, timeout, logger, days,
|
||||||
|
)))
|
||||||
|
for index, month_days in capped.items():
|
||||||
|
slots[index] = [by_day.get(day) for day in month_days]
|
||||||
|
|
||||||
|
payloads: List[Dict[str, Any]] = []
|
||||||
|
for slot in slots:
|
||||||
|
if slot is None:
|
||||||
continue
|
continue
|
||||||
payloads.append(payload)
|
if isinstance(slot, list):
|
||||||
|
payloads.extend(payload for payload in slot if payload is not None)
|
||||||
|
else:
|
||||||
|
payloads.append(slot)
|
||||||
|
|
||||||
if not payloads:
|
if not payloads:
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -115,8 +115,11 @@ def test_a_rejected_season_is_recovered_and_cached(service, cache):
|
|||||||
def test_a_full_season_costs_chunks_not_one_request_per_day(service):
|
def test_a_full_season_costs_chunks_not_one_request_per_day(service):
|
||||||
session = RangeRejectingSession({"202609": [{"id": "a"}]})
|
session = RangeRejectingSession({"202609": [{"id": "a"}]})
|
||||||
submit_and_wait(service, session, "20260801-20270301")
|
submit_and_wait(service, session, "20260801-20270301")
|
||||||
assert [call["dates"] for call in session.calls] == [
|
sent = [call["dates"] for call in session.calls]
|
||||||
"20260801-20270301",
|
# Eight chunks rather than 213 per-day requests. They are fetched
|
||||||
|
# concurrently, so the range is the only one pinned to a position.
|
||||||
|
assert sent[0] == "20260801-20270301"
|
||||||
|
assert sorted(sent[1:]) == [
|
||||||
"202608",
|
"202608",
|
||||||
"202609",
|
"202609",
|
||||||
"202610",
|
"202610",
|
||||||
|
|||||||
+123
-4
@@ -12,6 +12,8 @@ Nothing here touches the network. The fake session records what a caller would
|
|||||||
have sent, which is the part that regressed.
|
have sent, which is the part that regressed.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
from datetime import date, timedelta
|
from datetime import date, timedelta
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -245,7 +247,10 @@ class TestFetch:
|
|||||||
)
|
)
|
||||||
assert [e["id"] for e in data["events"]] == ["a", "b", "c"]
|
assert [e["id"] for e in data["events"]] == ["a", "b", "c"]
|
||||||
sent = [call["dates"] for call in session.calls]
|
sent = [call["dates"] for call in session.calls]
|
||||||
assert sent == ["20260901-20261001", "202609", "20261001"]
|
# Chunks race, so only the rejected range is pinned to a position --
|
||||||
|
# the merged event order above is what has to stay deterministic.
|
||||||
|
assert sent[0] == "20260901-20261001"
|
||||||
|
assert sorted(sent[1:]) == ["202609", "20261001"]
|
||||||
|
|
||||||
def test_chunk_requests_keep_the_clamped_limit(self):
|
def test_chunk_requests_keep_the_clamped_limit(self):
|
||||||
session = FakeSession({"202609": []})
|
session = FakeSession({"202609": []})
|
||||||
@@ -301,9 +306,11 @@ class TestMonthCap:
|
|||||||
|
|
||||||
data = fetch_espn_date_chunks(session, URL, params={"dates": "20260301-20260331"})
|
data = fetch_espn_date_chunks(session, URL, params={"dates": "20260301-20260331"})
|
||||||
|
|
||||||
assert [call["dates"] for call in session.calls] == ["202603"] + [
|
sent = [call["dates"] for call in session.calls]
|
||||||
"202603%02d" % day for day in range(1, 32)
|
# The month has to be asked before its days can be known to be needed;
|
||||||
]
|
# the days themselves race, so compare them as a set.
|
||||||
|
assert sent[0] == "202603"
|
||||||
|
assert sorted(sent[1:]) == ["202603%02d" % day for day in range(1, 32)]
|
||||||
# The truncated month payload is dropped, not merged with the days.
|
# The truncated month payload is dropped, not merged with the days.
|
||||||
assert [event["id"] for event in data["events"]] == [
|
assert [event["id"] for event in data["events"]] == [
|
||||||
"d%d" % day for day in range(1, 32)
|
"d%d" % day for day in range(1, 32)
|
||||||
@@ -370,3 +377,115 @@ class TestRejectedRangeMemo:
|
|||||||
with pytest.raises(RuntimeError):
|
with pytest.raises(RuntimeError):
|
||||||
fetch_espn_scoreboard(session, URL, params={"dates": "20260914-20260915"})
|
fetch_espn_scoreboard(session, URL, params={"dates": "20260914-20260915"})
|
||||||
assert session.calls[-1]["dates"] == "20260914-20260915"
|
assert session.calls[-1]["dates"] == "20260914-20260915"
|
||||||
|
|
||||||
|
|
||||||
|
class TestConcurrency:
|
||||||
|
"""Chunks go out in parallel, which must not change what comes back.
|
||||||
|
|
||||||
|
A cold college-baseball season is ~130 chunks once February through May
|
||||||
|
are re-asked day by day. Sequentially that outran the 20s plugin update()
|
||||||
|
timeout on a Pi, so the requests now overlap -- but the merged payload has
|
||||||
|
to stay exactly what the sequential version produced.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_events_keep_chunk_order_however_the_requests_race(self):
|
||||||
|
# Answer the later chunks fastest, so completion order is the reverse
|
||||||
|
# of chunk order and a naive gather would interleave them wrongly.
|
||||||
|
class RacingSession(FakeSession):
|
||||||
|
def get(self, url, params=None, headers=None, timeout=None):
|
||||||
|
dates = str((params or {}).get("dates", ""))
|
||||||
|
if len(dates) == 6:
|
||||||
|
time.sleep(0.02 / (int(dates[4:]) or 1))
|
||||||
|
return super().get(url, params=params, headers=headers, timeout=timeout)
|
||||||
|
|
||||||
|
session = RacingSession(
|
||||||
|
{
|
||||||
|
"202609": [{"id": "sep"}],
|
||||||
|
"202610": [{"id": "oct"}],
|
||||||
|
"202611": [{"id": "nov"}],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
data = fetch_espn_date_chunks(
|
||||||
|
session, URL, params={"dates": "20260901-20261130"}
|
||||||
|
)
|
||||||
|
assert [event["id"] for event in data["events"]] == ["sep", "oct", "nov"]
|
||||||
|
|
||||||
|
def test_a_capped_month_splices_its_days_in_place(self):
|
||||||
|
# October is capped and expands to 31 days; September and November
|
||||||
|
# must still bracket those days in the merged result.
|
||||||
|
full = [{"id": "cap%d" % i} for i in range(ESPN_MAX_LIMIT)]
|
||||||
|
by_chunk = {
|
||||||
|
"202609": [{"id": "sep"}],
|
||||||
|
"202610": full,
|
||||||
|
"202611": [{"id": "nov"}],
|
||||||
|
}
|
||||||
|
by_chunk.update(
|
||||||
|
{"202610%02d" % day: [{"id": "oct%02d" % day}] for day in range(1, 32)}
|
||||||
|
)
|
||||||
|
session = FakeSession(by_chunk)
|
||||||
|
|
||||||
|
data = fetch_espn_date_chunks(
|
||||||
|
session, URL, params={"dates": "20260901-20261130"}
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = ["sep"] + ["oct%02d" % day for day in range(1, 32)] + ["nov"]
|
||||||
|
assert [event["id"] for event in data["events"]] == expected
|
||||||
|
|
||||||
|
def test_two_capped_months_expand_without_crossing_over(self):
|
||||||
|
full = [{"id": "cap%d" % i} for i in range(ESPN_MAX_LIMIT)]
|
||||||
|
by_chunk = {"202609": full, "202610": full}
|
||||||
|
by_chunk.update(
|
||||||
|
{"202609%02d" % day: [{"id": "s%02d" % day}] for day in range(1, 31)}
|
||||||
|
)
|
||||||
|
by_chunk.update(
|
||||||
|
{"202610%02d" % day: [{"id": "o%02d" % day}] for day in range(1, 32)}
|
||||||
|
)
|
||||||
|
session = FakeSession(by_chunk)
|
||||||
|
|
||||||
|
data = fetch_espn_date_chunks(
|
||||||
|
session, URL, params={"dates": "20260901-20261031"}
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = ["s%02d" % day for day in range(1, 31)] + [
|
||||||
|
"o%02d" % day for day in range(1, 32)
|
||||||
|
]
|
||||||
|
assert [event["id"] for event in data["events"]] == expected
|
||||||
|
|
||||||
|
def test_a_failed_day_inside_a_capped_month_only_costs_that_day(self):
|
||||||
|
full = [{"id": "cap%d" % i} for i in range(ESPN_MAX_LIMIT)]
|
||||||
|
by_chunk = {"202610": full}
|
||||||
|
by_chunk.update(
|
||||||
|
{"202610%02d" % day: [{"id": "o%02d" % day}] for day in range(1, 32)}
|
||||||
|
)
|
||||||
|
session = FakeSession(by_chunk, fail_chunks={"20261015"})
|
||||||
|
|
||||||
|
data = fetch_espn_date_chunks(
|
||||||
|
session, URL, params={"dates": "20261001-20261031"}
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = ["o%02d" % day for day in range(1, 32) if day != 15]
|
||||||
|
assert [event["id"] for event in data["events"]] == expected
|
||||||
|
|
||||||
|
def test_no_more_than_the_worker_cap_are_in_flight_at_once(self):
|
||||||
|
live = {"now": 0, "peak": 0}
|
||||||
|
guard = threading.Lock()
|
||||||
|
|
||||||
|
class CountingSession(FakeSession):
|
||||||
|
def get(self, url, params=None, headers=None, timeout=None):
|
||||||
|
with guard:
|
||||||
|
live["now"] += 1
|
||||||
|
live["peak"] = max(live["peak"], live["now"])
|
||||||
|
try:
|
||||||
|
time.sleep(0.01)
|
||||||
|
return super().get(
|
||||||
|
url, params=params, headers=headers, timeout=timeout
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
with guard:
|
||||||
|
live["now"] -= 1
|
||||||
|
|
||||||
|
session = CountingSession()
|
||||||
|
fetch_espn_date_chunks(session, URL, params={"dates": "20260101-20261231"})
|
||||||
|
|
||||||
|
assert live["peak"] <= espn_dates.ESPN_CHUNK_WORKERS
|
||||||
|
assert live["peak"] > 1, "chunks should actually overlap"
|
||||||
|
|||||||
@@ -30,6 +30,8 @@ from src.plugin_system.plugin_manager import PluginManager
|
|||||||
from src.plugin_system.schema_manager import (
|
from src.plugin_system.schema_manager import (
|
||||||
legacy_bool_as_object,
|
legacy_bool_as_object,
|
||||||
normalize_legacy_booleans,
|
normalize_legacy_booleans,
|
||||||
|
plugin_config_defaults,
|
||||||
|
prepare_plugin_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -361,26 +363,33 @@ def _form_checkboxes(stored):
|
|||||||
return parser.checked
|
return parser.checked
|
||||||
|
|
||||||
|
|
||||||
def _loader_enabled(normalized, *path):
|
def _loader_enabled(prepared, *path):
|
||||||
node = normalized
|
node = prepared
|
||||||
for key in path:
|
for key in path:
|
||||||
node = node.get(key) if isinstance(node, dict) else None
|
node = node.get(key) if isinstance(node, dict) else None
|
||||||
return isinstance(node, dict) and node.get("enabled") is True
|
if not isinstance(node, dict):
|
||||||
|
# A value the loader cannot read as the object (1, "true") fails
|
||||||
|
# validation; the form treats it as missing and draws the schema
|
||||||
|
# default, as it does for every other missing field.
|
||||||
|
return True
|
||||||
|
return node.get("enabled") is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("value", [True, False, 1, "true", None])
|
@pytest.mark.parametrize("value", [True, False, 1, "true", None])
|
||||||
def test_form_and_loader_agree_on_what_a_stored_value_means(value):
|
def test_form_and_loader_agree_on_what_a_stored_value_means(value):
|
||||||
"""Where the loader reads a stored value as ``{"enabled": true}``, the form
|
"""The form draws the object's ``enabled`` checkbox ticked exactly when the
|
||||||
draws the object's ``enabled`` checkbox ticked, and nowhere else."""
|
plugin runs with it on -- legacy booleans read as objects, then schema
|
||||||
|
defaults filled in (prepare_plugin_config)."""
|
||||||
stored = {"enabled": True,
|
stored = {"enabled": True,
|
||||||
"global": {"dynamic_duration": value, "outer": {"inner": value}}}
|
"global": {"dynamic_duration": value, "outer": {"inner": value}}}
|
||||||
normalized = normalize_legacy_booleans(stored, PARITY_SCHEMA)
|
prepared = prepare_plugin_config(stored, PARITY_SCHEMA,
|
||||||
|
plugin_config_defaults(PARITY_SCHEMA))
|
||||||
boxes = _form_checkboxes(stored)
|
boxes = _form_checkboxes(stored)
|
||||||
|
|
||||||
for path in (("global", "dynamic_duration"), ("global", "outer", "inner")):
|
for path in (("global", "dynamic_duration"), ("global", "outer", "inner")):
|
||||||
name = ".".join(path) + ".enabled"
|
name = ".".join(path) + ".enabled"
|
||||||
assert name in boxes, f"form drew no {name} checkbox"
|
assert name in boxes, f"form drew no {name} checkbox"
|
||||||
assert boxes[name] is _loader_enabled(normalized, *path), name
|
assert boxes[name] is _loader_enabled(prepared, *path), name
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("value", [True, False])
|
@pytest.mark.parametrize("value", [True, False])
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
"""The server-rendered plugin form must show schema defaults for unsaved keys.
|
||||||
|
|
||||||
|
A plugin update that adds a boolean option with ``"default": true`` (geochron
|
||||||
|
1.2.0's ``show_date`` / ``show_date_line``) leaves every existing install with
|
||||||
|
a saved config that lacks the key. The partial rendered it from the raw config,
|
||||||
|
so the box came up unchecked -- and the save route treats a drawn but unposted
|
||||||
|
checkbox as false, so the first save turned the option off for good.
|
||||||
|
|
||||||
|
These drive the real partial route (pages_v3) and post what a browser would
|
||||||
|
submit from its HTML back through the real save route (api_v3).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from flask import Flask
|
||||||
|
from werkzeug.datastructures import MultiDict
|
||||||
|
|
||||||
|
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
||||||
|
sys.path.insert(0, str(PROJECT_ROOT))
|
||||||
|
|
||||||
|
SCHEMA = {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"enabled": {"type": "boolean", "default": True},
|
||||||
|
"show_date": {"type": "boolean", "default": True},
|
||||||
|
"show_date_line": {"type": "boolean", "default": True},
|
||||||
|
"show_seconds": {"type": "boolean", "default": False},
|
||||||
|
"style": {"type": "string", "enum": ["flat", "globe", "night"],
|
||||||
|
"default": "globe"},
|
||||||
|
"brightness": {"type": "integer", "default": 70,
|
||||||
|
"minimum": 0, "maximum": 100},
|
||||||
|
"label": {"type": "string", "default": "UTC"},
|
||||||
|
# An object with its own default: extract_schema_defaults stops here,
|
||||||
|
# so the route merge leaves the children missing.
|
||||||
|
"overlay": {"type": "object", "default": {}, "properties": {
|
||||||
|
"show_sun": {"type": "boolean", "default": True},
|
||||||
|
"mode": {"type": "string", "enum": ["dot", "ring"],
|
||||||
|
"default": "ring"},
|
||||||
|
}, "additionalProperties": False},
|
||||||
|
# An object without one: the route merge fills the children in.
|
||||||
|
"grid": {"type": "object", "properties": {
|
||||||
|
"show_lines": {"type": "boolean", "default": True},
|
||||||
|
}, "additionalProperties": False},
|
||||||
|
"api_key": {"type": "string", "x-secret": True,
|
||||||
|
"default": "not-a-real-secret"},
|
||||||
|
},
|
||||||
|
"required": ["enabled"],
|
||||||
|
"additionalProperties": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Saved before any of the options above existed.
|
||||||
|
STORED = {"enabled": True, "show_seconds": True}
|
||||||
|
|
||||||
|
|
||||||
|
class _FormFields(HTMLParser):
|
||||||
|
"""Collect what a browser would submit from the rendered form."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.pairs = []
|
||||||
|
self._select = None
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
a = dict(attrs)
|
||||||
|
if tag == "select":
|
||||||
|
self._select = a.get("name")
|
||||||
|
return
|
||||||
|
if tag == "option" and self._select and "selected" in a:
|
||||||
|
self.pairs.append((self._select, a.get("value", "")))
|
||||||
|
return
|
||||||
|
if tag != "input" or not a.get("name") or "disabled" in a:
|
||||||
|
return
|
||||||
|
if a.get("type") in ("checkbox", "radio") and "checked" not in a:
|
||||||
|
return
|
||||||
|
if a.get("type") in ("button", "submit", "file"):
|
||||||
|
return
|
||||||
|
self.pairs.append((a["name"], a.get("value", "")))
|
||||||
|
|
||||||
|
def handle_endtag(self, tag):
|
||||||
|
if tag == "select":
|
||||||
|
self._select = None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def app_client(tmp_path):
|
||||||
|
from src.plugin_system.schema_manager import SchemaManager
|
||||||
|
from web_interface.blueprints import api_v3 as api
|
||||||
|
from web_interface.blueprints import pages_v3 as pages
|
||||||
|
|
||||||
|
plugins_dir = tmp_path / "plugin-repos"
|
||||||
|
pdir = plugins_dir / "demo"
|
||||||
|
pdir.mkdir(parents=True)
|
||||||
|
(pdir / "config_schema.json").write_text(json.dumps(SCHEMA), encoding="utf-8")
|
||||||
|
(pdir / "manifest.json").write_text(
|
||||||
|
json.dumps({"id": "demo", "name": "Demo", "version": "1.0.0"}),
|
||||||
|
encoding="utf-8")
|
||||||
|
|
||||||
|
store = {"demo": json.loads(json.dumps(STORED))}
|
||||||
|
cm = MagicMock()
|
||||||
|
cm.load_config.side_effect = lambda: json.loads(json.dumps(store))
|
||||||
|
cm.get_raw_file_content.return_value = {}
|
||||||
|
cm.get_config_path.return_value = str(tmp_path / "config.json")
|
||||||
|
|
||||||
|
def _save(cfg, **_kw):
|
||||||
|
store.clear()
|
||||||
|
store.update(cfg)
|
||||||
|
return type("R", (), {"status": type("S", (), {"value": "success"})(),
|
||||||
|
"message": None})()
|
||||||
|
|
||||||
|
cm.save_config_atomic.side_effect = _save
|
||||||
|
|
||||||
|
pm = MagicMock()
|
||||||
|
pm.plugins = {}
|
||||||
|
pm.plugins_dir = plugins_dir
|
||||||
|
pm.get_plugin.return_value = None
|
||||||
|
pm.get_plugin_info.return_value = {"name": "Demo", "version": "1.0.0"}
|
||||||
|
sm = SchemaManager(plugins_dir=plugins_dir, project_root=tmp_path)
|
||||||
|
|
||||||
|
names = ("config_manager", "schema_manager", "plugin_manager")
|
||||||
|
originals = {(bp, k): getattr(bp, k, None)
|
||||||
|
for bp in (api.api_v3, pages.pages_v3) for k in names}
|
||||||
|
for bp in (api.api_v3, pages.pages_v3):
|
||||||
|
bp.config_manager = cm
|
||||||
|
bp.schema_manager = sm
|
||||||
|
bp.plugin_manager = pm
|
||||||
|
|
||||||
|
base = Path(pages.__file__).resolve().parent.parent
|
||||||
|
app = Flask(__name__, template_folder=str(base / "templates"),
|
||||||
|
static_folder=str(base / "static"))
|
||||||
|
app.config["TESTING"] = True
|
||||||
|
app.register_blueprint(pages.pages_v3, url_prefix="/v3")
|
||||||
|
app.register_blueprint(api.api_v3, url_prefix="/api/v3")
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield app.test_client(), store
|
||||||
|
finally:
|
||||||
|
for (bp, k), v in originals.items():
|
||||||
|
setattr(bp, k, v)
|
||||||
|
|
||||||
|
|
||||||
|
def _render(client):
|
||||||
|
resp = client.get("/v3/partials/plugin-config/demo")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
return resp.get_data(as_text=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _fields(html):
|
||||||
|
parser = _FormFields()
|
||||||
|
parser.feed(html)
|
||||||
|
return dict(parser.pairs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_booleans_render_their_schema_default(app_client):
|
||||||
|
client, _ = app_client
|
||||||
|
fields = _fields(_render(client))
|
||||||
|
# Checked, so a browser posts them.
|
||||||
|
assert fields.get("show_date") == "true"
|
||||||
|
assert fields.get("show_date_line") == "true"
|
||||||
|
assert fields.get("grid.show_lines") == "true"
|
||||||
|
assert fields.get("overlay.show_sun") == "true"
|
||||||
|
# A saved value still wins over the default.
|
||||||
|
assert fields.get("show_seconds") == "true"
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_non_boolean_fields_render_their_schema_default(app_client):
|
||||||
|
client, _ = app_client
|
||||||
|
fields = _fields(_render(client))
|
||||||
|
assert fields.get("style") == "globe" # not the first option
|
||||||
|
assert fields.get("overlay.mode") == "ring"
|
||||||
|
assert fields.get("brightness") == "70"
|
||||||
|
assert fields.get("label") == "UTC"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_secret_default_is_still_masked(app_client):
|
||||||
|
client, _ = app_client
|
||||||
|
assert "not-a-real-secret" not in _render(client)
|
||||||
|
|
||||||
|
|
||||||
|
def test_saving_the_rendered_form_keeps_default_true_booleans_on(app_client):
|
||||||
|
client, store = app_client
|
||||||
|
pairs = list(_fields(_render(client)).items())
|
||||||
|
resp = client.post("/api/v3/plugins/config?plugin_id=demo",
|
||||||
|
data=MultiDict(pairs))
|
||||||
|
assert resp.status_code == 200, resp.get_json()
|
||||||
|
|
||||||
|
saved = store["demo"]
|
||||||
|
assert saved["show_date"] is True
|
||||||
|
assert saved["show_date_line"] is True
|
||||||
|
assert saved["grid"]["show_lines"] is True
|
||||||
|
assert saved["overlay"]["show_sun"] is True
|
||||||
|
assert saved["overlay"]["mode"] == "ring"
|
||||||
|
assert saved["style"] == "globe"
|
||||||
|
assert saved["show_seconds"] is True
|
||||||
@@ -13,6 +13,7 @@ _SAFE_WEB_UI_FILE_RE = re.compile(r'^[a-zA-Z0-9_-]{1,64}\.html$')
|
|||||||
_SAFE_WIDGET_NAME_RE = re.compile(r'^[a-zA-Z0-9_-]{1,64}$')
|
_SAFE_WIDGET_NAME_RE = re.compile(r'^[a-zA-Z0-9_-]{1,64}$')
|
||||||
_SAFE_WIDGET_SCRIPT_RE = re.compile(r'^[a-zA-Z0-9_-]{1,64}\.js$')
|
_SAFE_WIDGET_SCRIPT_RE = re.compile(r'^[a-zA-Z0-9_-]{1,64}\.js$')
|
||||||
from src.web_interface.secret_helpers import mask_secret_fields
|
from src.web_interface.secret_helpers import mask_secret_fields
|
||||||
|
from src.plugin_system.schema_manager import plugin_config_defaults, prepare_plugin_config
|
||||||
from src.common.path_safety import resolve_under, safe_path_component
|
from src.common.path_safety import resolve_under, safe_path_component
|
||||||
from src.pi5_matrix_support import is_raspberry_pi_5
|
from src.pi5_matrix_support import is_raspberry_pi_5
|
||||||
from web_interface import widget_bundle
|
from web_interface import widget_bundle
|
||||||
@@ -917,10 +918,25 @@ def _load_plugin_config_partial(plugin_id):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Could not load manifest for plugin: %s", e)
|
logger.warning("Could not load manifest for plugin: %s", e)
|
||||||
|
|
||||||
# Mask secret fields before rendering template (fail closed — never leak secrets)
|
|
||||||
schema_properties = schema.get('properties') if isinstance(schema, dict) else None
|
schema_properties = schema.get('properties') if isinstance(schema, dict) else None
|
||||||
if not isinstance(schema_properties, dict):
|
if not isinstance(schema_properties, dict):
|
||||||
return '<div class="text-red-500 p-4">Error loading plugin config securely: schema unavailable.</div>', 500
|
return '<div class="text-red-500 p-4">Error loading plugin config securely: schema unavailable.</div>', 500
|
||||||
|
|
||||||
|
# Fill in schema defaults for keys the saved config doesn't have yet,
|
||||||
|
# as GET /api/v3/plugins/config does. Without this, an option added in
|
||||||
|
# a plugin update (geochron 1.2.0's show_date, default true) renders
|
||||||
|
# as an unchecked box, and because the save treats every drawn but
|
||||||
|
# unposted checkbox as false, the first save turns it off for good.
|
||||||
|
try:
|
||||||
|
defaults = plugin_config_defaults(schema)
|
||||||
|
if schema_mgr is not None:
|
||||||
|
defaults = schema_mgr.apply_device_location(defaults)
|
||||||
|
config = prepare_plugin_config(config, schema, defaults)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Could not merge schema defaults for %s: %s", plugin_id, e)
|
||||||
|
|
||||||
|
# Mask secret fields before rendering template (fail closed — never
|
||||||
|
# leak secrets). After the merge, so a secret's default is masked too.
|
||||||
config = mask_secret_fields(config, schema_properties)
|
config = mask_secret_fields(config, schema_properties)
|
||||||
|
|
||||||
# Determine enabled status
|
# Determine enabled status
|
||||||
|
|||||||
@@ -29,6 +29,11 @@
|
|||||||
|
|
||||||
{% macro render_field(key, prop, value, prefix='', plugin_id='') %}
|
{% macro render_field(key, prop, value, prefix='', plugin_id='') %}
|
||||||
{% if not prop_is_hidden(prop)|trim %}
|
{% if not prop_is_hidden(prop)|trim %}
|
||||||
|
{# A key the saved config doesn't have renders its schema default. The
|
||||||
|
route merges defaults first, but that merge stops at an object that
|
||||||
|
declares its own default (often {}), leaving its children missing here.
|
||||||
|
Booleans matter most: an unchecked box saves as false. #}
|
||||||
|
{% if value is none and prop.default is defined %}{% set value = prop.default %}{% endif %}
|
||||||
{% set full_key = (prefix ~ '.' ~ key) if prefix else key %}
|
{% set full_key = (prefix ~ '.' ~ key) if prefix else key %}
|
||||||
{% set field_id = (plugin_id ~ '-' ~ full_key)|replace('.', '-')|replace('_', '-') %}
|
{% set field_id = (plugin_id ~ '-' ~ full_key)|replace('.', '-')|replace('_', '-') %}
|
||||||
{% set label = prop.title if prop.title else key|replace('_', ' ')|title %}
|
{% set label = prop.title if prop.title else key|replace('_', ' ')|title %}
|
||||||
|
|||||||
Reference in New Issue
Block a user