Merge refactor/api-v3-blueprint-split into feat/tools-mqtt-bridge-pixlet-editor-on-split

PR #554 was branched from #553 before its CodeRabbit-fix commit
(4e91cf1) landed, leaving GitHub reporting mergeable_state: dirty and
the branch missing eight fixes to shared api_v3 package code —
including a guaranteed ModuleNotFoundError on display.py's on-demand
restart path, a credential-redaction gap, and an unguarded manifest
read-modify-write race.

Resolved the two import-list conflicts (backup.py, starlark.py) by
unioning both branches' additions; the rest of the merge was clean.
Verified the merged tree keeps every usage of the fix branch's new
helpers (_starlark_manifest_lock, _coerce_to_bool) intact, the URL map
fixture still lists all 117 routes, and the full suite passes with only
the two pre-existing tzdata-related failures (unrelated, present on
origin/main too).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GwNEnaWDTzDbFE2M7MDjaq

test/test_api_v3_font_upload_size.py

@@ -0,0 +1,60 @@
+"""Regression test: POST /fonts/upload must enforce its own stated size limit.
+
+validate_file_upload(filename, max_size_mb=10, allowed_extensions=[...]) reads
+like it checks the upload's size, but it only ever validated the filename
+(traversal characters, extension) -- max_size_mb was accepted and silently
+ignored. Nothing else in the handler checked the actual upload size either,
+so it saved whatever was posted to assets/fonts/<family><ext> regardless of
+size, unlike the sibling .star and plugin-asset upload routes, which check
+`file.tell()` against a stated limit before saving.
+"""
+
+import io
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from test._api_v3_test_helpers import api_v3_client, api_v3_module  # noqa: F401,E402
+
+URL = "/api/v3/fonts/upload"
+TEN_MB = 10 * 1024 * 1024
+
+
+@pytest.fixture
+def fonts_root(tmp_path):
+    # PROJECT_ROOT is bound by value in fonts.py, so it is patched there.
+    with patch("web_interface.blueprints.api_v3.fonts.PROJECT_ROOT", tmp_path):
+        yield tmp_path
+
+
+def upload(client, content, filename="myfont.ttf", family="myfont"):
+    data = {
+        "font_file": (io.BytesIO(content), filename),
+        "font_family": family,
+    }
+    return client.post(URL, data=data, content_type="multipart/form-data")
+
+
+class TestFontUploadSizeLimit:
+    def test_oversized_font_is_rejected(self, api_v3_client, fonts_root):
+        response = upload(api_v3_client, b"x" * (TEN_MB + 1))
+        assert response.status_code == 400
+        assert "too large" in response.get_json()["message"].lower()
+        assert not (fonts_root / "assets" / "fonts" / "myfont.ttf").exists(), (
+            "an oversized font was saved to disk before being rejected")
+
+    def test_a_font_right_at_the_limit_is_accepted(self, api_v3_client, fonts_root):
+        response = upload(api_v3_client, b"x" * TEN_MB,
+                          filename="atlimit.ttf", family="atlimit")
+        assert response.status_code == 200, response.get_json()
+        assert (fonts_root / "assets" / "fonts" / "atlimit.ttf").exists()
+
+    def test_an_ordinary_small_font_is_still_accepted(self, api_v3_client, fonts_root):
+        response = upload(api_v3_client, b"fake font bytes",
+                          filename="small.ttf", family="small")
+        assert response.status_code == 200, response.get_json()
+        assert (fonts_root / "assets" / "fonts" / "small.ttf").read_bytes() == b"fake font bytes"

test/test_api_v3_on_demand_restart.py

@@ -0,0 +1,93 @@
+"""Regression test: POST /display/on-demand/start restarting a running
+service must not import a name that does not exist.
+
+display.py has `import web_interface.blueprints.api_v3 as _pkg` and reads
+mutable, test-patched attributes back through it (`_pkg.time.time()`,
+`_pkg._get_starlark_plugin()`, ...) rather than binding them by value, per
+the package's own docstring. One spot went further and wrote a genuine
+`import` *statement* against that alias --
+
+    import _pkg.time as time_module
+
+-- but `_pkg` is a local name bound by `import ... as _pkg` in this module,
+not a real top-level package, so `import _pkg.time` is not something Python
+can resolve; it raises ModuleNotFoundError. That line only runs when the
+display service is already running and the caller also asked to (re)start
+it, so this endpoint failed on exactly the restart path -- the one where a
+cache write recording the new on-demand request had already happened.
+
+The route wraps its body in `except Exception`, so the failure reached the
+caller as a handled 500 with a generic message, not an unhandled crash --
+but a 500 all the same on a request that should have restarted the service
+and reported success.
+"""
+
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from test._api_v3_test_helpers import api_v3_client, api_v3_module  # noqa: F401,E402
+
+URL = "/api/v3/display/on-demand/start"
+
+
+@pytest.fixture
+def restart_path(api_v3_module):
+    """Force the `service_was_running and start_service` branch.
+
+    plugin_manager and config_manager are set to None so the route takes
+    the simplest path to that branch rather than tripping over unrelated
+    MagicMock plumbing; _ensure_cache_manager, _get_display_service_status,
+    _stop_display_service and _ensure_display_service_running are bound by
+    value in display.py (see its own docstring), so they are patched on
+    that submodule rather than on the package.
+    """
+    api_v3_module.api_v3.plugin_manager = None
+    api_v3_module.api_v3.config_manager = None
+
+    with patch("web_interface.blueprints.api_v3.display._ensure_cache_manager") as ensure_cache, \
+         patch("web_interface.blueprints.api_v3.display._get_display_service_status") as get_status, \
+         patch("web_interface.blueprints.api_v3.display._stop_display_service") as stop_service, \
+         patch("web_interface.blueprints.api_v3.display._ensure_display_service_running") as ensure_running:
+        ensure_cache.return_value = MagicMock()
+        # Active before the request: service_was_running becomes True.
+        get_status.return_value = {"active": True}
+        ensure_running.return_value = {"active": True}
+        yield {
+            "ensure_cache": ensure_cache,
+            "get_status": get_status,
+            "stop_service": stop_service,
+            "ensure_running": ensure_running,
+        }
+
+
+class TestRestartingARunningService:
+    def test_it_does_not_500(self, api_v3_client, restart_path):
+        response = api_v3_client.post(
+            URL, json={"plugin_id": "weather", "start_service": True})
+        body = response.get_json()
+        assert response.status_code == 200, body
+        assert body["status"] == "success", body
+
+    def test_the_service_is_actually_stopped_and_restarted(
+            self, api_v3_client, restart_path):
+        api_v3_client.post(
+            URL, json={"plugin_id": "weather", "start_service": True})
+        restart_path["stop_service"].assert_called_once()
+        restart_path["ensure_running"].assert_called_once()
+
+    def test_a_service_that_was_not_running_is_not_stopped_first(
+            self, api_v3_client, restart_path):
+        # The buggy import sits inside `if service_was_running and
+        # start_service`, so it only ever fired on the restart path --
+        # this is the other side of that branch, unaffected either way,
+        # kept here so the branch condition itself stays covered.
+        restart_path["get_status"].return_value = {"active": False}
+        response = api_v3_client.post(
+            URL, json={"plugin_id": "weather", "start_service": True})
+        assert response.status_code == 200, response.get_json()
+        restart_path["stop_service"].assert_not_called()

test/test_api_v3_schedule_error_messages.py

@@ -0,0 +1,54 @@
+"""Regression test: per-day schedule validation errors must say "time", not
+"_pkg.time".
+
+The split into a package rewrote every bare `time` reference that needed to
+read through the shared module as `_pkg.time` (see the package's own
+docstring on why -- tests patch it, so it has to be read back live rather
+than bound by value). That rewrite was mechanical and matched the substring
+"time" inside string literals and comments too, so the user-facing message
+
+    "Invalid start time for {day}: ..."
+
+came out as
+
+    "Invalid start _pkg.time for {day}: ..."
+
+in both POST /config/schedule and POST /config/dim-schedule, for both the
+start and end time of a per-day entry.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from test._api_v3_test_helpers import api_v3_client, api_v3_module  # noqa: F401,E402
+
+
+@pytest.mark.parametrize("url", [
+    "/api/v3/config/schedule",
+    "/api/v3/config/dim-schedule",
+])
+class TestPerDayTimeErrorsAreNotCorrupted:
+    def test_invalid_start_time_message(self, api_v3_client, api_v3_module, url):
+        response = api_v3_client.post(url, json={
+            "mode": "per_day",
+            "monday_start": "not-a-time",
+        })
+        assert response.status_code == 400
+        message = response.get_json()["message"]
+        assert "_pkg" not in message, message
+        assert message.startswith("Invalid start time for monday:"), message
+
+    def test_invalid_end_time_message(self, api_v3_client, api_v3_module, url):
+        response = api_v3_client.post(url, json={
+            "mode": "per_day",
+            "monday_start": "07:00",
+            "monday_end": "not-a-time",
+        })
+        assert response.status_code == 400
+        message = response.get_json()["message"]
+        assert "_pkg" not in message, message
+        assert message.startswith("Invalid end time for monday:"), message

test/test_api_v3_wifi_endpoints.py

@@ -229,6 +229,61 @@ class TestRadio:
         assert api_v3_client.post(self.URL, json={"enabled": True}).status_code == 500
 
 
+class TestAutoEnableApMode:
+    URL = "/api/v3/wifi/ap/auto-enable"
+
+    def test_requires_the_field(self, api_v3_client, wifi_manager):
+        response = api_v3_client.post(self.URL, json={})
+        assert response.status_code == 400
+
+    @pytest.mark.parametrize("raw,expected", [
+        (True, True), (False, False),
+        ("true", True), ("True", True), ("1", True), ("yes", True),
+        ("false", False), ("False", False), ("0", False), ("no", False),
+        (1, True), (0, False),
+    ])
+    def test_value_is_coerced_not_just_truthy(
+            self, api_v3_client, wifi_manager, raw, expected):
+        # Regression: bool(data['auto_enable_ap_mode']) meant a caller who
+        # sent the JSON string "false" got it stored as True — bool("false")
+        # is True, since any non-empty string is truthy.
+        wifi_manager.config = {}
+        response = api_v3_client.post(self.URL, json={"auto_enable_ap_mode": raw})
+        assert response.status_code == 200, response.get_json()
+        assert response.get_json()["data"]["auto_enable_ap_mode"] is expected
+        assert wifi_manager.config["auto_enable_ap_mode"] is expected
+
+    def test_a_string_false_does_not_enable_it(self, api_v3_client, wifi_manager):
+        # The exact shape of the bug.
+        wifi_manager.config = {}
+        api_v3_client.post(self.URL, json={"auto_enable_ap_mode": "false"})
+        assert wifi_manager.config["auto_enable_ap_mode"] is False
+
+
+class TestRadioEnabledAndForceAcceptIntegers:
+    """`{"enabled": 1}` / `{"enabled": 0}` used to be mishandled: the old
+    coercion was `raw is True or (isinstance(raw, str) and ...)`, and
+    `1 is True` is False in Python -- an int is never the `True` singleton
+    even though it equals it -- so a plain integer fell through to False
+    regardless of its value.
+    """
+    URL = "/api/v3/wifi/radio"
+
+    @pytest.mark.parametrize("raw,expected", [(1, True), (0, False)])
+    def test_enabled_as_an_integer(self, api_v3_client, wifi_manager, raw, expected):
+        wifi_manager.set_wifi_radio.return_value = (True, "ok", None)
+        wifi_manager.get_wifi_radio_state.return_value = {}
+        api_v3_client.post(self.URL, json={"enabled": raw})
+        wifi_manager.set_wifi_radio.assert_called_once_with(expected, force=False)
+
+    @pytest.mark.parametrize("raw,expected", [(1, True), (0, False)])
+    def test_force_as_an_integer(self, api_v3_client, wifi_manager, raw, expected):
+        wifi_manager.set_wifi_radio.return_value = (True, "ok", None)
+        wifi_manager.get_wifi_radio_state.return_value = {}
+        api_v3_client.post(self.URL, json={"enabled": True, "force": raw})
+        wifi_manager.set_wifi_radio.assert_called_once_with(True, force=expected)
+
+
 class TestNoRealNetworking:
     def test_wifi_manager_is_never_constructed_for_real(self, api_v3_client):
         # Guard against a future refactor moving the import to module level,

test/test_config_main_redacts_secrets.py

@@ -103,6 +103,43 @@ def test_non_dict_input_passes_through():
     assert _redact_credentials(None) is None
 
 
+def test_a_list_of_bare_credentials_is_redacted():
+    """A credential-named key whose value is a list of scalars, not a list
+    of named-field dicts, used to pass through untouched: the dict branch
+    recursed into `v` on its own value only, so a bare string list item had
+    no field name to test and fell through the base case unredacted.
+    """
+    config = {"tokens": ["abc123", "def456"], "timezone": "America/New_York"}
+    out = _redact_credentials(config)
+    assert out["tokens"] == ["", ""]
+    assert out["timezone"] == "America/New_York"
+
+
+def test_a_list_nested_inside_a_credential_named_container_is_also_blanked():
+    """The list fix applies at any depth under a credential-named key, not
+    only when the list is the key's direct value.
+    """
+    config = {"auth": {"backup_tokens": ["a", "b"], "note": "keep"}}
+    out = _redact_credentials(config)
+    assert out["auth"]["backup_tokens"] == ["", ""]
+    assert out["auth"]["note"] == "keep"
+
+
+def test_credential_shaped_container_with_named_fields_still_only_blanks_those():
+    """Unchanged behaviour for the case the container test above already
+    covers: a dict whose sub-keys are semantically named fields is walked
+    normally, not blanket-blanked, so an ordinary field next to a
+    credential-named one survives.
+    """
+    config = {"secrets": {"api_key": "k", "note": "keep", "list": ["a", "b"]}}
+    out = _redact_credentials(config)
+    assert out["secrets"]["api_key"] == ""
+    assert out["secrets"]["note"] == "keep"
+    # "list" isn't itself credential-named, so it is walked, not blanked --
+    # but it holds no credential-named field either, so it survives whole.
+    assert out["secrets"]["list"] == ["a", "b"]
+
+
 def test_the_endpoint_itself_redacts():
     """Through the view function, not the helper.
 

test/web_interface/test_api_v3_backup_restore.py

@@ -140,6 +140,41 @@ class TestRequestValidation:
         assert restore.call_args[0][2].restore_config is True
 
 
+class TestOptionsAreBooleanAware:
+    """Regression: bool("false") is True in Python.
+
+    Every restore flag used bare bool() coercion, so a caller that sends its
+    options as JSON strings rather than real booleans -- a form field, a
+    hand-built request -- had `{"restore_secrets": "false"}` restore secrets
+    anyway, the opposite of what was asked. Fixed with the same
+    string-aware `_coerce_to_bool` already used for checkbox-style config
+    fields elsewhere in this package (config.py, plugins.py).
+    """
+
+    @pytest.mark.parametrize("raw,expected", [
+        ("false", False), ("False", False), ("FALSE", False),
+        ("0", False),
+        ("true", True), ("True", True), ("1", True),
+    ])
+    def test_string_valued_flags_are_parsed_not_just_truthy(
+            self, client, restore, raw, expected):
+        post(client, options=json.dumps({"restore_secrets": raw}))
+        assert restore.call_args[0][2].restore_secrets is expected
+
+    def test_a_string_false_does_not_restore_secrets(self, client, restore):
+        # The exact shape of the bug: a truthy non-empty string coerced by
+        # bare bool() to True regardless of its contents.
+        post(client, options=json.dumps({"restore_secrets": "false"}))
+        assert restore.call_args[0][2].restore_secrets is False
+
+    def test_real_json_booleans_still_work(self, client, restore):
+        post(client, options=json.dumps({"restore_secrets": False,
+                                          "restore_config": True}))
+        options = restore.call_args[0][2]
+        assert options.restore_secrets is False
+        assert options.restore_config is True
+
+
 class TestSuccess:
     def test_success_returns_the_result(self, client, restore):
         restore.return_value = FakeResult(success=True, restored=["config", "secrets"])

test/web_interface/test_calendar_oauth_endpoints.py

@@ -363,6 +363,22 @@ class TestDiagnosticsAreRedacted:
         assert 'hunter2' not in error, error
         assert '<redacted>' in error, error
 
+    def test_script_stderr_is_redacted_in_the_log_too(self, tmp_path, caplog):
+        # Regression: the return value went through redact_text (asserted
+        # above), but the logger.error call right next to it logged `raw`
+        # verbatim -- a script that handles OAuth client secrets and can
+        # quote them in its stderr, landing unredacted in the log (CWE-532).
+        script = tmp_path / 'calendar_registration.py'
+        script.write_text(
+            'import sys\n'
+            'sys.stderr.write("boom client_secret=hunter2 more\\n")\n',
+            encoding='utf-8')
+        with caplog.at_level('ERROR', logger=mod.logger.name):
+            mod._run_calendar_registration(tmp_path, '')
+        logged = '\n'.join(r.getMessage() for r in caplog.records)
+        assert 'hunter2' not in logged, logged
+        assert '<redacted>' in logged, logged
+
     def test_a_failing_script_payload_is_redacted(self, client):
         (client.plugin_dir / 'credentials.json').write_text('{}', encoding='utf-8')
         (client.plugin_dir / 'calendar_registration.py').write_text(

test/web_interface/test_starlark_pixlet_routes.py

@@ -395,6 +395,132 @@ class TestTheManifestStaysRelocatable:
         assert self._install(tmp_path)['star_file'] == 'demo.star'
 
 
+class TestManifestLockPreventsLostUpdates:
+    """The standalone manifest fallback (no plugin instance loaded) reads,
+    mutates and writes manifest.json with no coordination across requests.
+    Each write is atomic on its own (temp file + rename), but two concurrent
+    read-modify-write cycles can still race: both read the same starting
+    manifest, and the second write silently discards whatever the first one
+    added. _starlark_manifest_lock closes that window -- mirrors
+    StarlarkAppsPlugin._update_manifest_safe, which already does this when
+    the plugin instance is loaded.
+    """
+
+    @pytest.fixture
+    def starlark_dir(self, tmp_path, monkeypatch):
+        from web_interface.blueprints import api_v3 as module
+        apps_dir = tmp_path / "starlark-apps"
+        apps_dir.mkdir()
+        monkeypatch.setattr(module, '_STARLARK_APPS_DIR', apps_dir)
+        monkeypatch.setattr(module, '_STARLARK_MANIFEST_FILE', apps_dir / 'manifest.json')
+        module._write_starlark_manifest({'apps': {}})
+        return apps_dir
+
+    def test_two_concurrent_updates_are_both_kept(self, starlark_dir):
+        import threading
+        import time as _time
+
+        from web_interface.blueprints import api_v3 as module
+
+        def add_app(app_id):
+            with module._starlark_manifest_lock():
+                manifest = module._read_starlark_manifest()
+                # Widen the window between read and write. Without the lock
+                # both threads read here before either writes, and whichever
+                # writes second overwrites the other's addition; with the
+                # lock, the second thread cannot even start its read until
+                # the first has written and released.
+                _time.sleep(0.05)
+                manifest.setdefault('apps', {})[app_id] = {'enabled': True}
+                module._write_starlark_manifest(manifest)
+
+        threads = [threading.Thread(target=add_app, args=(app_id,))
+                   for app_id in ('a', 'b')]
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join(timeout=5)
+
+        manifest = module._read_starlark_manifest()
+        assert set(manifest['apps']) == {'a', 'b'}, (
+            "a concurrent update was lost: %r" % (manifest,))
+
+    def test_the_lock_is_reentrant_safe_across_sequential_calls(self, starlark_dir):
+        """Not reentrant within one thread -- just that using it twice in a
+        row (the ordinary case: one request, then the next) works cleanly
+        and does not leak the lock file descriptor or leave it locked."""
+        from web_interface.blueprints import api_v3 as module
+
+        for app_id in ('first', 'second'):
+            with module._starlark_manifest_lock():
+                manifest = module._read_starlark_manifest()
+                manifest.setdefault('apps', {})[app_id] = {'enabled': True}
+                module._write_starlark_manifest(manifest)
+
+        manifest = module._read_starlark_manifest()
+        assert set(manifest['apps']) == {'first', 'second'}
+
+
+class TestConfigAndManifestStayInSync:
+    """Standalone-mode PUT /starlark/apps/<id>/config (no plugin instance
+    loaded) writes config.json and then the manifest. If the manifest write
+    fails after config.json was already written, the two disagree about
+    what was saved unless config.json is rolled back.
+    """
+
+    @pytest.fixture
+    def app_dir(self, tmp_path, monkeypatch):
+        from web_interface.blueprints import api_v3 as module
+        apps_dir = tmp_path / "starlark-apps"
+        apps_dir.mkdir()
+        monkeypatch.setattr(module, '_STARLARK_APPS_DIR', apps_dir)
+        monkeypatch.setattr(module, '_STARLARK_MANIFEST_FILE', apps_dir / 'manifest.json')
+        one_app_dir = apps_dir / 'demo'
+        one_app_dir.mkdir()
+        module._write_starlark_manifest({'apps': {'demo': {'name': 'Demo', 'enabled': True}}})
+        return one_app_dir
+
+    def test_manifest_write_failure_rolls_back_an_existing_config_json(self, client, app_dir):
+        config_file = app_dir / 'config.json'
+        config_file.write_text(json.dumps({'existing': 'value'}))
+
+        with patch('web_interface.blueprints.api_v3._get_starlark_plugin', return_value=None), \
+             patch('web_interface.blueprints.api_v3._write_starlark_manifest', return_value=False):
+            resp = client.put('/api/v3/starlark/apps/demo/config',
+                              json={'new_field': 'x'})
+
+        assert resp.status_code == 500
+        assert json.loads(config_file.read_text()) == {'existing': 'value'}, (
+            "config.json kept the new value even though the manifest write "
+            "that was supposed to follow it failed")
+
+    def test_manifest_write_failure_removes_a_freshly_created_config_json(self, client, app_dir):
+        config_file = app_dir / 'config.json'
+        assert not config_file.exists()
+
+        with patch('web_interface.blueprints.api_v3._get_starlark_plugin', return_value=None), \
+             patch('web_interface.blueprints.api_v3._write_starlark_manifest', return_value=False):
+            resp = client.put('/api/v3/starlark/apps/demo/config',
+                              json={'new_field': 'x'})
+
+        assert resp.status_code == 500
+        assert not config_file.exists(), (
+            "config.json was left behind even though the manifest write "
+            "that was supposed to follow it failed")
+
+    def test_success_updates_both_config_and_manifest(self, client, app_dir):
+        from web_interface.blueprints import api_v3 as module
+
+        with patch('web_interface.blueprints.api_v3._get_starlark_plugin', return_value=None):
+            resp = client.put('/api/v3/starlark/apps/demo/config',
+                              json={'new_field': 'x'})
+
+        assert resp.status_code == 200, resp.get_json()
+        assert json.loads((app_dir / 'config.json').read_text())['new_field'] == 'x'
+        manifest = json.loads(module._STARLARK_MANIFEST_FILE.read_text())
+        assert manifest['apps']['demo']['config']['new_field'] == 'x'
+
+
 # ---------------------------------------------------------------------------
 # The store loaded, then stopped loading, and nothing anywhere said why.
 #