refactor(web): use canonical secret helpers in api_v3; make ConfigManager secret strip/merge array-aware

api_v3.py carried three inline nested copies of find_secret_fields/
separate_secrets (main-config save, plugin-config save, plugin-config
reset). They drifted from each other (one lacked isinstance guards) and
none supported the canonical module's array-item secrets
(accounts[].token). All three endpoints now import from
src/web_interface/secret_helpers.

Adopting the canonical behavior makes array-item secrets reachable, and
their parallel-placeholder shape ([{'token': ...}, {}] alongside the
regular list) was not survivable by ConfigManager's round-trip:
_strip_secrets_recursive dropped the whole key (losing the regular
fields from config.json) and _deep_merge replaced the regular list
wholesale on load. Both are now array-aware:

- strip removes the secret fields from each item and ALWAYS keeps the
  list so indices survive for merge-on-load; whole-key secrets (scalar
  lists, shape mismatches) still drop the key entirely — never leak.
- merge folds each secrets item into the config item at the same index,
  skipping {} placeholders. The regular list's length is authoritative
  in both directions: a user deleting an array item never has it
  resurrected from a stale secrets entry (extras warn and are ignored).

api_v3's own deep_merge intentionally still replaces lists wholesale —
form posts carry complete arrays and index-merging would resurrect
deleted items; a comment now documents that.

Tests: the parity guard flips from 'exactly 3 inline copies' to 'zero,
and the canonical import must exist'; TestArraySecretStripAndMerge
covers the new strip/merge semantics incl. length-mismatch contracts;
new test_api_v3_secret_roundtrip.py drives all three endpoints through
a Flask client with a REAL ConfigManager+SchemaManager over tmp_path,
proving secrets land in config_secrets.json, config.json stays clean,
and a fresh load merges them back into the right array items.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NohXi78cwsAKtN1sCfxjUh

test/web_interface/test_api_v3_secret_roundtrip.py

@@ -0,0 +1,259 @@
+"""
+End-to-end secret round-trips through the three api_v3 endpoints that
+separate secrets from regular config (main-config save, plugin-config save,
+plugin-config reset) — now backed by the canonical
+src/web_interface/secret_helpers implementations.
+
+Unlike test_web_api.py (which mocks the config manager), these tests run a
+REAL ConfigManager and a REAL SchemaManager over tmp_path files, so they
+prove the whole chain: endpoint separation -> config_secrets.json write ->
+atomic config.json save (strip) -> load_config (merge back), including the
+array-item secret shape (accounts[].token) the inline copies never
+supported.
+"""
+
+import json
+import sys
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+from flask import Flask
+
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+from src.config_manager import ConfigManager  # noqa: E402
+from src.plugin_system.schema_manager import SchemaManager  # noqa: E402
+from web_interface.blueprints.api_v3 import api_v3  # noqa: E402
+
+
+PLUGIN_ID = "testplugin"
+
+SCHEMA = {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "properties": {
+        "enabled": {"type": "boolean", "default": True},
+        "display_duration": {"type": "number", "default": 15},
+        "api_key": {"type": "string", "x-secret": True, "default": ""},
+        "city": {"type": "string", "default": "Austin"},
+        "accounts": {
+            "type": "array",
+            "default": [],
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "token": {"type": "string", "x-secret": True},
+                },
+            },
+        },
+    },
+}
+
+
+@pytest.fixture
+def env(tmp_path):
+    """Real ConfigManager + SchemaManager over tmp_path, wired onto the
+    api_v3 blueprint with the remaining managers mocked."""
+    config_file = tmp_path / "config.json"
+    config_file.write_text("{}")
+
+    plugins_dir = tmp_path / "plugins"
+    plugin_dir = plugins_dir / PLUGIN_ID
+    plugin_dir.mkdir(parents=True)
+    (plugin_dir / "config_schema.json").write_text(json.dumps(SCHEMA))
+    (plugin_dir / "manifest.json").write_text(json.dumps({
+        "id": PLUGIN_ID, "name": "Test Plugin", "version": "1.0.0",
+    }))
+
+    config_manager = ConfigManager(
+        config_path=str(config_file),
+        secrets_path=str(tmp_path / "config_secrets.json"))
+    config_manager.template_path = str(tmp_path / "no-template.json")
+
+    schema_manager = SchemaManager(plugins_dir=plugins_dir,
+                                   project_root=tmp_path)
+
+    plugin_manager = MagicMock()
+    plugin_manager.plugin_manifests = {PLUGIN_ID: {"id": PLUGIN_ID}}
+    plugin_manager.plugins_dir = plugins_dir
+    plugin_manager.get_plugin.return_value = None
+
+    api_v3.config_manager = config_manager
+    api_v3.schema_manager = schema_manager
+    api_v3.plugin_manager = plugin_manager
+    api_v3.plugin_store_manager = MagicMock()
+    api_v3.saved_repositories_manager = MagicMock()
+    api_v3.operation_queue = MagicMock()
+    api_v3.plugin_state_manager = MagicMock()
+    api_v3.operation_history = MagicMock()
+    api_v3.cache_manager = MagicMock()
+
+    app = Flask(__name__)
+    app.config["TESTING"] = True
+    app.register_blueprint(api_v3, url_prefix="/api/v3")
+
+    class Env:
+        pass
+
+    e = Env()
+    e.client = app.test_client()
+    e.config_manager = config_manager
+    e.config_file = config_file
+    e.secrets_file = tmp_path / "config_secrets.json"
+    e.tmp_path = tmp_path
+
+    def fresh_load():
+        """Load via a NEW ConfigManager, as the next request/process would.
+
+        The endpoint's manager serves its post-save in-memory config via the
+        mtime fast path, and that copy predates the secrets it just
+        separated out — a pre-existing quirk that applies to scalar secrets
+        too. On-disk truth is what these tests care about.
+        """
+        fresh = ConfigManager(config_path=str(config_file),
+                              secrets_path=str(e.secrets_file))
+        fresh.template_path = str(tmp_path / "no-template.json")
+        return fresh.load_config()
+
+    e.fresh_load = fresh_load
+    return e
+
+
+def _on_disk(path):
+    return json.loads(path.read_text())
+
+
+class TestSaveMainConfig:
+    """Site A: POST /config/main with a plugin-id key."""
+
+    def test_array_and_scalar_secrets_routed_to_secrets_file(self, env):
+        resp = env.client.post("/api/v3/config/main", json={
+            PLUGIN_ID: {
+                "city": "Dallas",
+                "api_key": "s3cret-key",
+                "accounts": [
+                    {"name": "a", "token": "s3cret-a"},
+                    {"name": "b"},
+                ],
+            },
+        })
+        assert resp.status_code == 200, resp.get_json()
+
+        on_disk = _on_disk(env.config_file)
+        assert on_disk[PLUGIN_ID]["city"] == "Dallas"
+        assert "api_key" not in on_disk[PLUGIN_ID]
+        assert on_disk[PLUGIN_ID]["accounts"] == [{"name": "a"}, {"name": "b"}]
+        assert "s3cret" not in env.config_file.read_text()
+
+        secrets = _on_disk(env.secrets_file)
+        assert secrets[PLUGIN_ID]["api_key"] == "s3cret-key"
+        assert secrets[PLUGIN_ID]["accounts"] == [{"token": "s3cret-a"}, {}]
+
+    def test_load_config_merges_secrets_back(self, env):
+        env.client.post("/api/v3/config/main", json={
+            PLUGIN_ID: {"accounts": [{"name": "a", "token": "s3cret-a"}]},
+        })
+        merged = env.fresh_load()
+        assert merged[PLUGIN_ID]["accounts"] == [
+            {"name": "a", "token": "s3cret-a"}]
+
+
+class TestSavePluginConfig:
+    """Site B: POST /plugins/config (JSON body)."""
+
+    def _save(self, env, config):
+        return env.client.post("/api/v3/plugins/config", json={
+            "plugin_id": PLUGIN_ID, "config": config,
+        })
+
+    def test_round_trip_with_array_secrets(self, env):
+        resp = self._save(env, {
+            "enabled": True,
+            "city": "Houston",
+            "api_key": "s3cret-key",
+            "accounts": [
+                {"name": "a", "token": "s3cret-a"},
+                {"name": "b", "token": "s3cret-b"},
+            ],
+        })
+        assert resp.status_code == 200, resp.get_json()
+
+        assert "s3cret" not in env.config_file.read_text()
+        on_disk = _on_disk(env.config_file)
+        assert on_disk[PLUGIN_ID]["accounts"] == [{"name": "a"}, {"name": "b"}]
+
+        secrets = _on_disk(env.secrets_file)
+        assert secrets[PLUGIN_ID]["accounts"] == [
+            {"token": "s3cret-a"}, {"token": "s3cret-b"}]
+
+        merged = env.fresh_load()
+        assert merged[PLUGIN_ID]["accounts"][1]["token"] == "s3cret-b"
+
+    def test_secret_count_message_counts_top_level_keys(self, env):
+        # Pinned: the "(N secret field(s))" message counts TOP-LEVEL keys of
+        # the separated secrets dict. Here that is 2: the posted accounts
+        # array (all its item tokens count as ONE key) plus the schema's
+        # api_key default ("") that merge_with_defaults adds before
+        # separation.
+        resp = self._save(env, {
+            "accounts": [{"name": "a", "token": "t"}],
+        })
+        message = resp.get_json()["message"]
+        assert "(2 secret field(s) saved to config_secrets.json)" in message
+
+    def test_resave_replaces_stored_secrets_list_wholesale(self, env):
+        # Characterized: api_v3's deep_merge intentionally replaces lists,
+        # so a re-save's parallel secrets list is authoritative.
+        self._save(env, {"accounts": [
+            {"name": "a", "token": "old-a"},
+            {"name": "b", "token": "old-b"},
+        ]})
+        self._save(env, {"accounts": [{"name": "only", "token": "new-only"}]})
+
+        secrets = _on_disk(env.secrets_file)
+        assert secrets[PLUGIN_ID]["accounts"] == [{"token": "new-only"}]
+        merged = env.fresh_load()
+        assert merged[PLUGIN_ID]["accounts"] == [
+            {"name": "only", "token": "new-only"}]
+
+
+class TestResetPluginConfig:
+    """Site C: POST /plugins/config/reset."""
+
+    def _seed(self, env):
+        env.client.post("/api/v3/plugins/config", json={
+            "plugin_id": PLUGIN_ID,
+            "config": {"city": "Houston", "api_key": "s3cret-key",
+                       "accounts": [{"name": "a", "token": "s3cret-a"}]},
+        })
+
+    def test_reset_preserving_secrets(self, env):
+        self._seed(env)
+        resp = env.client.post("/api/v3/plugins/config/reset", json={
+            "plugin_id": PLUGIN_ID, "preserve_secrets": True,
+        })
+        assert resp.status_code == 200, resp.get_json()
+
+        on_disk = _on_disk(env.config_file)
+        assert on_disk[PLUGIN_ID]["city"] == "Austin"  # schema default
+        assert on_disk[PLUGIN_ID]["accounts"] == []    # schema default
+
+        # Existing secrets survive (top-level-only preserve merge, pinned).
+        secrets = _on_disk(env.secrets_file)
+        assert secrets[PLUGIN_ID]["api_key"] == "s3cret-key"
+        assert secrets[PLUGIN_ID]["accounts"] == [{"token": "s3cret-a"}]
+
+    def test_reset_without_preserving_secrets(self, env):
+        self._seed(env)
+        resp = env.client.post("/api/v3/plugins/config/reset", json={
+            "plugin_id": PLUGIN_ID, "preserve_secrets": False,
+        })
+        assert resp.status_code == 200, resp.get_json()
+
+        secrets = _on_disk(env.secrets_file)
+        # Replaced with schema-default secrets — the schema declares no
+        # secret defaults, so the plugin's secrets are emptied.
+        assert secrets[PLUGIN_ID] in ({}, {"api_key": ""})

test/web_interface/test_secret_separation_parity.py

@@ -1,18 +1,13 @@
 """
-Drift guard for the duplicated secret-separation logic.
+Drift guard: api_v3 must use the canonical secret helpers.
 
-src/web_interface/secret_helpers.py is the canonical implementation of
-find_secret_fields / separate_secrets, but web_interface/blueprints/api_v3.py
-still carries THREE inline nested-function copies of each (in the plugin
-config GET, POST, and reset endpoints). The copies lack the canonical
-module's array-item support (`accounts[].token`), so migrating an endpoint
-onto the module is a behavior change that must be made deliberately.
-
-This file guards two things:
-1. The copy count can only go DOWN. A fourth copy appearing means someone
-   re-implemented the logic again instead of importing secret_helpers.
-2. The known behavioral gap is documented as an executable fact, so whoever
-   migrates the endpoints knows exactly what changes.
+Historically web_interface/blueprints/api_v3.py carried THREE inline
+nested-function copies of ``find_secret_fields``/``separate_secrets`` (in the
+main-config save, plugin-config save, and plugin-config reset endpoints).
+They lacked the canonical module's array-item secret support and drifted from
+each other. They have been migrated onto
+``src/web_interface/secret_helpers`` — this file now guards against copies
+REAPPEARING, and keeps the canonical array-item behavior executable.
 """
 
 import re
@@ -23,77 +18,44 @@ from src.web_interface.secret_helpers import find_secret_fields, separate_secret
 API_V3_PATH = (Path(__file__).resolve().parents[2]
                / "web_interface" / "blueprints" / "api_v3.py")
 
-# Update DOWNWARD as endpoints migrate onto src/web_interface/secret_helpers.
-EXPECTED_INLINE_COPIES = 3
+# The migration is complete: any inline reimplementation is a regression.
+EXPECTED_INLINE_COPIES = 0
 
 
-class TestInlineCopyCount:
+class TestNoInlineCopies:
     def _count(self, name: str) -> int:
         source = API_V3_PATH.read_text(encoding="utf-8")
         return len(re.findall(rf"^\s*def {name}\(", source, flags=re.MULTILINE))
 
-    def test_find_secret_fields_copy_count(self):
+    def test_no_inline_find_secret_fields(self):
         count = self._count("find_secret_fields")
         assert count == EXPECTED_INLINE_COPIES, (
             f"api_v3.py has {count} inline find_secret_fields definitions, "
-            f"expected {EXPECTED_INLINE_COPIES}. New code must import it from "
-            f"src/web_interface/secret_helpers instead of re-implementing it; "
-            f"if you migrated an endpoint, lower EXPECTED_INLINE_COPIES."
+            f"expected {EXPECTED_INLINE_COPIES}. Import it from "
+            f"src/web_interface/secret_helpers instead of re-implementing it."
         )
 
-    def test_separate_secrets_copy_count(self):
+    def test_no_inline_separate_secrets(self):
         count = self._count("separate_secrets")
         assert count == EXPECTED_INLINE_COPIES, (
             f"api_v3.py has {count} inline separate_secrets definitions, "
-            f"expected {EXPECTED_INLINE_COPIES}. New code must import it from "
-            f"src/web_interface/secret_helpers instead of re-implementing it; "
-            f"if you migrated an endpoint, lower EXPECTED_INLINE_COPIES."
+            f"expected {EXPECTED_INLINE_COPIES}. Import it from "
+            f"src/web_interface/secret_helpers instead of re-implementing it."
         )
 
-    def test_inline_copies_lack_array_item_support(self):
-        """The documented gap: no inline copy recurses into array `items`
-        schemas, so array-item secrets (accounts[].token) are NOT routed to
-        config_secrets.json by these endpoints. The canonical module handles
-        them. When an endpoint migrates onto the module that behavior
-        changes (a fix, but a deliberate one).
-
-        If this fails, an inline copy has grown array support — duplicating
-        the canonical module even harder. Migrate the endpoint onto
-        src/web_interface/secret_helpers instead.
-        """
-        for body in self._inline_bodies("find_secret_fields"):
-            # Array handling requires checking type == 'array'; no inline
-            # copy does. (Can't grep bare "items" — properties.items() the
-            # dict method appears legitimately.)
-            assert "'array'" not in body and '"array"' not in body
-
-    @staticmethod
-    def _inline_bodies(name: str):
-        """Extract each inline def's body from api_v3.py by indentation."""
-        lines = API_V3_PATH.read_text(encoding="utf-8").splitlines()
-        bodies = []
-        i = 0
-        while i < len(lines):
-            match = re.match(rf"^(\s+)def {name}\(", lines[i])
-            if not match:
-                i += 1
-                continue
-            indent = len(match.group(1))
-            body = [lines[i]]
-            i += 1
-            while i < len(lines):
-                line = lines[i]
-                if line.strip() and (len(line) - len(line.lstrip())) <= indent:
-                    break
-                body.append(line)
-                i += 1
-            bodies.append("\n".join(body))
-        assert bodies, f"no inline {name} definitions found"
-        return bodies
+    def test_canonical_import_present(self):
+        # Tripwire: the endpoints still need the helpers, so removing the
+        # import means either dead secret handling or a new local copy.
+        source = API_V3_PATH.read_text(encoding="utf-8")
+        assert re.search(
+            r"from src\.web_interface\.secret_helpers import .*find_secret_fields",
+            source,
+        ), "api_v3.py no longer imports the canonical secret helpers"
 
 
 class TestCanonicalArrayItemBehavior:
-    """Executable documentation of what migrating endpoints will change."""
+    """Executable documentation of the array-item secret contract the
+    endpoints now inherit from the canonical module."""
 
     SCHEMA = {
         "accounts": {