Merge origin/main (#592, #593, #594) into the September 16 core audit fixes

config.py conflict: #594 discovers plugins before /config/main matches plugin
keys; this branch routes those sections through the /plugins/config
preparation (F13). Kept both.

#594's secret-separation test posts a plugin section to /config/main, which
this branch now validates through api_v3.schema_manager; the test gets a real
SchemaManager over its temporary plugins directory instead of the helper's
mock. It still fails if the discovery call is reverted.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

web_interface/blueprints/api_v3/__init__.py

@@ -568,21 +568,41 @@ def _installed_plugin_ids():
     enumerate the installed plugins and read each one's persisted summary by ID
     instead of relying on the tracker's in-memory `get_all_*` view.
     """
-    pm = api_v3.plugin_manager
-    manifests = getattr(pm, 'plugin_manifests', None)
-    if not manifests:
-        # Only pay for a discovery scan when we haven't discovered anything yet;
-        # subsequent polls reuse the already-populated manifest map.
-        try:
-            pm.discover_plugins()
-        except Exception:
-            logger.debug('discover_plugins failed while listing plugin ids', exc_info=True)
-        manifests = getattr(pm, 'plugin_manifests', None)
+    manifests = _discovered_plugin_manifests()
     try:
         return list(manifests.keys()) if manifests else []
     except Exception:
         logger.debug('listing plugin_manifests failed while building plugin ids', exc_info=True)
         return []
+def _discovered_plugin_manifests(plugin_id=None, rescan=False):
+    """The plugin manager's manifests, discovering plugins first if needed.
+
+    The web process discovers plugins lazily (see app.py): nothing scans at
+    startup, so plugin_manifests is empty until some endpoint calls
+    discover_plugins(). A route that looks a plugin up without coming through
+    here answers "not found" for every installed plugin until something else
+    has run -- after a web restart, POST /display/on-demand/start returned 404
+    for minutes on a real rig, and only API-only callers ever noticed.
+
+    Scans when nothing is discovered yet, when ``plugin_id`` is given and not
+    among the manifests (it may have been installed since the last scan), or
+    when ``rescan`` is set (for lookups that are not by id, such as a mode).
+    Otherwise the existing map is reused, so a steady stream of requests
+    for known plugins costs nothing.
+
+    Returns the manifest map, or {} when there is no plugin manager.
+    """
+    pm = api_v3.plugin_manager
+    if pm is None:
+        return {}
+    manifests = getattr(pm, 'plugin_manifests', None)
+    if not manifests or rescan or (plugin_id is not None and plugin_id not in manifests):
+        try:
+            pm.discover_plugins()
+        except Exception:
+            logger.warning('Plugin discovery failed', exc_info=True)
+        manifests = getattr(pm, 'plugin_manifests', None)
+    return manifests or {}
 def _do_transactional_uninstall(plugin_id, preserve_config):
     """Execute an uninstall with snapshot-based rollback.
 

web_interface/blueprints/api_v3/config.py

@@ -1001,9 +1001,14 @@ def save_main_config():
         # flagged degraded at its next load).
         plugin_keys_to_remove = []
         plugin_secrets_updates = {}
+        # Discovered first: a plugin key not recognised here skips secret
+        # separation below and falls through to the generic merge, which wrote
+        # the plugin's API key into config.json in plain text whenever nothing
+        # had yet discovered plugins in this process (any save after a restart).
+        plugin_manifests = _pkg._discovered_plugin_manifests()
         for key in data:
             # Check if this key is a plugin ID
-            if api_v3.plugin_manager and key in api_v3.plugin_manager.plugin_manifests:
+            if api_v3.plugin_manager and key in plugin_manifests:
                 plugin_id = key
                 submitted_config = data[key]
                 if not isinstance(submitted_config, dict):

web_interface/blueprints/api_v3/display.py

@@ -138,7 +138,11 @@ def get_on_demand_status():
     """Return the current on-demand display state."""
     try:
         cache = _ensure_cache_manager()
-        state = cache.get('display_on_demand_state', max_age=120)
+        # memory_ttl=0: the display service writes this key, so only the file
+        # is current. This process's memory tier would keep serving the first
+        # copy it read for the full max_age -- "active" for two minutes after
+        # the display had already stopped.
+        state = cache.get('display_on_demand_state', max_age=120, memory_ttl=0)
         if state is None:
             state = {
                 'active': False,
@@ -174,14 +178,20 @@ def start_on_demand_display():
         resolved_mode = mode
 
         if api_v3.plugin_manager:
-            if resolved_plugin and resolved_plugin not in api_v3.plugin_manager.plugin_manifests:
+            if resolved_plugin and resolved_plugin not in _pkg._discovered_plugin_manifests(resolved_plugin):
                 return jsonify({'status': 'error', 'message': f'Plugin {resolved_plugin} not found'}), 404
 
             if resolved_plugin and not resolved_mode:
                 modes = api_v3.plugin_manager.get_plugin_display_modes(resolved_plugin)
                 resolved_mode = modes[0] if modes else resolved_plugin
             elif resolved_mode and not resolved_plugin:
+                _pkg._discovered_plugin_manifests()
                 resolved_plugin = api_v3.plugin_manager.find_plugin_for_mode(resolved_mode)
+                if not resolved_plugin:
+                    # Not among what was discovered: the plugin that declares
+                    # it may have been installed since. Scan once more.
+                    _pkg._discovered_plugin_manifests(rescan=True)
+                    resolved_plugin = api_v3.plugin_manager.find_plugin_for_mode(resolved_mode)
                 if not resolved_plugin:
                     return jsonify({'status': 'error', 'message': f'Mode {resolved_mode} not found'}), 404
 
@@ -304,7 +314,8 @@ def get_current_display_status():
     """
     try:
         cache = _ensure_cache_manager()
-        state = cache.get('display_current_state', max_age=120)
+        # memory_ttl=0: written by the display service; see get_on_demand_status.
+        state = cache.get('display_current_state', max_age=120, memory_ttl=0)
         if state is None:
             state = {
                 'mode': None,

web_interface/blueprints/api_v3/plugins.py

@@ -453,7 +453,7 @@ def toggle_plugin():
             return _toggle_starlark_app(plugin_id[len('starlark:'):], enabled)
 
         # Check if plugin exists in manifests (discovered but may not be loaded)
-        if plugin_id not in api_v3.plugin_manager.plugin_manifests:
+        if plugin_id not in _pkg._discovered_plugin_manifests(plugin_id):
             return jsonify({'status': 'error', 'message': 'Plugin not found'}), 404
 
         # Update config (this is what the display controller reads)