refactor(plugins): the display publishes plugin runtime state; retire plugin_state.json (#690)

Stage 2 of the web plugin catalog, after #688.

- The display publishes a plugin runtime snapshot (plugin_runtime.py) to
  the shared cache: per plugin loaded, lifecycle state, a short redacted
  error summary, the version it loaded and when, plus published_at /
  stale_after / running. Written on change (throttled to 10 s; the
  RUNNING/ENABLED flip of an ordinary update is not a change) and once a
  minute otherwise; cleanup() publishes running: false.
- The web reads it back and restores loaded / state / error_info in
  /api/v3/plugins/installed (plus loaded_version, loaded_at and
  data.runtime). Only a live snapshot counts; stale, stopped or missing
  answers null and says which.
- data/plugin_state.json is retired: every reader and writer moved to
  config + disk (desired) or the snapshot (observed). Nothing in it was
  non-derivable, so nothing is migrated and an existing file is left
  unread. The web-side PluginStateManager (state_manager.py) is removed;
  the display's plugin_state.PluginStateManager is the only state machine.
- StateReconciliation compares config + disk with the snapshot, reporting
  enabled-but-not-loaded and older-version-loaded as no_action findings.
- Backups list installed manifests with enabled from config.json.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

web_interface/app.py

@@ -39,7 +39,6 @@ from src.plugin_system.store_manager import PluginStoreManager
 from src.plugin_system.saved_repositories import SavedRepositoriesManager
 from src.plugin_system.schema_manager import SchemaManager
 from src.plugin_system.operation_queue import PluginOperationQueue
-from src.plugin_system.state_manager import PluginStateManager
 from src.plugin_system.operation_history import OperationHistory
 
 _JOURNALCTL = shutil.which('journalctl')
@@ -158,13 +157,10 @@ plugin_catalog = PluginCatalog(
 # Initialize operation queue for plugin operations
 operation_queue = PluginOperationQueue(max_history=500)
 
-# Initialize plugin state manager
-# Use lazy_load=True to defer file loading until first use (improves startup time)
-plugin_state_manager = PluginStateManager(
-    state_file=str(project_root / "data" / "plugin_state.json"),
-    auto_save=True,
-    lazy_load=True
-)
+# No plugin state file: data/plugin_state.json is retired. Desired state is
+# config.json plus the plugins on disk, observed state is the runtime
+# snapshot the display publishes (src/plugin_system/plugin_runtime.py). An
+# existing file is left where it is, unread; see docs/ARCHITECTURE.md.
 
 # Initialize operation history
 # Use lazy_load=True to defer file loading until first use (improves startup time)
@@ -194,7 +190,6 @@ api_v3.plugin_store_manager = plugin_store_manager
 api_v3.saved_repositories_manager = saved_repositories_manager
 api_v3.schema_manager = schema_manager
 api_v3.operation_queue = operation_queue
-api_v3.plugin_state_manager = plugin_state_manager
 api_v3.operation_history = operation_history
 # Initialize cache manager for API endpoints
 from src.cache_manager import CacheManager
@@ -965,12 +960,12 @@ def _run_startup_reconciliation() -> None:
 
     try:
         from src.plugin_system.state_reconciliation import StateReconciliation
+        from src.plugin_system.plugin_runtime import read_plugin_runtime
         reconciler = StateReconciliation(
-            state_manager=plugin_state_manager,
             config_manager=config_manager,
-            plugin_manager=plugin_catalog,
             plugins_dir=plugins_dir,
-            store_manager=plugin_store_manager
+            store_manager=plugin_store_manager,
+            runtime_source=lambda: read_plugin_runtime(api_v3.cache_manager),
         )
         result = reconciler.reconcile_state()
         if result.inconsistencies_found:

web_interface/blueprints/api_v3/__init__.py

@@ -720,8 +720,6 @@ def _do_transactional_uninstall(plugin_id, preserve_config):
     # --- Step 4: finish ---
     if api_v3.schema_manager:
         api_v3.schema_manager.invalidate_cache(plugin_id)
-    if api_v3.plugin_state_manager:
-        api_v3.plugin_state_manager.remove_plugin_state(plugin_id)
     # Persistently record the uninstall so a later core `git pull` update
     # cannot resurrect a built-in plugin (committed under plugin-repos/) that
     # the user removed. Best-effort: never fail the uninstall over this.
@@ -730,6 +728,17 @@ def _do_transactional_uninstall(plugin_id, preserve_config):
     except Exception as record_err:
         logger.warning("Could not record uninstall for %s: %s", plugin_id, record_err)
     return True, None
+def _plugin_runtime_view():
+    """What the display publishes about its plugins (loaded, lifecycle
+    state, last error, version loaded), judged for staleness.
+
+    Only a ``live`` view reports those facts; a stale, stopped or missing
+    snapshot answers None for them (see src/plugin_system/plugin_runtime.py).
+    """
+    from src.plugin_system.plugin_runtime import read_plugin_runtime
+    return read_plugin_runtime(getattr(api_v3, 'cache_manager', None))
+
+
 def _plugin_enabled_in_config(plugin_id: str) -> bool:
     """Whether config.json enables ``plugin_id``, by the display's rule.
 

web_interface/blueprints/api_v3/plugin_operations.py

@@ -8,6 +8,7 @@ from web_interface.blueprints.api_v3 import (
     exception_error_response, json, jsonify, logger, os, request, stat,
     success_response, tempfile,
 )
+import web_interface.blueprints.api_v3 as _pkg
 
 
 @api_v3.route('/plugins/operation/<operation_id>', methods=['GET'])
@@ -81,54 +82,102 @@ def clear_operation_history() -> Response:
     return success_response(message='Operation history cleared')
 
 
+def _reconciler(store_manager=None):
+    """A StateReconciliation over config + disk (desired) and the display's
+    runtime snapshot (observed); None when the catalog is not set up."""
+    if not api_v3.plugin_catalog or not api_v3.config_manager:
+        return None
+    from src.plugin_system.state_reconciliation import StateReconciliation
+    return StateReconciliation(
+        config_manager=api_v3.config_manager,
+        plugins_dir=Path(api_v3.plugin_catalog.plugins_dir),
+        store_manager=store_manager,
+        runtime_source=_pkg._plugin_runtime_view,
+    )
+
+
+def _operation_times(plugin_ids):
+    """``installed_at`` / ``last_updated`` per plugin from the operation
+    history: the newest successful install, and the newest successful
+    install or update. None where the history has no record (it keeps the
+    last 1000 operations, and can be cleared)."""
+    times = {pid: {'installed_at': None, 'last_updated': None} for pid in plugin_ids}
+    history = getattr(api_v3, 'operation_history', None)
+    if not history or not times:
+        return times
+    try:
+        records = history.get_history(limit=100000)
+    except Exception:
+        logger.debug("[PluginState] Could not read the operation history", exc_info=True)
+        return times
+    for record in records:  # newest first
+        entry = times.get(record.plugin_id)
+        if entry is None or record.status != 'success':
+            continue
+        when = record.timestamp.isoformat()
+        if record.operation_type == 'install' and entry['installed_at'] is None:
+            entry['installed_at'] = when
+        if record.operation_type in ('install', 'update') and entry['last_updated'] is None:
+            entry['last_updated'] = when
+    return times
+
+
+def _plugin_state_status(record):
+    """The old plugin_state.json ``status`` vocabulary, derived."""
+    if record.get('state') == 'error':
+        return 'error'
+    if not record.get('installed'):
+        return 'unknown'
+    return 'enabled' if record.get('enabled') else 'disabled'
+
+
 @api_v3.route('/plugins/state', methods=['GET'])
 def get_plugin_state():
-    """Get plugin state from state manager"""
+    """Every plugin's state: desired (config + disk) and observed (the
+    display's runtime snapshot).
+
+    Built on each request -- there is no state file any more
+    (data/plugin_state.json is retired). ``loaded``, ``state``,
+    ``error_info``, ``loaded_version`` and ``loaded_at`` are null unless the
+    display's snapshot is live; ``runtime`` beside ``data`` says whether it is.
+    """
     try:
-        if not api_v3.plugin_state_manager:
+        reconciler = _reconciler()
+        if reconciler is None:
             return error_response(
                 ErrorCode.SYSTEM_ERROR,
-                'State manager not initialized',
+                'Plugin catalog not initialized',
                 status_code=500
             )
 
-        plugin_id = request.args.get('plugin_id')
+        states = reconciler.plugin_states()
+        times = _operation_times(states.keys())
+        for plugin_id, record in states.items():
+            record['status'] = _plugin_state_status(record)
+            record.update(times.get(plugin_id, {}))
+        runtime = _pkg._plugin_runtime_view().describe()
 
+        plugin_id = request.args.get('plugin_id')
         if plugin_id:
-            # Get state for specific plugin
-            state = api_v3.plugin_state_manager.get_plugin_state(plugin_id)
+            state = states.get(plugin_id)
             if not state:
                 return error_response(
                     ErrorCode.PLUGIN_NOT_FOUND,
-                    f'Plugin {plugin_id} not found in state manager',
+                    f'Plugin {plugin_id} is neither installed nor configured',
                     context={'plugin_id': plugin_id},
                     status_code=404
                 )
-            return success_response(data=state.to_dict())
-        else:
-            # Get all plugin states
-            all_states = api_v3.plugin_state_manager.get_all_states()
-            return success_response(data={
-                plugin_id: state.to_dict()
-                for plugin_id, state in all_states.items()
-            })
+            return success_response(data=state, extra={'runtime': runtime})
+        return success_response(data=states, extra={'runtime': runtime})
     except Exception as e:
         return exception_error_response(e, ErrorCode.SYSTEM_ERROR)
 
 
 @api_v3.route('/plugins/state/reconcile', methods=['POST'])
 def reconcile_plugin_state():
-    """Reconcile plugin state across all sources"""
+    """Reconcile desired state (config + disk) with what is installed and
+    what the display reports running."""
     try:
-        if not api_v3.plugin_state_manager or not api_v3.plugin_catalog:
-            return error_response(
-                ErrorCode.SYSTEM_ERROR,
-                'State manager or plugin catalog not initialized',
-                status_code=500
-            )
-
-        from src.plugin_system.state_reconciliation import StateReconciliation
-
         # Parse optional `force` flag from request body, guarding against
         # non-dict bodies (bare string, array, null) that would raise AttributeError.
         payload = request.get_json(silent=True)
@@ -136,12 +185,13 @@ def reconcile_plugin_state():
             payload = {}
         force = _coerce_to_bool(payload.get('force', False))
 
-        reconciler = StateReconciliation(
-            state_manager=api_v3.plugin_state_manager,
-            config_manager=api_v3.config_manager,
-            plugin_manager=api_v3.plugin_catalog,
-            plugins_dir=Path(api_v3.plugin_catalog.plugins_dir)
-        )
+        reconciler = _reconciler()
+        if reconciler is None:
+            return error_response(
+                ErrorCode.SYSTEM_ERROR,
+                'Config manager or plugin catalog not initialized',
+                status_code=500
+            )
 
         result = reconciler.reconcile_state(force=force)
 

web_interface/blueprints/api_v3/plugin_store.py

@@ -8,7 +8,7 @@ from web_interface.blueprints.api_v3 import (
     ErrorCode, OperationType, Path, _do_transactional_uninstall,
     _non_plugin_id_error, _get_plugin_version, _plugin_directory,
     _plugin_enabled_in_config, _store_restart_fields, api_v3,
-    datetime, error_response, exception_error_response, json, jsonify, logger,
+    error_response, exception_error_response, json, jsonify, logger,
     request, success_response, validate_request_json,
 )
 from src.common.path_safety import resolve_under, safe_path_component
@@ -236,12 +236,8 @@ def update_plugin():
             if api_v3.plugin_catalog:
                 api_v3.plugin_catalog.discover_plugins()
 
-            # Update state and history
-            if api_v3.plugin_state_manager:
-                api_v3.plugin_state_manager.update_plugin_state(
-                    plugin_id,
-                    {'last_updated': datetime.now()}
-                )
+            # Record in history (the only record of when it was updated;
+            # the version is the manifest on disk).
             if api_v3.operation_history:
                 version = _get_plugin_version(plugin_id)
                 api_v3.operation_history.record_operation(
@@ -467,10 +463,6 @@ def install_plugin():
                 if api_v3.plugin_catalog:
                     api_v3.plugin_catalog.discover_plugins()
 
-                # Update state manager
-                if api_v3.plugin_state_manager:
-                    api_v3.plugin_state_manager.set_plugin_installed(plugin_id)
-
                 # Record in history
                 if api_v3.operation_history:
                     version = _get_plugin_version(plugin_id)
@@ -529,8 +521,6 @@ def install_plugin():
                 api_v3.schema_manager.invalidate_cache(plugin_id)
             if api_v3.plugin_catalog:
                 api_v3.plugin_catalog.discover_plugins()
-            if api_v3.plugin_state_manager:
-                api_v3.plugin_state_manager.set_plugin_installed(plugin_id)
             if api_v3.operation_history:
                 version = _get_plugin_version(plugin_id)
                 api_v3.operation_history.record_operation(

web_interface/blueprints/api_v3/plugins.py

@@ -47,10 +47,13 @@ def get_installed_plugins():
     """Get installed plugins.
 
     Metadata comes from the plugin catalog (manifests on disk), ``enabled``
-    from config.json. ``loaded``, ``state`` and ``error_info`` are always
-    null: they would describe the display process's plugin instances, and
-    the display does not publish which plugins it has loaded. What it does
-    publish -- health, metrics, errors -- is served by /plugins/health,
+    from config.json. ``loaded``, ``state``, ``error_info``,
+    ``loaded_version`` and ``loaded_at`` come from the runtime snapshot the
+    display publishes (src/plugin_system/plugin_runtime.py), and only while
+    that snapshot is live: when the display is stopped, hung or has never
+    published, they are null and ``data.runtime.status`` says why
+    (``stale``, ``stopped``, ``unknown``) instead of passing on old truth.
+    Health, metrics and errors are served by /plugins/health,
     /plugins/metrics and /errors.
     """
     if not api_v3.plugin_catalog or not api_v3.plugin_store_manager:
@@ -65,6 +68,8 @@ def get_installed_plugins():
 
     # Load config once before the loop (not per-plugin)
     full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {}
+    # One read of the display's snapshot for the whole listing.
+    runtime = _pkg._plugin_runtime_view()
 
     def _build_plugin_entry(plugin_info):
         plugin_id = plugin_info.get('id')
@@ -154,10 +159,9 @@ def get_installed_plugins():
             'icon': plugin_info.get('icon') if isinstance(plugin_info.get('icon'), str) else None,
             'enabled': enabled,
             'verified': verified,
-            # Not published by the display process; see the docstring.
-            'loaded': None,
-            'state': None,
-            'error_info': None,
+            # loaded, state, error_info, loaded_version, loaded_at: the
+            # display's snapshot, null unless it is live (see the docstring).
+            **runtime.plugin(plugin_id),
             'last_updated': last_updated,
             'last_commit': last_commit,
             'last_commit_message': last_commit_message,
@@ -175,7 +179,8 @@ def get_installed_plugins():
     plugins = [r for r in results if r is not None]
     plugins.extend(_starlark_virtual_plugins())
 
-    return jsonify({'status': 'success', 'data': {'plugins': plugins}})
+    return jsonify({'status': 'success', 'data': {'plugins': plugins,
+                                                  'runtime': runtime.describe()}})
 
 
 @api_v3.route('/plugins/toggle', methods=['POST'])
@@ -247,10 +252,6 @@ def toggle_plugin():
                 status_code=500
             )
 
-        # Update state manager if available
-        if api_v3.plugin_state_manager:
-            api_v3.plugin_state_manager.set_plugin_enabled(plugin_id, enabled)
-
         # Log operation
         if api_v3.operation_history:
             api_v3.operation_history.record_operation(