refactor(plugins): the display publishes plugin runtime state; retire plugin_state.json (#690)

Stage 2 of the web plugin catalog, after #688. - The display publishes a plugin runtime snapshot (plugin_runtime.py) to the shared cache: per plugin loaded, lifecycle state, a short redacted error summary, the version it loaded and when, plus published_at / stale_after / running. Written on change (throttled to 10 s; the RUNNING/ENABLED flip of an ordinary update is not a change) and once a minute otherwise; cleanup() publishes running: false. - The web reads it back and restores loaded / state / error_info in /api/v3/plugins/installed (plus loaded_version, loaded_at and data.runtime). Only a live snapshot counts; stale, stopped or missing answers null and says which. - data/plugin_state.json is retired: every reader and writer moved to config + disk (desired) or the snapshot (observed). Nothing in it was non-derivable, so nothing is migrated and an existing file is left unread. The web-side PluginStateManager (state_manager.py) is removed; the display's plugin_state.PluginStateManager is the only state machine. - StateReconciliation compares config + disk with the snapshot, reporting enabled-but-not-loaded and older-version-loaded as no_action findings. - Backups list installed manifests with enabled from config.json. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 14:25:08 +00:00 · 2026-09-30 10:48:14 -04:00
parent 7ab6fb1aff
commit b09434a418
38 changed files with 1893 additions and 940 deletions
@@ -720,8 +720,6 @@ def _do_transactional_uninstall(plugin_id, preserve_config):
    # --- Step 4: finish ---
    if api_v3.schema_manager:
        api_v3.schema_manager.invalidate_cache(plugin_id)
-    if api_v3.plugin_state_manager:
-        api_v3.plugin_state_manager.remove_plugin_state(plugin_id)
    # Persistently record the uninstall so a later core `git pull` update
    # cannot resurrect a built-in plugin (committed under plugin-repos/) that
    # the user removed. Best-effort: never fail the uninstall over this.
@@ -730,6 +728,17 @@ def _do_transactional_uninstall(plugin_id, preserve_config):
    except Exception as record_err:
        logger.warning("Could not record uninstall for %s: %s", plugin_id, record_err)
    return True, None
+def _plugin_runtime_view():
+    """What the display publishes about its plugins (loaded, lifecycle
+    state, last error, version loaded), judged for staleness.
+
+    Only a ``live`` view reports those facts; a stale, stopped or missing
+    snapshot answers None for them (see src/plugin_system/plugin_runtime.py).
+    """
+    from src.plugin_system.plugin_runtime import read_plugin_runtime
+    return read_plugin_runtime(getattr(api_v3, 'cache_manager', None))
+
+
 def _plugin_enabled_in_config(plugin_id: str) -> bool:
    """Whether config.json enables ``plugin_id``, by the display's rule.

@@ -8,6 +8,7 @@ from web_interface.blueprints.api_v3 import (
    exception_error_response, json, jsonify, logger, os, request, stat,
    success_response, tempfile,
 )
+import web_interface.blueprints.api_v3 as _pkg


@api_v3.route('/plugins/operation/<operation_id>', methods=['GET'])
@@ -81,54 +82,102 @@ def clear_operation_history() -> Response:
    return success_response(message='Operation history cleared')


+def _reconciler(store_manager=None):
+    """A StateReconciliation over config + disk (desired) and the display's
+    runtime snapshot (observed); None when the catalog is not set up."""
+    if not api_v3.plugin_catalog or not api_v3.config_manager:
+        return None
+    from src.plugin_system.state_reconciliation import StateReconciliation
+    return StateReconciliation(
+        config_manager=api_v3.config_manager,
+        plugins_dir=Path(api_v3.plugin_catalog.plugins_dir),
+        store_manager=store_manager,
+        runtime_source=_pkg._plugin_runtime_view,
+    )
+
+
+def _operation_times(plugin_ids):
+    """``installed_at`` / ``last_updated`` per plugin from the operation
+    history: the newest successful install, and the newest successful
+    install or update. None where the history has no record (it keeps the
+    last 1000 operations, and can be cleared)."""
+    times = {pid: {'installed_at': None, 'last_updated': None} for pid in plugin_ids}
+    history = getattr(api_v3, 'operation_history', None)
+    if not history or not times:
+        return times
+    try:
+        records = history.get_history(limit=100000)
+    except Exception:
+        logger.debug("[PluginState] Could not read the operation history", exc_info=True)
+        return times
+    for record in records:  # newest first
+        entry = times.get(record.plugin_id)
+        if entry is None or record.status != 'success':
+            continue
+        when = record.timestamp.isoformat()
+        if record.operation_type == 'install' and entry['installed_at'] is None:
+            entry['installed_at'] = when
+        if record.operation_type in ('install', 'update') and entry['last_updated'] is None:
+            entry['last_updated'] = when
+    return times
+
+
+def _plugin_state_status(record):
+    """The old plugin_state.json ``status`` vocabulary, derived."""
+    if record.get('state') == 'error':
+        return 'error'
+    if not record.get('installed'):
+        return 'unknown'
+    return 'enabled' if record.get('enabled') else 'disabled'
+
+
@api_v3.route('/plugins/state', methods=['GET'])
 def get_plugin_state():
-    """Get plugin state from state manager"""
+    """Every plugin's state: desired (config + disk) and observed (the
+    display's runtime snapshot).
+
+    Built on each request -- there is no state file any more
+    (data/plugin_state.json is retired). ``loaded``, ``state``,
+    ``error_info``, ``loaded_version`` and ``loaded_at`` are null unless the
+    display's snapshot is live; ``runtime`` beside ``data`` says whether it is.
+    """
    try:
-        if not api_v3.plugin_state_manager:
+        reconciler = _reconciler()
+        if reconciler is None:
            return error_response(
                ErrorCode.SYSTEM_ERROR,
-                'State manager not initialized',
+                'Plugin catalog not initialized',
                status_code=500
            )

-        plugin_id = request.args.get('plugin_id')
+        states = reconciler.plugin_states()
+        times = _operation_times(states.keys())
+        for plugin_id, record in states.items():
+            record['status'] = _plugin_state_status(record)
+            record.update(times.get(plugin_id, {}))
+        runtime = _pkg._plugin_runtime_view().describe()

+        plugin_id = request.args.get('plugin_id')
        if plugin_id:
-            # Get state for specific plugin
-            state = api_v3.plugin_state_manager.get_plugin_state(plugin_id)
+            state = states.get(plugin_id)
            if not state:
                return error_response(
                    ErrorCode.PLUGIN_NOT_FOUND,
-                    f'Plugin {plugin_id} not found in state manager',
+                    f'Plugin {plugin_id} is neither installed nor configured',
                    context={'plugin_id': plugin_id},
                    status_code=404
                )
-            return success_response(data=state.to_dict())
-        else:
-            # Get all plugin states
-            all_states = api_v3.plugin_state_manager.get_all_states()
-            return success_response(data={
-                plugin_id: state.to_dict()
-                for plugin_id, state in all_states.items()
-            })
+            return success_response(data=state, extra={'runtime': runtime})
+        return success_response(data=states, extra={'runtime': runtime})
    except Exception as e:
        return exception_error_response(e, ErrorCode.SYSTEM_ERROR)


@api_v3.route('/plugins/state/reconcile', methods=['POST'])
 def reconcile_plugin_state():
-    """Reconcile plugin state across all sources"""
+    """Reconcile desired state (config + disk) with what is installed and
+    what the display reports running."""
    try:
-        if not api_v3.plugin_state_manager or not api_v3.plugin_catalog:
-            return error_response(
-                ErrorCode.SYSTEM_ERROR,
-                'State manager or plugin catalog not initialized',
-                status_code=500
-            )
-
-        from src.plugin_system.state_reconciliation import StateReconciliation
-
        # Parse optional `force` flag from request body, guarding against
        # non-dict bodies (bare string, array, null) that would raise AttributeError.
        payload = request.get_json(silent=True)
@@ -136,12 +185,13 @@ def reconcile_plugin_state():
            payload = {}
        force = _coerce_to_bool(payload.get('force', False))

-        reconciler = StateReconciliation(
-            state_manager=api_v3.plugin_state_manager,
-            config_manager=api_v3.config_manager,
-            plugin_manager=api_v3.plugin_catalog,
-            plugins_dir=Path(api_v3.plugin_catalog.plugins_dir)
-        )
+        reconciler = _reconciler()
+        if reconciler is None:
+            return error_response(
+                ErrorCode.SYSTEM_ERROR,
+                'Config manager or plugin catalog not initialized',
+                status_code=500
+            )

        result = reconciler.reconcile_state(force=force)

@@ -8,7 +8,7 @@ from web_interface.blueprints.api_v3 import (
    ErrorCode, OperationType, Path, _do_transactional_uninstall,
    _non_plugin_id_error, _get_plugin_version, _plugin_directory,
    _plugin_enabled_in_config, _store_restart_fields, api_v3,
-    datetime, error_response, exception_error_response, json, jsonify, logger,
+    error_response, exception_error_response, json, jsonify, logger,
    request, success_response, validate_request_json,
 )
 from src.common.path_safety import resolve_under, safe_path_component
@@ -236,12 +236,8 @@ def update_plugin():
            if api_v3.plugin_catalog:
                api_v3.plugin_catalog.discover_plugins()

-            # Update state and history
-            if api_v3.plugin_state_manager:
-                api_v3.plugin_state_manager.update_plugin_state(
-                    plugin_id,
-                    {'last_updated': datetime.now()}
-                )
+            # Record in history (the only record of when it was updated;
+            # the version is the manifest on disk).
            if api_v3.operation_history:
                version = _get_plugin_version(plugin_id)
                api_v3.operation_history.record_operation(
@@ -467,10 +463,6 @@ def install_plugin():
                if api_v3.plugin_catalog:
                    api_v3.plugin_catalog.discover_plugins()

-                # Update state manager
-                if api_v3.plugin_state_manager:
-                    api_v3.plugin_state_manager.set_plugin_installed(plugin_id)
-
                # Record in history
                if api_v3.operation_history:
                    version = _get_plugin_version(plugin_id)
@@ -529,8 +521,6 @@ def install_plugin():
                api_v3.schema_manager.invalidate_cache(plugin_id)
            if api_v3.plugin_catalog:
                api_v3.plugin_catalog.discover_plugins()
-            if api_v3.plugin_state_manager:
-                api_v3.plugin_state_manager.set_plugin_installed(plugin_id)
            if api_v3.operation_history:
                version = _get_plugin_version(plugin_id)
                api_v3.operation_history.record_operation(
@@ -47,10 +47,13 @@ def get_installed_plugins():
    """Get installed plugins.

    Metadata comes from the plugin catalog (manifests on disk), ``enabled``
-    from config.json. ``loaded``, ``state`` and ``error_info`` are always
-    null: they would describe the display process's plugin instances, and
-    the display does not publish which plugins it has loaded. What it does
-    publish -- health, metrics, errors -- is served by /plugins/health,
+    from config.json. ``loaded``, ``state``, ``error_info``,
+    ``loaded_version`` and ``loaded_at`` come from the runtime snapshot the
+    display publishes (src/plugin_system/plugin_runtime.py), and only while
+    that snapshot is live: when the display is stopped, hung or has never
+    published, they are null and ``data.runtime.status`` says why
+    (``stale``, ``stopped``, ``unknown``) instead of passing on old truth.
+    Health, metrics and errors are served by /plugins/health,
    /plugins/metrics and /errors.
    """
    if not api_v3.plugin_catalog or not api_v3.plugin_store_manager:
@@ -65,6 +68,8 @@ def get_installed_plugins():

    # Load config once before the loop (not per-plugin)
    full_config = api_v3.config_manager.load_config() if api_v3.config_manager else {}
+    # One read of the display's snapshot for the whole listing.
+    runtime = _pkg._plugin_runtime_view()

    def _build_plugin_entry(plugin_info):
        plugin_id = plugin_info.get('id')
@@ -154,10 +159,9 @@ def get_installed_plugins():
            'icon': plugin_info.get('icon') if isinstance(plugin_info.get('icon'), str) else None,
            'enabled': enabled,
            'verified': verified,
-            # Not published by the display process; see the docstring.
-            'loaded': None,
-            'state': None,
-            'error_info': None,
+            # loaded, state, error_info, loaded_version, loaded_at: the
+            # display's snapshot, null unless it is live (see the docstring).
+            **runtime.plugin(plugin_id),
            'last_updated': last_updated,
            'last_commit': last_commit,
            'last_commit_message': last_commit_message,
@@ -175,7 +179,8 @@ def get_installed_plugins():
    plugins = [r for r in results if r is not None]
    plugins.extend(_starlark_virtual_plugins())

-    return jsonify({'status': 'success', 'data': {'plugins': plugins}})
+    return jsonify({'status': 'success', 'data': {'plugins': plugins,
+                                                  'runtime': runtime.describe()}})


@api_v3.route('/plugins/toggle', methods=['POST'])
@@ -247,10 +252,6 @@ def toggle_plugin():
                status_code=500
            )

-        # Update state manager if available
-        if api_v3.plugin_state_manager:
-            api_v3.plugin_state_manager.set_plugin_enabled(plugin_id, enabled)
-
        # Log operation
        if api_v3.operation_history:
            api_v3.operation_history.record_operation(