Merge audit16-core/configsave into the September 16 core audit fixes

# Conflicts:
#	src/plugin_system/plugin_manager.py

src/config_manager.py

@@ -29,6 +29,7 @@ import os
 import logging
 from pathlib import Path
 from typing import Dict, Any, Optional, List
+from src.core_config_keys import CORE_CONFIG_KEYS, CORE_SECRETS_KEYS
 from src.exceptions import ConfigError
 from src.logging_config import get_logger
 from src.config_manager_atomic import (
@@ -756,12 +757,13 @@ class ConfigManager:
             
             valid_set = set(valid_plugin_ids)
             
-            # Find orphaned plugins in main config
-            main_plugins = set(main_config.keys())
+            # Find orphaned plugins in main config. Core sections (display,
+            # schedule, auto_update, ...) are not plugins and never orphans.
+            main_plugins = set(main_config.keys()) - CORE_CONFIG_KEYS
             orphaned_main = main_plugins - valid_set
             
             # Find orphaned plugins in secrets config
-            secrets_plugins = set(secrets_config.keys())
+            secrets_plugins = set(secrets_config.keys()) - CORE_CONFIG_KEYS - CORE_SECRETS_KEYS
             orphaned_secrets = secrets_plugins - valid_set
             
             all_orphaned = orphaned_main | orphaned_secrets
@@ -815,8 +817,8 @@ class ConfigManager:
                 if not isinstance(plugin_config, dict):
                     continue
                 
-                # Skip non-plugin config sections
-                if plugin_id in ['display', 'schedule', 'timezone', 'plugin_system']:
+                # Skip core config sections
+                if plugin_id in CORE_CONFIG_KEYS:
                     continue
                 
                 schema = plugin_schema_manager.load_schema(plugin_id, use_cache=True)

src/core_config_keys.py

@@ -41,3 +41,13 @@ CORE_CONFIG_KEYS = frozenset({
     'vegas_excluded_plugins',
     'vegas_scroll_enabled',
 })
+
+#: Top-level keys of ``config_secrets.json`` that belong to the core rather than
+#: to a plugin: the GitHub token the Plugin Store reads, and the historical
+#: ``youtube`` section. Plugin secrets are namespaced by plugin id, so anything
+#: deciding whether a secrets section is a plugin's needs this as well as
+#: ``CORE_CONFIG_KEYS``.
+CORE_SECRETS_KEYS = frozenset({
+    'github',
+    'youtube',
+})

src/display_controller.py

@@ -2976,6 +2976,14 @@ class DisplayController:
                                        _pid: str = plugin_id, _plugin: Any = plugin_instance) -> None:
                 """Callback for plugin config changes."""
                 try:
+                    # ConfigService hands over the raw config.json section.
+                    # Prepare it as loading did (legacy booleans read as
+                    # objects, schema defaults filled in), so the plugin gets
+                    # the same shape it was constructed with.
+                    prepare = getattr(self.plugin_manager, 'prepare_plugin_config', None)
+                    prepared = prepare(_pid, new_config) if callable(prepare) else None
+                    if isinstance(prepared, dict):
+                        new_config = prepared
                     _plugin.on_config_change(new_config)
                     logger.debug("Plugin %s notified of config change", _pid)
                 except Exception as e:

src/plugin_system/plugin_manager.py

@@ -22,7 +22,9 @@ from src.logging_config import get_logger
 from src.plugin_system.plugin_loader import PluginLoader
 from src.plugin_system.plugin_executor import PluginExecutor
 from src.plugin_system.plugin_state import PluginStateManager, PluginState
-from src.plugin_system.schema_manager import SchemaManager, normalize_legacy_booleans
+from src.plugin_system.schema_manager import (
+    CORE_VEGAS_TUNING_KEYS, SchemaManager, normalize_legacy_booleans,
+)
 from src.common.path_safety import safe_path_component
 from src.common.permission_utils import (
     ensure_directory_permissions,
@@ -370,32 +372,11 @@ class PluginManager:
                         f"The schema may be invalid. Please verify the schema file at: {schema_path}"
                     )
 
-            # A plugin that turned an on/off boolean into an {enabled, ...}
-            # object still finds the boolean in config.json until the user saves
-            # its settings form, which carries it over (plugin_config.html).
-            # Read it the same way here, before the defaults fill in the rest
-            # of the object and before schema validation, so the plugin doesn't
-            # start with a schema warning and a degraded flag. In memory only:
-            # config.json is written by saves, never by loading a plugin.
-            if schema:
-                upgraded: List[str] = []
-                config = normalize_legacy_booleans(config, schema, upgraded)
-                if upgraded:
-                    self.logger.info(
-                        "Plugin %s: reading legacy boolean setting %s as "
-                        "{\"enabled\": ...}; saving the plugin's settings "
-                        "stores the new shape",
-                        plugin_id, ", ".join(upgraded),
-                    )
-
-            # Merge config with schema defaults to ensure all defaults are applied
-            try:
-                defaults = self.schema_manager.generate_default_config(plugin_id, use_cache=True)
-                config = self.schema_manager.merge_with_defaults(config, defaults)
-                self.logger.debug(f"Merged config with schema defaults for {plugin_id}")
-            except Exception as e:
-                self.logger.warning(f"Could not apply schema defaults for {plugin_id}: {e}")
-                # Continue with original config if defaults can't be applied
+            # Legacy booleans read as objects, then schema defaults: the same
+            # preparation saves, GET /plugins/config and hot reload apply
+            # (prepare_plugin_config). In memory only: config.json is written
+            # by saves, never by loading a plugin.
+            config = self.prepare_plugin_config(plugin_id, config, schema=schema)
             
             # Use PluginLoader to load plugin
             plugin_instance, module = self.plugin_loader.load_plugin(
@@ -490,11 +471,55 @@ class PluginManager:
     #:
     #: Read by: ``vegas_mode/plugin_adapter.py`` (``vegas_width_pct``,
     #: ``vegas_overflow``) and ``base_plugin.py`` (``vegas_max_width_screens``).
-    CORE_OWNED_CONFIG_KEYS = frozenset({
-        'vegas_width_pct',
-        'vegas_overflow',
-        'vegas_max_width_screens',
-    })
+    #:
+    #: The list itself lives with the other core-owned per-plugin properties in
+    #: ``schema_manager.CORE_PLUGIN_PROPERTIES``, which the web save path also
+    #: uses to keep these keys.
+    CORE_OWNED_CONFIG_KEYS = CORE_VEGAS_TUNING_KEYS
+
+    def prepare_plugin_config(self, plugin_id: str, config: Any,
+                              schema: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+        """The config a plugin runs with, built from its raw config.json section.
+
+        A plugin that turned an on/off boolean into an ``{enabled, ...}``
+        object still finds the boolean in config.json until its settings are
+        next saved; it is read as the object, and schema defaults fill in the
+        rest (``SchemaManager.prepare_plugin_config``). Used when loading a
+        plugin and on hot reload (DisplayController), so ``on_config_change``
+        receives the same shape the plugin was constructed with.
+
+        Never raises: on failure the legacy-boolean pass alone is applied, or
+        failing that the section is returned as it was.
+        """
+        if schema is None:
+            try:
+                schema = self.schema_manager.load_schema(plugin_id)
+            except Exception as e:
+                self.logger.debug("Could not load schema for %s: %s", plugin_id, e)
+                schema = None
+        upgraded: List[str] = []
+        try:
+            prepared = self.schema_manager.prepare_plugin_config(
+                plugin_id, config, schema=schema, changed_paths=upgraded)
+            self.logger.debug("Merged config with schema defaults for %s", plugin_id)
+        except Exception as e:
+            self.logger.warning("Could not apply schema defaults for %s: %s", plugin_id, e)
+            # Continue without defaults if they can't be applied
+            upgraded = []
+            prepared = config if isinstance(config, dict) else {}
+            if schema:
+                try:
+                    prepared = normalize_legacy_booleans(prepared, schema, upgraded)
+                except Exception:
+                    pass
+        if upgraded:
+            self.logger.info(
+                "Plugin %s: reading legacy boolean setting %s as "
+                "{\"enabled\": ...}; saving the plugin's settings "
+                "stores the new shape",
+                plugin_id, ", ".join(upgraded),
+            )
+        return prepared
 
     def _strip_core_owned_keys(self, config: Dict[str, Any]) -> Dict[str, Any]:
         """A shallow copy of ``config`` without the core's own tuning keys.

src/plugin_system/schema_manager.py

@@ -13,6 +13,8 @@ from typing import Any, Dict, List, Optional, Tuple
 import jsonschema
 from jsonschema import Draft7Validator, ValidationError
 
+from src.core_config_keys import CORE_CONFIG_KEYS
+
 
 def _renders_as_object(prop: Dict[str, Any]) -> bool:
     """``field_type == 'object'`` as ``plugin_config.html`` computes it.
@@ -93,6 +95,222 @@ def normalize_legacy_booleans(config: Any, schema: Any,
     return result
 
 
+#: Per-plugin settings the **core** owns: it reads them out of each plugin's
+#: config section, so they are allowed in every plugin's config whether or not
+#: the plugin's schema declares them. The one list for validation, for the web
+#: save filter and for the load-time checks -- a private copy is how JSON saves
+#: came to drop ``skin`` and the ``vegas_*`` keys while the validator accepted
+#: them.
+#:
+#: Values are the schema used when the plugin does not declare the property.
+CORE_PLUGIN_PROPERTIES: Dict[str, Dict[str, Any]] = {
+    # Defaults match BasePlugin behavior: enabled=True, display_duration=15,
+    # live_priority=False.
+    "enabled": {
+        "type": "boolean",
+        "default": True,
+        "description": "Enable or disable this plugin"
+    },
+    "display_duration": {
+        "type": "number",
+        "default": 15,
+        "minimum": 1,
+        "maximum": 300,
+        "description": "How long to display this plugin in seconds"
+    },
+    "live_priority": {
+        "type": "boolean",
+        "default": False,
+        "description": "Enable live priority takeover when plugin has live content"
+    },
+    # Skin selection (docs/SKIN_SYSTEM.md). Deliberately NOT an enum here:
+    # validation must keep passing when a configured skin gets uninstalled
+    # (rendering falls back to built-in). The install-dependent enum is
+    # injected only at serve time (inject_skin_selector) for the web UI
+    # dropdown.
+    "skin": {
+        "type": ["string", "object", "null"],
+        "description": "Visual skin id, or a per-mode mapping like {\"live\": \"my-skin\"}"
+    },
+    "skin_options": {
+        "type": "object",
+        "description": "Options passed through to the selected skin"
+    },
+    # Vegas tuning read by vegas_mode/plugin_adapter.py and base_plugin.py.
+    # Left untyped: the adapter validates them itself and ignores a bad
+    # value with a log line, so a stored one must never block a save.
+    "vegas_width_pct": {
+        "description": "Vegas mode: width of this plugin's card, as a percentage of the panel"
+    },
+    "vegas_overflow": {
+        "description": "Vegas mode: 'rotate' or 'truncate' when this plugin's content overflows"
+    },
+    "vegas_max_width_screens": {
+        "description": "Vegas mode: widest this plugin's card may be, in screens"
+    },
+}
+
+#: The keys of CORE_PLUGIN_PROPERTIES that are Vegas tuning rather than plugin
+#: state. PluginManager strips these before its soft validation (see
+#: PluginManager.CORE_OWNED_CONFIG_KEYS).
+CORE_VEGAS_TUNING_KEYS = frozenset({
+    'vegas_width_pct', 'vegas_overflow', 'vegas_max_width_screens',
+})
+
+
+def with_core_plugin_properties(schema: Dict[str, Any]) -> Dict[str, Any]:
+    """A deep copy of a plugin schema with CORE_PLUGIN_PROPERTIES allowed.
+
+    Properties the plugin declares itself are left as declared. Core
+    properties are removed from ``required``: they are system-managed.
+    """
+    enhanced = copy.deepcopy(schema) if isinstance(schema, dict) else {}
+    properties = enhanced.setdefault("properties", {})
+    for name, definition in CORE_PLUGIN_PROPERTIES.items():
+        if name not in properties:
+            properties[name] = copy.deepcopy(definition)
+    if "required" in enhanced:
+        enhanced["required"] = [field for field in enhanced["required"]
+                                if field not in CORE_PLUGIN_PROPERTIES]
+    return enhanced
+
+
+def extract_schema_defaults(schema: Dict[str, Any]) -> Dict[str, Any]:
+    """Default values of a JSON Schema's properties, recursively.
+
+    A property's own ``default`` wins; otherwise a nested object contributes
+    its children's defaults, and an array contributes ``[]`` (or a one-item
+    list of its ``items`` default). This is what a device runs with, so the
+    dev tools use it too (src/plugin_system/testing/loading.py).
+    """
+    defaults: Dict[str, Any] = {}
+    properties = schema.get('properties', {}) if isinstance(schema, dict) else {}
+    if not isinstance(properties, dict):
+        return defaults
+
+    for key, prop_schema in properties.items():
+        if not isinstance(prop_schema, dict):
+            continue
+        # If property has a default, use it
+        if 'default' in prop_schema:
+            defaults[key] = prop_schema['default']
+            continue
+
+        # Handle nested objects
+        if prop_schema.get('type') == 'object' and 'properties' in prop_schema:
+            nested_defaults = extract_schema_defaults(prop_schema)
+            if nested_defaults:
+                defaults[key] = nested_defaults
+
+        # Handle arrays with object items
+        elif prop_schema.get('type') == 'array' and 'items' in prop_schema:
+            items_schema = prop_schema['items']
+            if items_schema.get('type') == 'object' and 'properties' in items_schema:
+                # For arrays of objects, use empty array as default
+                # Individual objects will use their defaults when created
+                defaults[key] = []
+            elif 'default' in items_schema:
+                # Array with default item value
+                defaults[key] = [items_schema['default']]
+            else:
+                # Empty array as default
+                defaults[key] = []
+
+        # For other types without defaults, don't add to defaults dict
+        # This allows plugins to handle missing values as needed
+
+    return defaults
+
+
+def plugin_config_defaults(schema: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+    """Every default a plugin's config gets: the schema's plus the core ones.
+
+    A plugin with no schema gets the minimal ``enabled: False,
+    display_duration: 15``. Device location is not applied here; that needs a
+    config manager (SchemaManager.generate_default_config).
+    """
+    if not schema:
+        return {
+            'enabled': False,
+            'display_duration': 15
+        }
+
+    defaults = extract_schema_defaults(schema)
+
+    # Ensure core properties have defaults (they may not be in the schema)
+    # These match BasePlugin behavior
+    for name in ('enabled', 'display_duration', 'live_priority'):
+        if name not in defaults:
+            defaults[name] = CORE_PLUGIN_PROPERTIES[name]['default']
+    return defaults
+
+
+def merge_config_defaults(config: Dict[str, Any], defaults: Dict[str, Any]) -> Dict[str, Any]:
+    """Merge configuration with defaults, preserving user values.
+
+    Also replaces None values with defaults so a config never starts with
+    None where a default exists. Neither argument is mutated.
+    """
+    merged = copy.deepcopy(defaults)
+
+    def deep_merge(target: Dict[str, Any], source: Dict[str, Any], default_dict: Dict[str, Any]) -> None:
+        """Recursively merge source into target, replacing None with defaults."""
+        for key, value in source.items():
+            default_value = default_dict.get(key)
+
+            if key in target and isinstance(target[key], dict) and isinstance(value, dict):
+                # Both are dicts, recursively merge
+                if isinstance(default_value, dict):
+                    deep_merge(target[key], value, default_value)
+                else:
+                    deep_merge(target[key], value, {})
+            elif value is None and default_value is not None:
+                # Value is None and we have a default, use the default
+                target[key] = copy.deepcopy(default_value) if isinstance(default_value, (dict, list)) else default_value
+            else:
+                # Normal merge: user value takes precedence (copy if dict/list)
+                if isinstance(value, (dict, list)):
+                    target[key] = copy.deepcopy(value)
+                else:
+                    target[key] = value
+
+    deep_merge(merged, config, defaults)
+
+    # Final pass: replace any remaining None values at any level with defaults
+    def replace_none_with_defaults(target: Dict[str, Any], default_dict: Dict[str, Any]) -> None:
+        """Recursively replace None values with defaults."""
+        for key in list(target.keys()):
+            value = target[key]
+            default_value = default_dict.get(key)
+
+            if value is None and default_value is not None:
+                # Replace None with default
+                target[key] = copy.deepcopy(default_value) if isinstance(default_value, (dict, list)) else default_value
+            elif isinstance(value, dict) and isinstance(default_value, dict):
+                # Recursively process nested dicts
+                replace_none_with_defaults(value, default_value)
+
+    replace_none_with_defaults(merged, defaults)
+    return merged
+
+
+def prepare_plugin_config(config: Any, schema: Optional[Dict[str, Any]],
+                          defaults: Dict[str, Any],
+                          changed_paths: Optional[List[str]] = None) -> Dict[str, Any]:
+    """The config a plugin runs with, from its stored (or submitted) section.
+
+    Legacy booleans are read as ``{"enabled": ...}`` objects
+    (normalize_legacy_booleans), then schema defaults fill in whatever is
+    missing. Loading a plugin, both config saves, GET /plugins/config, hot
+    reload and the dev tools all go through this, so a plugin sees the same
+    shape however its config reached it.
+    """
+    config = config if isinstance(config, dict) else {}
+    if schema:
+        config = normalize_legacy_booleans(config, schema, changed_paths)
+    return merge_config_defaults(config, defaults)
+
+
 class SchemaManager:
     """
     Manages plugin configuration schemas with caching and validation.
@@ -262,57 +480,12 @@ class SchemaManager:
     def extract_defaults_from_schema(self, schema: Dict[str, Any], prefix: str = '') -> Dict[str, Any]:
         """
         Recursively extract default values from a JSON Schema.
-        
-        Handles nested objects, arrays, and all schema types.
-        
-        Args:
-            schema: JSON Schema dictionary
-            prefix: Optional prefix for logging/debugging
-            
-        Returns:
-            Dictionary of default values
+
+        See :func:`extract_schema_defaults`; ``prefix`` is accepted for
+        compatibility and unused.
         """
-        defaults = {}
-        
-        # Handle schema with properties
-        properties = schema.get('properties', {})
-        if not properties:
-            return defaults
-        
-        for key, prop_schema in properties.items():
-            field_path = f"{prefix}.{key}" if prefix else key
-            
-            # If property has a default, use it
-            if 'default' in prop_schema:
-                defaults[key] = prop_schema['default']
-                self.logger.debug(f"Found default for {field_path}: {prop_schema['default']}")
-                continue
-            
-            # Handle nested objects
-            if prop_schema.get('type') == 'object' and 'properties' in prop_schema:
-                nested_defaults = self.extract_defaults_from_schema(prop_schema, field_path)
-                if nested_defaults:
-                    defaults[key] = nested_defaults
-            
-            # Handle arrays with object items
-            elif prop_schema.get('type') == 'array' and 'items' in prop_schema:
-                items_schema = prop_schema['items']
-                if items_schema.get('type') == 'object' and 'properties' in items_schema:
-                    # For arrays of objects, use empty array as default
-                    # Individual objects will use their defaults when created
-                    defaults[key] = []
-                elif 'default' in items_schema:
-                    # Array with default item value
-                    defaults[key] = [items_schema['default']]
-                else:
-                    # Empty array as default
-                    defaults[key] = []
-            
-            # For other types without defaults, don't add to defaults dict
-            # This allows plugins to handle missing values as needed
-        
-        return defaults
-    
+        return extract_schema_defaults(schema)
+
     def get_device_location(self) -> Optional[Dict[str, Any]]:
         """
         Return the device-wide ``location`` block from config.json, or None.
@@ -391,31 +564,39 @@ class SchemaManager:
         schema = self.load_schema(plugin_id, use_cache=use_cache)
         if not schema:
             # Return minimal defaults if no schema
-            return {
-                'enabled': False,
-                'display_duration': 15
-            }
-        
-        # Extract defaults from schema
-        defaults = self.extract_defaults_from_schema(schema)
-        
-        # Ensure core properties have defaults (they may not be in the schema)
-        # These match BasePlugin behavior
-        if 'enabled' not in defaults:
-            defaults['enabled'] = schema.get('properties', {}).get('enabled', {}).get('default', True)
-        
-        if 'display_duration' not in defaults:
-            defaults['display_duration'] = schema.get('properties', {}).get('display_duration', {}).get('default', 15)
-        
-        if 'live_priority' not in defaults:
-            defaults['live_priority'] = schema.get('properties', {}).get('live_priority', {}).get('default', False)
-        
+            return plugin_config_defaults(None)
+
+        # Schema defaults plus the core properties' (they may not be in the
+        # schema)
+        defaults = plugin_config_defaults(schema)
+
         # Cache the defaults *before* the device location is layered on, so a
         # later change to the device location is picked up by the next call.
         self._defaults_cache[plugin_id] = defaults.copy()
         
         return self.apply_device_location(defaults)
     
+    def prepare_plugin_config(self, plugin_id: str, config: Any,
+                              schema: Optional[Dict[str, Any]] = None,
+                              changed_paths: Optional[List[str]] = None) -> Dict[str, Any]:
+        """
+        The config a plugin runs with: see :func:`prepare_plugin_config`.
+
+        Args:
+            plugin_id: Plugin identifier
+            config: The plugin's stored or submitted config section
+            schema: The plugin's schema, when the caller already has it
+            changed_paths: Receives the dotted path of each legacy boolean
+                read as an object
+
+        Returns:
+            A new dict; ``config`` is not mutated
+        """
+        if schema is None:
+            schema = self.load_schema(plugin_id, use_cache=True)
+        defaults = self.generate_default_config(plugin_id, use_cache=True)
+        return prepare_plugin_config(config, schema, defaults, changed_paths)
+
     def validate_config_against_schema(self, config: Dict[str, Any], schema: Dict[str, Any], 
                                       plugin_id: Optional[str] = None) -> Tuple[bool, List[str]]:
         """
@@ -436,80 +617,18 @@ class SchemaManager:
         errors = []
         
         try:
-            # Core plugin properties that should always be allowed
-            # These are handled by the base plugin system and should not cause validation failures
-            # Defaults match BasePlugin behavior: enabled=True, display_duration=15, live_priority=False
-            core_properties = {
-                "enabled": {
-                    "type": "boolean",
-                    "default": True,
-                    "description": "Enable or disable this plugin"
-                },
-                "display_duration": {
-                    "type": "number",
-                    "default": 15,
-                    "minimum": 1,
-                    "maximum": 300,
-                    "description": "How long to display this plugin in seconds"
-                },
-                "live_priority": {
-                    "type": "boolean",
-                    "default": False,
-                    "description": "Enable live priority takeover when plugin has live content"
-                },
-                # Skin selection (docs/SKIN_SYSTEM.md). Deliberately NOT an
-                # enum here: validation must keep passing when a configured
-                # skin gets uninstalled (rendering falls back to built-in).
-                # The install-dependent enum is injected only at serve time
-                # (inject_skin_selector) for the web UI dropdown.
-                "skin": {
-                    "type": ["string", "object", "null"],
-                    "description": "Visual skin id, or a per-mode mapping like {\"live\": \"my-skin\"}"
-                },
-                "skin_options": {
-                    "type": "object",
-                    "description": "Options passed through to the selected skin"
-                }
-            }
-            
-            # Create a deep copy of the schema to modify (to avoid mutating the original)
-            enhanced_schema = copy.deepcopy(schema)
-            if "properties" not in enhanced_schema:
-                enhanced_schema["properties"] = {}
-            
-            # Inject core properties if they're not already defined in the schema
-            # This ensures core properties are always allowed even if not in the plugin's schema
-            properties_added = []
-            for prop_name, prop_def in core_properties.items():
-                if prop_name not in enhanced_schema["properties"]:
-                    enhanced_schema["properties"][prop_name] = copy.deepcopy(prop_def)
-                    properties_added.append(prop_name)
-            
-            # Log if we added any core properties (for debugging)
-            if properties_added and plugin_id:
+            # Core plugin properties (CORE_PLUGIN_PROPERTIES) are handled by
+            # the base plugin system and should not cause validation failures:
+            # they are allowed even when the plugin's schema doesn't declare
+            # them, and never required.
+            enhanced_schema = with_core_plugin_properties(schema)
+            if plugin_id:
+                declared = schema.get("properties", {}) if isinstance(schema, dict) else {}
                 self.logger.debug(
-                    f"Injected core properties into schema for {plugin_id}: {properties_added}"
+                    "Injected core properties into schema for %s: %s", plugin_id,
+                    [name for name in CORE_PLUGIN_PROPERTIES if name not in declared]
                 )
-            
-            # Remove core properties from required array (they're system-managed)
-            # Core properties should be allowed but not required for validation
-            if "required" in enhanced_schema:
-                core_prop_names = list(core_properties.keys())
-                removed_from_required = [
-                    field for field in enhanced_schema["required"]
-                    if field in core_prop_names
-                ]
-                enhanced_schema["required"] = [
-                    field for field in enhanced_schema["required"] 
-                    if field not in core_prop_names
-                ]
-                
-                # Log if we removed any core properties from required (for debugging)
-                if removed_from_required and plugin_id:
-                    self.logger.debug(
-                        f"Removed core properties from required array for {plugin_id}: {removed_from_required}"
-                    )
-            
+
             # Create validator with enhanced schema
             validator = Draft7Validator(enhanced_schema)
             
@@ -626,55 +745,15 @@ class SchemaManager:
         """
         Merge configuration with defaults, preserving user values.
         Also replaces None values with defaults to ensure config never has None from the start.
-        
+
         Args:
             config: User configuration
             defaults: Default values from schema
-            
+
         Returns:
             Merged configuration with defaults applied where missing or None
         """
-        merged = copy.deepcopy(defaults)
-        
-        def deep_merge(target: Dict[str, Any], source: Dict[str, Any], default_dict: Dict[str, Any]) -> None:
-            """Recursively merge source into target, replacing None with defaults."""
-            for key, value in source.items():
-                default_value = default_dict.get(key)
-                
-                if key in target and isinstance(target[key], dict) and isinstance(value, dict):
-                    # Both are dicts, recursively merge
-                    if isinstance(default_value, dict):
-                        deep_merge(target[key], value, default_value)
-                    else:
-                        deep_merge(target[key], value, {})
-                elif value is None and default_value is not None:
-                    # Value is None and we have a default, use the default
-                    target[key] = copy.deepcopy(default_value) if isinstance(default_value, (dict, list)) else default_value
-                else:
-                    # Normal merge: user value takes precedence (copy if dict/list)
-                    if isinstance(value, (dict, list)):
-                        target[key] = copy.deepcopy(value)
-                    else:
-                        target[key] = value
-        
-        deep_merge(merged, config, defaults)
-        
-        # Final pass: replace any remaining None values at any level with defaults
-        def replace_none_with_defaults(target: Dict[str, Any], default_dict: Dict[str, Any]) -> None:
-            """Recursively replace None values with defaults."""
-            for key in list(target.keys()):
-                value = target[key]
-                default_value = default_dict.get(key)
-                
-                if value is None and default_value is not None:
-                    # Replace None with default
-                    target[key] = copy.deepcopy(default_value) if isinstance(default_value, (dict, list)) else default_value
-                elif isinstance(value, dict) and isinstance(default_value, dict):
-                    # Recursively process nested dicts
-                    replace_none_with_defaults(value, default_value)
-        
-        replace_none_with_defaults(merged, defaults)
-        return merged
+        return merge_config_defaults(config, defaults)
 
     def detect_config_key_collisions(
         self,
@@ -698,10 +777,11 @@ class SchemaManager:
         """
         collisions = []
 
-        # Reserved top-level config keys that plugins should not use as IDs
-        reserved_keys = {
-            'display', 'schedule', 'timezone', 'plugin_system',
-            'display_modes', 'system', 'hardware', 'debug',
+        # Reserved top-level config keys that plugins should not use as IDs:
+        # every core section (src/core_config_keys.py), plus a few names that
+        # read as core even though no current section uses them.
+        reserved_keys = set(CORE_CONFIG_KEYS) | {
+            'display_modes', 'hardware', 'debug',
             'log_level', 'emulator', 'web_interface'
         }
 

src/plugin_system/testing/harness.py

@@ -27,7 +27,7 @@ from PIL import Image, ImageChops
 
 from src.logging_config import get_logger
 from .bounds_display_manager import BoundsCheckingDisplayManager
-from .loading import load_config_defaults, load_manifest, merge_config
+from .loading import build_config, load_manifest
 from .sizes import DEFAULT_TEST_SIZES, safe_mode_filename, size_label
 
 logger = get_logger("[Plugin Harness]")
@@ -305,9 +305,7 @@ def render_plugin_matrix(
     manifest = load_manifest(plugin_dir)
     # Start from config_schema.json defaults so the plugin behaves like a real
     # install; explicit caller config still wins over a schema default.
-    config = merge_config(
-        merge_config({"enabled": True}, load_config_defaults(plugin_dir)),
-        config or {})
+    config = build_config(plugin_dir, config)
     sizes = sizes or DEFAULT_TEST_SIZES
     results: List[RenderResult] = []
 

src/plugin_system/testing/loading.py

@@ -42,29 +42,6 @@ def load_manifest(plugin_dir: Union[str, Path]) -> Dict[str, Any]:
         return json.load(f)
 
 
-def _defaults_from_properties(properties: Dict[str, Any]) -> Dict[str, Any]:
-    """Defaults for one `properties` block, recursing into nested objects.
-
-    An object property carries its defaults on its children, not on itself, so
-    reading only the top level dropped everything nested. That is most of the
-    fleet: config organised by league, or under customization/display_options,
-    lost 2,386 defaults across 37 of 44 plugins -- soccer-scoreboard alone lost
-    539 of 565 -- and the harness rendered them with a config no install would
-    ever have.
-    """
-    defaults: Dict[str, Any] = {}
-    for key, prop in (properties or {}).items():
-        if not isinstance(prop, dict):
-            continue
-        if prop.get('type') == 'object' and isinstance(prop.get('properties'), dict):
-            nested = _defaults_from_properties(prop['properties'])
-            if nested:
-                defaults[key] = nested
-        elif 'default' in prop:
-            defaults[key] = prop['default']
-    return defaults
-
-
 def merge_config(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
     """Deep-merge override onto base, without dropping sibling defaults.
 
@@ -81,14 +58,45 @@ def merge_config(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, An
     return merged
 
 
-def load_config_defaults(plugin_dir: Union[str, Path]) -> Dict[str, Any]:
-    """Extract default values from a plugin's config_schema.json (empty if none)."""
+def load_schema(plugin_dir: Union[str, Path]) -> Optional[Dict[str, Any]]:
+    """A plugin's config_schema.json, or None when it has none."""
     schema_path = Path(plugin_dir) / 'config_schema.json'
     if not schema_path.exists():
-        return {}
+        return None
     with open(schema_path, 'r', encoding='utf-8') as f:
-        schema = json.load(f)
-    return _defaults_from_properties(schema.get('properties', {}))
+        return json.load(f)
+
+
+def load_config_defaults(plugin_dir: Union[str, Path]) -> Dict[str, Any]:
+    """Default values from a plugin's config_schema.json (empty if none).
+
+    The device's own extraction (schema_manager.extract_schema_defaults), so a
+    harness run starts from the config an install would have: nested objects
+    contribute their children's defaults (organised-by-league configs lost
+    thousands of them when only the top level was read), and arrays without a
+    default start as [].
+    """
+    from src.plugin_system.schema_manager import extract_schema_defaults
+    schema = load_schema(plugin_dir)
+    return extract_schema_defaults(schema) if schema else {}
+
+
+def build_config(plugin_dir: Union[str, Path],
+                 overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    """The config a device would give this plugin, with ``overrides`` applied.
+
+    Starts from a forced ``enabled: True``, deep-merges ``overrides`` onto it
+    (merge_config), then prepares the result exactly as the device does when it
+    loads a plugin: legacy booleans read as objects, and schema plus core
+    defaults filled in (schema_manager.prepare_plugin_config). Used by the
+    harness, check_plugin, render_plugin and the dev preview server.
+    """
+    from src.plugin_system.schema_manager import (
+        plugin_config_defaults, prepare_plugin_config,
+    )
+    schema = load_schema(plugin_dir)
+    requested = merge_config({"enabled": True}, overrides or {})
+    return prepare_plugin_config(requested, schema, plugin_config_defaults(schema))
 
 
 def load_harness_spec(plugin_dir: Union[str, Path]) -> Dict[str, Any]:
@@ -143,16 +151,14 @@ def build_full_config(
 
     Merge order: config_schema.json defaults, then a forced ``enabled: True``,
     then harness.json's config overlay, then the caller's explicit config --
-    most specific wins. `enabled` is re-asserted *after* the schema defaults
-    so a plugin that reasonably ships `enabled: false` (e.g. a seasonal or
-    opt-in plugin) can't silently make every harness run test "disabled, do
-    nothing" by accident -- callers that genuinely want to test the disabled
-    path can still do so via `cli_config={"enabled": False}`.
+    most specific wins, and each layer deep-merges (a nested override such as
+    ``{"nhl": {"enabled": true}}`` keeps the other nhl defaults). `enabled` is
+    asserted over the schema defaults so a plugin that reasonably ships
+    `enabled: false` (e.g. a seasonal or opt-in plugin) can't silently make
+    every harness run test "disabled, do nothing" by accident -- callers that
+    genuinely want to test the disabled path can still do so via
+    `cli_config={"enabled": False}`. See build_config.
     """
     spec = spec or {}
-    config: Dict[str, Any] = {}
-    config.update(load_config_defaults(plugin_dir))
-    config["enabled"] = True
-    config.update(spec.get("config", {}))
-    config.update(cli_config or {})
-    return config
+    overrides = merge_config(spec.get("config", {}) or {}, cli_config or {})
+    return build_config(plugin_dir, overrides)

src/startup_validator.py

@@ -8,6 +8,7 @@ Fails fast with clear error messages to prevent runtime issues.
 import os
 from typing import Any, List, Optional, Tuple
 from pathlib import Path
+from src.core_config_keys import CORE_CONFIG_KEYS
 from src.exceptions import ConfigError, PluginError, CacheError
 from src.logging_config import get_logger
 
@@ -275,8 +276,9 @@ class StartupValidator:
             
             # Check for enabled plugins that don't exist
             for plugin_id, plugin_config in config.items():
-                # Skip non-plugin config sections
-                if plugin_id in ['display', 'schedule', 'timezone', 'plugin_system']:
+                # Skip core sections: auto_update and dim_schedule have an
+                # 'enabled' key too, and are not plugins that went missing.
+                if plugin_id in CORE_CONFIG_KEYS:
                     continue
                 
                 if not isinstance(plugin_config, dict):