perf: import package re-exports on first use (#724)

src/common/__init__.py and src/plugin_system/__init__.py resolve their re-exports lazily (PEP 562 __getattr__, __all__ and __dir__ unchanged, TYPE_CHECKING imports for mypy), and sync_manager imports numpy only where send_frame uses it. The web process no longer loads numpy, freetype helpers and PluginManager just to import path_safety, store_manager or schema_manager (~67 MB to ~54 MB RSS on a Pi 4). from src.common import X and from src.plugin_system import X keep working, including submodule imports.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -281,6 +281,27 @@ accepts both, but the store flags the old spelling as deprecated
   `scripts/render_bench.py` records the same. Diagnostic only: nothing tunes,
   freezes or disables the collector.
 
+### Web interface: lighter package imports
+
+- `src.common` and `src.plugin_system` now import their re-exported names on
+  first use (PEP 562 module `__getattr__`) instead of in `__init__.py`.
+  `from src.common import ScrollHelper`, `src.plugin_system.PluginManager`,
+  `from src.common import *` and every submodule import work as before and
+  return the same objects. What changes is that importing a submodule --
+  the web interface's `src.common.path_safety`, `src.plugin_system.store_manager`
+  and the like -- no longer loads `ScrollHelper`, `LogoHelper`, `APIHelper`,
+  the adaptive layout helpers and `PluginManager` with it. `sync_manager`
+  imports numpy inside `send_frame`, the one place it uses it, since the API
+  blueprint imports that module only for its constants.
+- The web process no longer loads numpy at all. On a Pi 4 (Python 3.13),
+  importing `web_interface.app` went from ~67 MB to ~54 MB RSS and from
+  ~2.8 s to ~1.3 s (`-X importtime`, median of five). A bare
+  `import src.common` went from ~50 MB / ~0.85 s to ~10 MB / ~30 ms. The
+  display process loads the same modules as before, only later.
+- A misspelt name in `from src.common import ...` still raises `ImportError`.
+  `test/test_lazy_package_imports.py` checks that the packages import nothing
+  heavy and that every name in `__all__` resolves to its home module's object.
+
 ### Outlined text: one rasterization
 
 - New `draw_text_outlined(draw, xy, text, font, fill, outline_color=(0, 0,

src/common/README.md

@@ -17,7 +17,9 @@ Rules for the package:
 - `from src.common import ...` re-exports `APIHelper`, `ScrollHelper`,
   `LogoHelper`, `TextHelper`, `scroll_config` (plus `ScrollSettings`,
   `configure_scroll`, `resolve_scroll_settings`, `refresh_hz_from_config`) and
-  the adaptive layout names below ([`__init__.py`](__init__.py)).
+  the adaptive layout names below ([`__init__.py`](__init__.py)). Each is
+  imported on first use, so `import src.common` or a submodule import stays
+  cheap; add a new re-export to `_LAZY` there as well as `__all__`.
 
 ## Summary
 

src/common/__init__.py

@@ -6,45 +6,90 @@ This package provides reusable functionality for plugins and core modules:
 - Logo helpers
 - Text/scroll helpers
 - Adaptive layout and image helpers
+
+The names below are imported on first use (PEP 562), not when the package is
+imported. ``from src.common import ScrollHelper`` and
+``src.common.ScrollHelper`` work as before and return the same objects, but
+``import src.common`` -- or importing any submodule, such as
+``src.common.path_safety`` -- no longer loads numpy, requests and freetype
+along with every helper. The web interface imports src.common only for a few
+small modules and never needs those.
 """
 
-# Export commonly used utilities
-from src.common.api_helper import APIHelper
-from src.common.scroll_helper import ScrollHelper
-from src.common import scroll_config
-from src.common.scroll_config import (
-    ScrollSettings,
-    configure as configure_scroll,
-    resolve as resolve_scroll_settings,
-    refresh_hz_from_config,
-)
-from src.common.logo_helper import LogoHelper
-from src.common.text_helper import TextHelper
+import importlib
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
-# Adaptive layout & images (canonical homes: src.adaptive_layout /
-# src.adaptive_images — re-exported here so plugin authors find them in the
-# blessed-helpers package). See docs/ADAPTIVE_LAYOUT.md.
-from src.adaptive_layout import (
-    Region,
-    LayoutContext,
-    FontStep,
-    FontLadder,
-    LADDER_GRID,
-    LADDER_ARCADE,
-    FitResult,
-    draw_fitted_text,
-    ScoreboardRegions,
-    scoreboard_regions,
-    MediaRow,
-    media_row,
-)
-from src.adaptive_images import (
-    ImageFitResult,
-    fit_image,
-    draw_fitted_image,
-    RESAMPLE_LANCZOS,
-    RESAMPLE_NEAREST,
-)
+if TYPE_CHECKING:
+    # What mypy and editors see: the real names and their types.
+    from src.common.api_helper import APIHelper
+    from src.common.scroll_helper import ScrollHelper
+    from src.common import scroll_config
+    from src.common.scroll_config import (
+        ScrollSettings,
+        configure as configure_scroll,
+        resolve as resolve_scroll_settings,
+        refresh_hz_from_config,
+    )
+    from src.common.logo_helper import LogoHelper
+    from src.common.text_helper import TextHelper
+
+    # Adaptive layout & images (canonical homes: src.adaptive_layout /
+    # src.adaptive_images — re-exported here so plugin authors find them in the
+    # blessed-helpers package). See docs/ADAPTIVE_LAYOUT.md.
+    from src.adaptive_layout import (
+        Region,
+        LayoutContext,
+        FontStep,
+        FontLadder,
+        LADDER_GRID,
+        LADDER_ARCADE,
+        FitResult,
+        draw_fitted_text,
+        ScoreboardRegions,
+        scoreboard_regions,
+        MediaRow,
+        media_row,
+    )
+    from src.adaptive_images import (
+        ImageFitResult,
+        fit_image,
+        draw_fitted_image,
+        RESAMPLE_LANCZOS,
+        RESAMPLE_NEAREST,
+    )
+
+#: Exported name -> (module it lives in, attribute name there). An attribute
+#: of None means the name is the module itself. Keep in step with the
+#: TYPE_CHECKING imports above and with __all__.
+_LAZY: Dict[str, Tuple[str, Optional[str]]] = {
+    'APIHelper': ('src.common.api_helper', 'APIHelper'),
+    'ScrollHelper': ('src.common.scroll_helper', 'ScrollHelper'),
+    'scroll_config': ('src.common.scroll_config', None),
+    'ScrollSettings': ('src.common.scroll_config', 'ScrollSettings'),
+    'configure_scroll': ('src.common.scroll_config', 'configure'),
+    'resolve_scroll_settings': ('src.common.scroll_config', 'resolve'),
+    'refresh_hz_from_config': ('src.common.scroll_config', 'refresh_hz_from_config'),
+    'LogoHelper': ('src.common.logo_helper', 'LogoHelper'),
+    'TextHelper': ('src.common.text_helper', 'TextHelper'),
+    # adaptive layout & images
+    'Region': ('src.adaptive_layout', 'Region'),
+    'LayoutContext': ('src.adaptive_layout', 'LayoutContext'),
+    'FontStep': ('src.adaptive_layout', 'FontStep'),
+    'FontLadder': ('src.adaptive_layout', 'FontLadder'),
+    'LADDER_GRID': ('src.adaptive_layout', 'LADDER_GRID'),
+    'LADDER_ARCADE': ('src.adaptive_layout', 'LADDER_ARCADE'),
+    'FitResult': ('src.adaptive_layout', 'FitResult'),
+    'draw_fitted_text': ('src.adaptive_layout', 'draw_fitted_text'),
+    'ScoreboardRegions': ('src.adaptive_layout', 'ScoreboardRegions'),
+    'scoreboard_regions': ('src.adaptive_layout', 'scoreboard_regions'),
+    'MediaRow': ('src.adaptive_layout', 'MediaRow'),
+    'media_row': ('src.adaptive_layout', 'media_row'),
+    'ImageFitResult': ('src.adaptive_images', 'ImageFitResult'),
+    'fit_image': ('src.adaptive_images', 'fit_image'),
+    'draw_fitted_image': ('src.adaptive_images', 'draw_fitted_image'),
+    'RESAMPLE_LANCZOS': ('src.adaptive_images', 'RESAMPLE_LANCZOS'),
+    'RESAMPLE_NEAREST': ('src.adaptive_images', 'RESAMPLE_NEAREST'),
+}
 
 __all__ = [
     'APIHelper',
@@ -75,3 +120,25 @@ __all__ = [
     'RESAMPLE_LANCZOS',
     'RESAMPLE_NEAREST',
 ]
+
+
+def __getattr__(name: str) -> Any:
+    """Import an exported name on first access (PEP 562).
+
+    Only called for names not already in the module namespace, so after the
+    first access the cached value below is returned directly. Unknown names
+    raise AttributeError, which ``from src.common import <submodule>`` relies
+    on to fall through to importing the submodule.
+    """
+    try:
+        module_name, attr = _LAZY[name]
+    except KeyError:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+    module = importlib.import_module(module_name)  # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import -- module_name comes from the fixed _LAZY table
+    value = module if attr is None else getattr(module, attr)
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> List[str]:
+    return sorted(set(globals()) | set(__all__))

src/common/sync_manager.py

@@ -29,7 +29,6 @@ import time
 import logging
 from enum import Enum
 from typing import Callable, Optional
-import numpy as np
 from PIL import Image
 
 from src.config_manager_atomic import _replace
@@ -434,6 +433,12 @@ class DisplaySyncManager:
             return
         if self._leader_state != LeaderState.CONNECTED or not self._peer_ip:
             return
+        # numpy is imported here, not at module level: the web interface
+        # imports this module for its constants (STATUS_FILE, SYNC_PORT) and
+        # would otherwise load numpy for nothing. Only a connected leader
+        # gets this far, and after the first frame the import is a
+        # sys.modules lookup.
+        import numpy as np
         try:
             arr = np.asarray(image.convert("RGB"), dtype=np.uint8)
             header = _RAW_MAGIC + _RAW_HEADER.pack(image.width, image.height)

src/plugin_system/__init__.py

@@ -3,15 +3,46 @@ LEDMatrix Plugin System
 
 This module provides the core plugin infrastructure for the LEDMatrix project.
 It enables dynamic loading, management, and discovery of display plugins.
+
+BasePlugin and PluginManager are imported on first use (PEP 562), not when
+the package is imported: the web interface imports several submodules
+(store_manager, schema_manager, ...) and never needs PluginManager, which
+pulls in the loader, executor and the shared helpers behind them.
+``from src.plugin_system import BasePlugin`` works as before and returns the
+same class.
 """
 
+import importlib
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple
+
 __version__ = "1.0.0"
 
-from .base_plugin import BasePlugin
-from .plugin_manager import PluginManager
+if TYPE_CHECKING:
+    from .base_plugin import BasePlugin
+    from .plugin_manager import PluginManager
+
+#: Exported name -> (module it lives in, attribute name there).
+_LAZY: Dict[str, Tuple[str, str]] = {
+    'BasePlugin': ('src.plugin_system.base_plugin', 'BasePlugin'),
+    'PluginManager': ('src.plugin_system.plugin_manager', 'PluginManager'),
+}
 
 __all__ = [
     'BasePlugin',
     'PluginManager',
 ]
 
+
+def __getattr__(name: str) -> Any:
+    """Import an exported name on first access (PEP 562); see src.common."""
+    try:
+        module_name, attr = _LAZY[name]
+    except KeyError:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+    value = getattr(importlib.import_module(module_name), attr)  # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import -- module_name comes from the fixed _LAZY table
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> List[str]:
+    return sorted(set(globals()) | set(__all__))

test/test_lazy_package_imports.py

@@ -0,0 +1,120 @@
+"""src.common and src.plugin_system import their exports lazily (PEP 562).
+
+The web interface imports both packages only for small submodules
+(path_safety, snapshot_policy, store_manager, ...). When their __init__
+imported every export eagerly, that dragged numpy, freetype and the plugin
+manager into a process that never uses them -- about 13 MB of RSS on a Pi.
+
+These tests pin both halves of the change: the package import stays light,
+and every exported name still resolves to the very object its home module
+defines, so ``isinstance`` and ``is`` checks behave as before.
+"""
+
+import importlib
+import json
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+#: Modules the bare package import must not load. numpy is the one that
+#: matters for memory; the rest are what the eager __init__ used to import.
+HEAVY = (
+    "numpy",
+    "freetype",
+    "src.adaptive_layout",
+    "src.common.api_helper",
+    "src.common.logo_helper",
+    "src.common.scroll_helper",
+    "src.plugin_system.base_plugin",
+    "src.plugin_system.plugin_manager",
+)
+
+
+def _run(script):
+    result = subprocess.run(
+        [sys.executable, "-c", textwrap.dedent(script)], cwd=str(REPO_ROOT),
+        capture_output=True, text=True, timeout=120)
+    assert result.returncode == 0, result.stderr
+    return json.loads(result.stdout.strip().splitlines()[-1])
+
+
+@pytest.mark.parametrize("package", ["src.common", "src.plugin_system"])
+def test_package_import_loads_nothing_heavy(package):
+    # A fresh interpreter: this test process has long since imported them.
+    loaded = _run(f"""
+        import json, sys
+        import {package}
+        print(json.dumps(sorted(m for m in {HEAVY!r} if m in sys.modules)))
+    """)
+    assert loaded == [], f"importing {package} loaded {loaded}"
+
+
+def test_web_interface_submodules_do_not_load_numpy():
+    # The imports web_interface/app.py and its API blueprint make from these
+    # packages. numpy here costs the web process ~13 MB for nothing.
+    loaded = _run("""
+        import json, sys
+        from src.common import path_safety, snapshot_policy, sync_manager
+        from src.plugin_system import store_manager, schema_manager
+        print(json.dumps("numpy" in sys.modules))
+    """)
+    assert loaded is False
+
+
+@pytest.mark.parametrize("package", ["src.common", "src.plugin_system"])
+def test_every_exported_name_resolves_to_its_home_object(package):
+    pkg = importlib.import_module(package)
+    assert sorted(pkg.__all__) == sorted(pkg._LAZY), "__all__ and _LAZY differ"
+    for name in pkg.__all__:
+        module_name, attr = pkg._LAZY[name]
+        home = importlib.import_module(module_name)
+        expected = home if attr is None else getattr(home, attr)
+        assert getattr(pkg, name) is expected, name
+        assert name in dir(pkg)
+
+
+def test_from_import_forms_plugins_use():
+    # Every form found in ledmatrix-plugins and core: names, aliases,
+    # submodules through the package, dotted submodule imports.
+    from src.common import ScrollHelper, LogoHelper
+    from src.common import scroll_config as _scroll_config
+    from src.common import sports_card as _card
+    from src.common import draw_fitted_text
+    from src.plugin_system import BasePlugin, PluginManager
+    from src.plugin_system import compatibility
+    import src.common.scroll_helper
+    import src.plugin_system.base_plugin
+    import src.common
+    import src.plugin_system
+
+    assert ScrollHelper is src.common.scroll_helper.ScrollHelper
+    assert LogoHelper is src.common.LogoHelper
+    assert _scroll_config is src.common.scroll_config
+    assert _card is importlib.import_module("src.common.sports_card")
+    assert draw_fitted_text is importlib.import_module("src.adaptive_layout").draw_fitted_text
+    assert BasePlugin is src.plugin_system.base_plugin.BasePlugin
+    assert PluginManager is importlib.import_module(
+        "src.plugin_system.plugin_manager").PluginManager
+    assert compatibility is importlib.import_module("src.plugin_system.compatibility")
+    assert src.plugin_system.__version__ == "1.0.0"
+
+
+def test_star_import_still_binds_everything():
+    namespace = {}
+    exec("from src.common import *", namespace)
+    import src.common
+    assert set(src.common.__all__) <= set(namespace)
+
+
+@pytest.mark.parametrize("package", ["src.common", "src.plugin_system"])
+def test_unknown_name_raises_attribute_error(package):
+    pkg = importlib.import_module(package)
+    with pytest.raises(AttributeError, match="no_such_name"):
+        pkg.no_such_name  # noqa: B018
+    with pytest.raises(ImportError):
+        exec(f"from {package} import no_such_name")