From ef69201770f2b9103e569108be90f42adddaffc4 Mon Sep 17 00:00:00 2001 From: Chuck <33324927+ChuckBuilds@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:38:27 -0400 Subject: [PATCH] perf: import package re-exports on first use (#724) src/common/__init__.py and src/plugin_system/__init__.py resolve their re-exports lazily (PEP 562 __getattr__, __all__ and __dir__ unchanged, TYPE_CHECKING imports for mypy), and sync_manager imports numpy only where send_frame uses it. The web process no longer loads numpy, freetype helpers and PluginManager just to import path_safety, store_manager or schema_manager (~67 MB to ~54 MB RSS on a Pi 4). from src.common import X and from src.plugin_system import X keep working, including submodule imports. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 21 +++++ src/common/README.md | 4 +- src/common/__init__.py | 139 ++++++++++++++++++++++-------- src/common/sync_manager.py | 7 +- src/plugin_system/__init__.py | 35 +++++++- test/test_lazy_package_imports.py | 120 ++++++++++++++++++++++++++ 6 files changed, 286 insertions(+), 40 deletions(-) create mode 100644 test/test_lazy_package_imports.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 52c5ffa4..8fa3a386 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -281,6 +281,27 @@ accepts both, but the store flags the old spelling as deprecated `scripts/render_bench.py` records the same. Diagnostic only: nothing tunes, freezes or disables the collector. +### Web interface: lighter package imports + +- `src.common` and `src.plugin_system` now import their re-exported names on + first use (PEP 562 module `__getattr__`) instead of in `__init__.py`. + `from src.common import ScrollHelper`, `src.plugin_system.PluginManager`, + `from src.common import *` and every submodule import work as before and + return the same objects. What changes is that importing a submodule -- + the web interface's `src.common.path_safety`, `src.plugin_system.store_manager` + and the like -- no longer loads `ScrollHelper`, `LogoHelper`, `APIHelper`, + the adaptive layout helpers and `PluginManager` with it. `sync_manager` + imports numpy inside `send_frame`, the one place it uses it, since the API + blueprint imports that module only for its constants. +- The web process no longer loads numpy at all. On a Pi 4 (Python 3.13), + importing `web_interface.app` went from ~67 MB to ~54 MB RSS and from + ~2.8 s to ~1.3 s (`-X importtime`, median of five). A bare + `import src.common` went from ~50 MB / ~0.85 s to ~10 MB / ~30 ms. The + display process loads the same modules as before, only later. +- A misspelt name in `from src.common import ...` still raises `ImportError`. + `test/test_lazy_package_imports.py` checks that the packages import nothing + heavy and that every name in `__all__` resolves to its home module's object. + ### Outlined text: one rasterization - New `draw_text_outlined(draw, xy, text, font, fill, outline_color=(0, 0, diff --git a/src/common/README.md b/src/common/README.md index a2bcd29a..9181db49 100644 --- a/src/common/README.md +++ b/src/common/README.md @@ -17,7 +17,9 @@ Rules for the package: - `from src.common import ...` re-exports `APIHelper`, `ScrollHelper`, `LogoHelper`, `TextHelper`, `scroll_config` (plus `ScrollSettings`, `configure_scroll`, `resolve_scroll_settings`, `refresh_hz_from_config`) and - the adaptive layout names below ([`__init__.py`](__init__.py)). + the adaptive layout names below ([`__init__.py`](__init__.py)). Each is + imported on first use, so `import src.common` or a submodule import stays + cheap; add a new re-export to `_LAZY` there as well as `__all__`. ## Summary diff --git a/src/common/__init__.py b/src/common/__init__.py index 9b3a8925..b6409bde 100644 --- a/src/common/__init__.py +++ b/src/common/__init__.py @@ -6,45 +6,90 @@ This package provides reusable functionality for plugins and core modules: - Logo helpers - Text/scroll helpers - Adaptive layout and image helpers + +The names below are imported on first use (PEP 562), not when the package is +imported. ``from src.common import ScrollHelper`` and +``src.common.ScrollHelper`` work as before and return the same objects, but +``import src.common`` -- or importing any submodule, such as +``src.common.path_safety`` -- no longer loads numpy, requests and freetype +along with every helper. The web interface imports src.common only for a few +small modules and never needs those. """ -# Export commonly used utilities -from src.common.api_helper import APIHelper -from src.common.scroll_helper import ScrollHelper -from src.common import scroll_config -from src.common.scroll_config import ( - ScrollSettings, - configure as configure_scroll, - resolve as resolve_scroll_settings, - refresh_hz_from_config, -) -from src.common.logo_helper import LogoHelper -from src.common.text_helper import TextHelper +import importlib +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple -# Adaptive layout & images (canonical homes: src.adaptive_layout / -# src.adaptive_images — re-exported here so plugin authors find them in the -# blessed-helpers package). See docs/ADAPTIVE_LAYOUT.md. -from src.adaptive_layout import ( - Region, - LayoutContext, - FontStep, - FontLadder, - LADDER_GRID, - LADDER_ARCADE, - FitResult, - draw_fitted_text, - ScoreboardRegions, - scoreboard_regions, - MediaRow, - media_row, -) -from src.adaptive_images import ( - ImageFitResult, - fit_image, - draw_fitted_image, - RESAMPLE_LANCZOS, - RESAMPLE_NEAREST, -) +if TYPE_CHECKING: + # What mypy and editors see: the real names and their types. + from src.common.api_helper import APIHelper + from src.common.scroll_helper import ScrollHelper + from src.common import scroll_config + from src.common.scroll_config import ( + ScrollSettings, + configure as configure_scroll, + resolve as resolve_scroll_settings, + refresh_hz_from_config, + ) + from src.common.logo_helper import LogoHelper + from src.common.text_helper import TextHelper + + # Adaptive layout & images (canonical homes: src.adaptive_layout / + # src.adaptive_images — re-exported here so plugin authors find them in the + # blessed-helpers package). See docs/ADAPTIVE_LAYOUT.md. + from src.adaptive_layout import ( + Region, + LayoutContext, + FontStep, + FontLadder, + LADDER_GRID, + LADDER_ARCADE, + FitResult, + draw_fitted_text, + ScoreboardRegions, + scoreboard_regions, + MediaRow, + media_row, + ) + from src.adaptive_images import ( + ImageFitResult, + fit_image, + draw_fitted_image, + RESAMPLE_LANCZOS, + RESAMPLE_NEAREST, + ) + +#: Exported name -> (module it lives in, attribute name there). An attribute +#: of None means the name is the module itself. Keep in step with the +#: TYPE_CHECKING imports above and with __all__. +_LAZY: Dict[str, Tuple[str, Optional[str]]] = { + 'APIHelper': ('src.common.api_helper', 'APIHelper'), + 'ScrollHelper': ('src.common.scroll_helper', 'ScrollHelper'), + 'scroll_config': ('src.common.scroll_config', None), + 'ScrollSettings': ('src.common.scroll_config', 'ScrollSettings'), + 'configure_scroll': ('src.common.scroll_config', 'configure'), + 'resolve_scroll_settings': ('src.common.scroll_config', 'resolve'), + 'refresh_hz_from_config': ('src.common.scroll_config', 'refresh_hz_from_config'), + 'LogoHelper': ('src.common.logo_helper', 'LogoHelper'), + 'TextHelper': ('src.common.text_helper', 'TextHelper'), + # adaptive layout & images + 'Region': ('src.adaptive_layout', 'Region'), + 'LayoutContext': ('src.adaptive_layout', 'LayoutContext'), + 'FontStep': ('src.adaptive_layout', 'FontStep'), + 'FontLadder': ('src.adaptive_layout', 'FontLadder'), + 'LADDER_GRID': ('src.adaptive_layout', 'LADDER_GRID'), + 'LADDER_ARCADE': ('src.adaptive_layout', 'LADDER_ARCADE'), + 'FitResult': ('src.adaptive_layout', 'FitResult'), + 'draw_fitted_text': ('src.adaptive_layout', 'draw_fitted_text'), + 'ScoreboardRegions': ('src.adaptive_layout', 'ScoreboardRegions'), + 'scoreboard_regions': ('src.adaptive_layout', 'scoreboard_regions'), + 'MediaRow': ('src.adaptive_layout', 'MediaRow'), + 'media_row': ('src.adaptive_layout', 'media_row'), + 'ImageFitResult': ('src.adaptive_images', 'ImageFitResult'), + 'fit_image': ('src.adaptive_images', 'fit_image'), + 'draw_fitted_image': ('src.adaptive_images', 'draw_fitted_image'), + 'RESAMPLE_LANCZOS': ('src.adaptive_images', 'RESAMPLE_LANCZOS'), + 'RESAMPLE_NEAREST': ('src.adaptive_images', 'RESAMPLE_NEAREST'), +} __all__ = [ 'APIHelper', @@ -75,3 +120,25 @@ __all__ = [ 'RESAMPLE_LANCZOS', 'RESAMPLE_NEAREST', ] + + +def __getattr__(name: str) -> Any: + """Import an exported name on first access (PEP 562). + + Only called for names not already in the module namespace, so after the + first access the cached value below is returned directly. Unknown names + raise AttributeError, which ``from src.common import `` relies + on to fall through to importing the submodule. + """ + try: + module_name, attr = _LAZY[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + module = importlib.import_module(module_name) # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import -- module_name comes from the fixed _LAZY table + value = module if attr is None else getattr(module, attr) + globals()[name] = value + return value + + +def __dir__() -> List[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/src/common/sync_manager.py b/src/common/sync_manager.py index 2a200525..ce43de34 100644 --- a/src/common/sync_manager.py +++ b/src/common/sync_manager.py @@ -29,7 +29,6 @@ import time import logging from enum import Enum from typing import Callable, Optional -import numpy as np from PIL import Image from src.config_manager_atomic import _replace @@ -434,6 +433,12 @@ class DisplaySyncManager: return if self._leader_state != LeaderState.CONNECTED or not self._peer_ip: return + # numpy is imported here, not at module level: the web interface + # imports this module for its constants (STATUS_FILE, SYNC_PORT) and + # would otherwise load numpy for nothing. Only a connected leader + # gets this far, and after the first frame the import is a + # sys.modules lookup. + import numpy as np try: arr = np.asarray(image.convert("RGB"), dtype=np.uint8) header = _RAW_MAGIC + _RAW_HEADER.pack(image.width, image.height) diff --git a/src/plugin_system/__init__.py b/src/plugin_system/__init__.py index 9032c599..79eb0313 100644 --- a/src/plugin_system/__init__.py +++ b/src/plugin_system/__init__.py @@ -3,15 +3,46 @@ LEDMatrix Plugin System This module provides the core plugin infrastructure for the LEDMatrix project. It enables dynamic loading, management, and discovery of display plugins. + +BasePlugin and PluginManager are imported on first use (PEP 562), not when +the package is imported: the web interface imports several submodules +(store_manager, schema_manager, ...) and never needs PluginManager, which +pulls in the loader, executor and the shared helpers behind them. +``from src.plugin_system import BasePlugin`` works as before and returns the +same class. """ +import importlib +from typing import TYPE_CHECKING, Any, Dict, List, Tuple + __version__ = "1.0.0" -from .base_plugin import BasePlugin -from .plugin_manager import PluginManager +if TYPE_CHECKING: + from .base_plugin import BasePlugin + from .plugin_manager import PluginManager + +#: Exported name -> (module it lives in, attribute name there). +_LAZY: Dict[str, Tuple[str, str]] = { + 'BasePlugin': ('src.plugin_system.base_plugin', 'BasePlugin'), + 'PluginManager': ('src.plugin_system.plugin_manager', 'PluginManager'), +} __all__ = [ 'BasePlugin', 'PluginManager', ] + +def __getattr__(name: str) -> Any: + """Import an exported name on first access (PEP 562); see src.common.""" + try: + module_name, attr = _LAZY[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + value = getattr(importlib.import_module(module_name), attr) # nosemgrep: python.lang.security.audit.non-literal-import.non-literal-import -- module_name comes from the fixed _LAZY table + globals()[name] = value + return value + + +def __dir__() -> List[str]: + return sorted(set(globals()) | set(__all__)) diff --git a/test/test_lazy_package_imports.py b/test/test_lazy_package_imports.py new file mode 100644 index 00000000..28195fba --- /dev/null +++ b/test/test_lazy_package_imports.py @@ -0,0 +1,120 @@ +"""src.common and src.plugin_system import their exports lazily (PEP 562). + +The web interface imports both packages only for small submodules +(path_safety, snapshot_policy, store_manager, ...). When their __init__ +imported every export eagerly, that dragged numpy, freetype and the plugin +manager into a process that never uses them -- about 13 MB of RSS on a Pi. + +These tests pin both halves of the change: the package import stays light, +and every exported name still resolves to the very object its home module +defines, so ``isinstance`` and ``is`` checks behave as before. +""" + +import importlib +import json +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + +#: Modules the bare package import must not load. numpy is the one that +#: matters for memory; the rest are what the eager __init__ used to import. +HEAVY = ( + "numpy", + "freetype", + "src.adaptive_layout", + "src.common.api_helper", + "src.common.logo_helper", + "src.common.scroll_helper", + "src.plugin_system.base_plugin", + "src.plugin_system.plugin_manager", +) + + +def _run(script): + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(script)], cwd=str(REPO_ROOT), + capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout.strip().splitlines()[-1]) + + +@pytest.mark.parametrize("package", ["src.common", "src.plugin_system"]) +def test_package_import_loads_nothing_heavy(package): + # A fresh interpreter: this test process has long since imported them. + loaded = _run(f""" + import json, sys + import {package} + print(json.dumps(sorted(m for m in {HEAVY!r} if m in sys.modules))) + """) + assert loaded == [], f"importing {package} loaded {loaded}" + + +def test_web_interface_submodules_do_not_load_numpy(): + # The imports web_interface/app.py and its API blueprint make from these + # packages. numpy here costs the web process ~13 MB for nothing. + loaded = _run(""" + import json, sys + from src.common import path_safety, snapshot_policy, sync_manager + from src.plugin_system import store_manager, schema_manager + print(json.dumps("numpy" in sys.modules)) + """) + assert loaded is False + + +@pytest.mark.parametrize("package", ["src.common", "src.plugin_system"]) +def test_every_exported_name_resolves_to_its_home_object(package): + pkg = importlib.import_module(package) + assert sorted(pkg.__all__) == sorted(pkg._LAZY), "__all__ and _LAZY differ" + for name in pkg.__all__: + module_name, attr = pkg._LAZY[name] + home = importlib.import_module(module_name) + expected = home if attr is None else getattr(home, attr) + assert getattr(pkg, name) is expected, name + assert name in dir(pkg) + + +def test_from_import_forms_plugins_use(): + # Every form found in ledmatrix-plugins and core: names, aliases, + # submodules through the package, dotted submodule imports. + from src.common import ScrollHelper, LogoHelper + from src.common import scroll_config as _scroll_config + from src.common import sports_card as _card + from src.common import draw_fitted_text + from src.plugin_system import BasePlugin, PluginManager + from src.plugin_system import compatibility + import src.common.scroll_helper + import src.plugin_system.base_plugin + import src.common + import src.plugin_system + + assert ScrollHelper is src.common.scroll_helper.ScrollHelper + assert LogoHelper is src.common.LogoHelper + assert _scroll_config is src.common.scroll_config + assert _card is importlib.import_module("src.common.sports_card") + assert draw_fitted_text is importlib.import_module("src.adaptive_layout").draw_fitted_text + assert BasePlugin is src.plugin_system.base_plugin.BasePlugin + assert PluginManager is importlib.import_module( + "src.plugin_system.plugin_manager").PluginManager + assert compatibility is importlib.import_module("src.plugin_system.compatibility") + assert src.plugin_system.__version__ == "1.0.0" + + +def test_star_import_still_binds_everything(): + namespace = {} + exec("from src.common import *", namespace) + import src.common + assert set(src.common.__all__) <= set(namespace) + + +@pytest.mark.parametrize("package", ["src.common", "src.plugin_system"]) +def test_unknown_name_raises_attribute_error(package): + pkg = importlib.import_module(package) + with pytest.raises(AttributeError, match="no_such_name"): + pkg.no_such_name # noqa: B018 + with pytest.raises(ImportError): + exec(f"from {package} import no_such_name")