From 371bc69bd8ba7d27a6f057f3e4f9a39b5d1f12ad Mon Sep 17 00:00:00 2001 From: ChuckBuilds Date: Fri, 4 Sep 2026 21:39:57 -0400 Subject: [PATCH] fix(scripts): run standalone plugin tests instead of collecting nothing run_plugin_tests.py discovered every plugin test file and handed the lot to pytest. Most plugin tests are standalone scripts -- module-level main() plus an `if __name__ == "__main__"` guard, signalling through an exit code -- and pytest collects zero items from those. The run printed how many files it had *found*, then "no tests ran", and exited without executing any of them. On a rig with all 44 first-party plugins that is 151 of 248 files. Files are now classified and each kind runs under the right runner: pytest for real test modules, subprocess for scripts, honouring the 0 pass / 2 skip / 1 fail convention ledmatrix-plugins' own runner established (a script that wants a tty or an LED matrix is a skip, not a regression). Before: $ python3 scripts/run_plugin_tests.py -p countdown -d ~/LEDMatrix/plugin-repos Found 1 test file(s) collected 0 items no tests ran in 0.31s rc=0 After: Found 1 test file(s) -- 0 collectable, 1 standalone script(s) 1 passed, 0 skipped, 0 failed (scripts) rc=0 Verified across three shapes: countdown (1 script), jellyfin-now-playing and pomodoro-timer (pytest only, 16 and 42 tests), and ledmatrix-flights (11 files split 4 collectable / 7 scripts, all seven of which had never run). Closes #532. Running the flights scripts for the first time also surfaced four genuinely failing tests there, hidden by the mirror-image bug in the plugins repo's own runner -- filed as ChuckBuilds/ledmatrix-plugins#464 and #465. --- scripts/run_plugin_tests.py | 92 +++++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 8 deletions(-) diff --git a/scripts/run_plugin_tests.py b/scripts/run_plugin_tests.py index 61e59710..baaf1075 100755 --- a/scripts/run_plugin_tests.py +++ b/scripts/run_plugin_tests.py @@ -6,6 +6,9 @@ Discovers and runs tests for LEDMatrix plugins. Supports both unittest and pytest. """ +import os +import re +import subprocess import sys import argparse from pathlib import Path @@ -68,6 +71,65 @@ def _find_tests_in_dir(directory: Path) -> list: return sorted(set(test_files)) +def _is_script_style(path) -> bool: + """True when a test file is a standalone script, not a pytest module. + + Most plugin tests are written as `def main()` plus an `if __name__ == + "__main__"` guard and signal through an exit code. pytest collects zero + items from those, so handing them to pytest printed "no tests ran" and this + runner reported success over work it had not done -- 151 of 248 files on a + fully populated rig. + """ + try: + src = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return False + has_pytest_items = re.search(r"^\s*(def test_|class Test|async def test_)", src, re.M) + has_main_guard = "__main__" in src and "__name__" in src + return bool(has_main_guard and not has_pytest_items) + + +def run_script_tests(test_files: list, verbose: bool = False) -> int: + """Run standalone test scripts, honouring the 0 pass / 2 skip / 1 fail + convention that ledmatrix-plugins' own runner established. + + Scripts opt into skipping by printing "SKIP: " and exiting 2 -- + a script that needs a tty or an LED matrix is not a regression. + """ + env = dict(os.environ) + env.setdefault("PYTHONPATH", str(PROJECT_ROOT)) + env.setdefault("LEDMATRIX_CORE", str(PROJECT_ROOT)) + + passed = skipped = failed = 0 + failures = [] + for path in test_files: + try: + proc = subprocess.run([sys.executable, str(path)], cwd=str(Path(path).parent), + capture_output=True, text=True, env=env, + stdin=subprocess.DEVNULL, timeout=300) + rc = proc.returncode + tail = " | ".join((proc.stdout or proc.stderr or "").strip().splitlines()[-2:])[:200] + except subprocess.TimeoutExpired: + rc, tail = 1, "timed out after 300s" + if rc == 0: + passed += 1 + label = "pass" + elif rc == 2: + skipped += 1 + label = "SKIP" + else: + failed += 1 + label = "FAIL" + failures.append(f"{Path(path).name}: exit {rc} | {tail}") + if verbose or rc != 0: + print(f" [{label}] {Path(path).name}" + (f" -- {tail}" if rc != 0 else "")) + + print(f"\n{passed} passed, {skipped} skipped, {failed} failed (scripts)") + for f in failures: + print(f" - {f}", file=sys.stderr) + return 1 if failed else 0 + + def run_unittest_tests(test_files: list, verbose: bool = False) -> int: """ Run tests using unittest. @@ -186,11 +248,16 @@ def main(): print("No test files found in plugins directory") return 0 - print(f"Found {len(test_files)} test file(s)") + scripts = [f for f in test_files if _is_script_style(f)] + modules = [f for f in test_files if f not in scripts] + + print(f"Found {len(test_files)} test file(s)" + + (f" -- {len(modules)} collectable, {len(scripts)} standalone script(s)" + if scripts else "")) for test_file in test_files: print(f" - {test_file}") print() - + # Determine runner runner = args.runner if runner == 'auto': @@ -199,12 +266,21 @@ def main(): runner = 'pytest' except ImportError: runner = 'unittest' - - # Run tests - if runner == 'pytest': - return run_pytest_tests(test_files, args.verbose, args.coverage) - else: - return run_unittest_tests(test_files, args.verbose) + + # Standalone scripts cannot be collected by pytest or unittest -- run them + # as the scripts they are. Doing this rather than silently collecting zero + # items is the whole point: this runner used to report success having + # executed nothing. + rc = 0 + if scripts: + rc |= run_script_tests(scripts, args.verbose) + + if modules: + if runner == 'pytest': + rc |= run_pytest_tests(modules, args.verbose, args.coverage) + else: + rc |= run_unittest_tests(modules, args.verbose) + return rc if __name__ == '__main__':