fix(scripts): run standalone plugin tests instead of collecting nothing

run_plugin_tests.py discovered every plugin test file and handed the lot to
pytest. Most plugin tests are standalone scripts -- module-level main() plus an
`if __name__ == "__main__"` guard, signalling through an exit code -- and pytest
collects zero items from those. The run printed how many files it had *found*,
then "no tests ran", and exited without executing any of them. On a rig with all
44 first-party plugins that is 151 of 248 files.

Files are now classified and each kind runs under the right runner: pytest for
real test modules, subprocess for scripts, honouring the 0 pass / 2 skip / 1
fail convention ledmatrix-plugins' own runner established (a script that wants a
tty or an LED matrix is a skip, not a regression).

Before:
    $ python3 scripts/run_plugin_tests.py -p countdown -d ~/LEDMatrix/plugin-repos
    Found 1 test file(s)
    collected 0 items
    no tests ran in 0.31s                      rc=0

After:
    Found 1 test file(s) -- 0 collectable, 1 standalone script(s)
    1 passed, 0 skipped, 0 failed (scripts)    rc=0

Verified across three shapes: countdown (1 script), jellyfin-now-playing and
pomodoro-timer (pytest only, 16 and 42 tests), and ledmatrix-flights (11 files
split 4 collectable / 7 scripts, all seven of which had never run).

Closes #532.

Running the flights scripts for the first time also surfaced four genuinely
failing tests there, hidden by the mirror-image bug in the plugins repo's own
runner -- filed as ChuckBuilds/ledmatrix-plugins#464 and #465.
This commit is contained in:
ChuckBuilds
2026-09-04 23:20:26 -04:00
parent e50a100814
commit 371bc69bd8
+80 -4
View File
@@ -6,6 +6,9 @@ Discovers and runs tests for LEDMatrix plugins.
Supports both unittest and pytest. Supports both unittest and pytest.
""" """
import os
import re
import subprocess
import sys import sys
import argparse import argparse
from pathlib import Path from pathlib import Path
@@ -68,6 +71,65 @@ def _find_tests_in_dir(directory: Path) -> list:
return sorted(set(test_files)) return sorted(set(test_files))
def _is_script_style(path) -> bool:
"""True when a test file is a standalone script, not a pytest module.
Most plugin tests are written as `def main()` plus an `if __name__ ==
"__main__"` guard and signal through an exit code. pytest collects zero
items from those, so handing them to pytest printed "no tests ran" and this
runner reported success over work it had not done -- 151 of 248 files on a
fully populated rig.
"""
try:
src = Path(path).read_text(encoding="utf-8", errors="replace")
except OSError:
return False
has_pytest_items = re.search(r"^\s*(def test_|class Test|async def test_)", src, re.M)
has_main_guard = "__main__" in src and "__name__" in src
return bool(has_main_guard and not has_pytest_items)
def run_script_tests(test_files: list, verbose: bool = False) -> int:
"""Run standalone test scripts, honouring the 0 pass / 2 skip / 1 fail
convention that ledmatrix-plugins' own runner established.
Scripts opt into skipping by printing "SKIP: <reason>" and exiting 2 --
a script that needs a tty or an LED matrix is not a regression.
"""
env = dict(os.environ)
env.setdefault("PYTHONPATH", str(PROJECT_ROOT))
env.setdefault("LEDMATRIX_CORE", str(PROJECT_ROOT))
passed = skipped = failed = 0
failures = []
for path in test_files:
try:
proc = subprocess.run([sys.executable, str(path)], cwd=str(Path(path).parent),
capture_output=True, text=True, env=env,
stdin=subprocess.DEVNULL, timeout=300)
rc = proc.returncode
tail = " | ".join((proc.stdout or proc.stderr or "").strip().splitlines()[-2:])[:200]
except subprocess.TimeoutExpired:
rc, tail = 1, "timed out after 300s"
if rc == 0:
passed += 1
label = "pass"
elif rc == 2:
skipped += 1
label = "SKIP"
else:
failed += 1
label = "FAIL"
failures.append(f"{Path(path).name}: exit {rc} | {tail}")
if verbose or rc != 0:
print(f" [{label}] {Path(path).name}" + (f" -- {tail}" if rc != 0 else ""))
print(f"\n{passed} passed, {skipped} skipped, {failed} failed (scripts)")
for f in failures:
print(f" - {f}", file=sys.stderr)
return 1 if failed else 0
def run_unittest_tests(test_files: list, verbose: bool = False) -> int: def run_unittest_tests(test_files: list, verbose: bool = False) -> int:
""" """
Run tests using unittest. Run tests using unittest.
@@ -186,7 +248,12 @@ def main():
print("No test files found in plugins directory") print("No test files found in plugins directory")
return 0 return 0
print(f"Found {len(test_files)} test file(s)") scripts = [f for f in test_files if _is_script_style(f)]
modules = [f for f in test_files if f not in scripts]
print(f"Found {len(test_files)} test file(s)"
+ (f" -- {len(modules)} collectable, {len(scripts)} standalone script(s)"
if scripts else ""))
for test_file in test_files: for test_file in test_files:
print(f" - {test_file}") print(f" - {test_file}")
print() print()
@@ -200,11 +267,20 @@ def main():
except ImportError: except ImportError:
runner = 'unittest' runner = 'unittest'
# Run tests # Standalone scripts cannot be collected by pytest or unittest -- run them
# as the scripts they are. Doing this rather than silently collecting zero
# items is the whole point: this runner used to report success having
# executed nothing.
rc = 0
if scripts:
rc |= run_script_tests(scripts, args.verbose)
if modules:
if runner == 'pytest': if runner == 'pytest':
return run_pytest_tests(test_files, args.verbose, args.coverage) rc |= run_pytest_tests(modules, args.verbose, args.coverage)
else: else:
return run_unittest_tests(test_files, args.verbose) rc |= run_unittest_tests(modules, args.verbose)
return rc
if __name__ == '__main__': if __name__ == '__main__':