diff --git a/scripts/run_plugin_tests.py b/scripts/run_plugin_tests.py index 61e59710..baaf1075 100755 --- a/scripts/run_plugin_tests.py +++ b/scripts/run_plugin_tests.py @@ -6,6 +6,9 @@ Discovers and runs tests for LEDMatrix plugins. Supports both unittest and pytest. """ +import os +import re +import subprocess import sys import argparse from pathlib import Path @@ -68,6 +71,65 @@ def _find_tests_in_dir(directory: Path) -> list: return sorted(set(test_files)) +def _is_script_style(path) -> bool: + """True when a test file is a standalone script, not a pytest module. + + Most plugin tests are written as `def main()` plus an `if __name__ == + "__main__"` guard and signal through an exit code. pytest collects zero + items from those, so handing them to pytest printed "no tests ran" and this + runner reported success over work it had not done -- 151 of 248 files on a + fully populated rig. + """ + try: + src = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError: + return False + has_pytest_items = re.search(r"^\s*(def test_|class Test|async def test_)", src, re.M) + has_main_guard = "__main__" in src and "__name__" in src + return bool(has_main_guard and not has_pytest_items) + + +def run_script_tests(test_files: list, verbose: bool = False) -> int: + """Run standalone test scripts, honouring the 0 pass / 2 skip / 1 fail + convention that ledmatrix-plugins' own runner established. + + Scripts opt into skipping by printing "SKIP: " and exiting 2 -- + a script that needs a tty or an LED matrix is not a regression. + """ + env = dict(os.environ) + env.setdefault("PYTHONPATH", str(PROJECT_ROOT)) + env.setdefault("LEDMATRIX_CORE", str(PROJECT_ROOT)) + + passed = skipped = failed = 0 + failures = [] + for path in test_files: + try: + proc = subprocess.run([sys.executable, str(path)], cwd=str(Path(path).parent), + capture_output=True, text=True, env=env, + stdin=subprocess.DEVNULL, timeout=300) + rc = proc.returncode + tail = " | ".join((proc.stdout or proc.stderr or "").strip().splitlines()[-2:])[:200] + except subprocess.TimeoutExpired: + rc, tail = 1, "timed out after 300s" + if rc == 0: + passed += 1 + label = "pass" + elif rc == 2: + skipped += 1 + label = "SKIP" + else: + failed += 1 + label = "FAIL" + failures.append(f"{Path(path).name}: exit {rc} | {tail}") + if verbose or rc != 0: + print(f" [{label}] {Path(path).name}" + (f" -- {tail}" if rc != 0 else "")) + + print(f"\n{passed} passed, {skipped} skipped, {failed} failed (scripts)") + for f in failures: + print(f" - {f}", file=sys.stderr) + return 1 if failed else 0 + + def run_unittest_tests(test_files: list, verbose: bool = False) -> int: """ Run tests using unittest. @@ -186,11 +248,16 @@ def main(): print("No test files found in plugins directory") return 0 - print(f"Found {len(test_files)} test file(s)") + scripts = [f for f in test_files if _is_script_style(f)] + modules = [f for f in test_files if f not in scripts] + + print(f"Found {len(test_files)} test file(s)" + + (f" -- {len(modules)} collectable, {len(scripts)} standalone script(s)" + if scripts else "")) for test_file in test_files: print(f" - {test_file}") print() - + # Determine runner runner = args.runner if runner == 'auto': @@ -199,12 +266,21 @@ def main(): runner = 'pytest' except ImportError: runner = 'unittest' - - # Run tests - if runner == 'pytest': - return run_pytest_tests(test_files, args.verbose, args.coverage) - else: - return run_unittest_tests(test_files, args.verbose) + + # Standalone scripts cannot be collected by pytest or unittest -- run them + # as the scripts they are. Doing this rather than silently collecting zero + # items is the whole point: this runner used to report success having + # executed nothing. + rc = 0 + if scripts: + rc |= run_script_tests(scripts, args.verbose) + + if modules: + if runner == 'pytest': + rc |= run_pytest_tests(modules, args.verbose, args.coverage) + else: + rc |= run_unittest_tests(modules, args.verbose) + return rc if __name__ == '__main__':