mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-10-17 20:39:13 +00:00
fix(scripts): run standalone plugin tests instead of collecting nothing
run_plugin_tests.py discovered every plugin test file and handed the lot to
pytest. Most plugin tests are standalone scripts -- module-level main() plus an
`if __name__ == "__main__"` guard, signalling through an exit code -- and pytest
collects zero items from those. The run printed how many files it had *found*,
then "no tests ran", and exited without executing any of them. On a rig with all
44 first-party plugins that is 151 of 248 files.
Files are now classified and each kind runs under the right runner: pytest for
real test modules, subprocess for scripts, honouring the 0 pass / 2 skip / 1
fail convention ledmatrix-plugins' own runner established (a script that wants a
tty or an LED matrix is a skip, not a regression).
Before:
$ python3 scripts/run_plugin_tests.py -p countdown -d ~/LEDMatrix/plugin-repos
Found 1 test file(s)
collected 0 items
no tests ran in 0.31s rc=0
After:
Found 1 test file(s) -- 0 collectable, 1 standalone script(s)
1 passed, 0 skipped, 0 failed (scripts) rc=0
Verified across three shapes: countdown (1 script), jellyfin-now-playing and
pomodoro-timer (pytest only, 16 and 42 tests), and ledmatrix-flights (11 files
split 4 collectable / 7 scripts, all seven of which had never run).
Closes #532.
Running the flights scripts for the first time also surfaced four genuinely
failing tests there, hidden by the mirror-image bug in the plugins repo's own
runner -- filed as ChuckBuilds/ledmatrix-plugins#464 and #465.
This commit is contained in:
@@ -6,6 +6,9 @@ Discovers and runs tests for LEDMatrix plugins.
|
||||
Supports both unittest and pytest.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
@@ -68,6 +71,65 @@ def _find_tests_in_dir(directory: Path) -> list:
|
||||
return sorted(set(test_files))
|
||||
|
||||
|
||||
def _is_script_style(path) -> bool:
|
||||
"""True when a test file is a standalone script, not a pytest module.
|
||||
|
||||
Most plugin tests are written as `def main()` plus an `if __name__ ==
|
||||
"__main__"` guard and signal through an exit code. pytest collects zero
|
||||
items from those, so handing them to pytest printed "no tests ran" and this
|
||||
runner reported success over work it had not done -- 151 of 248 files on a
|
||||
fully populated rig.
|
||||
"""
|
||||
try:
|
||||
src = Path(path).read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return False
|
||||
has_pytest_items = re.search(r"^\s*(def test_|class Test|async def test_)", src, re.M)
|
||||
has_main_guard = "__main__" in src and "__name__" in src
|
||||
return bool(has_main_guard and not has_pytest_items)
|
||||
|
||||
|
||||
def run_script_tests(test_files: list, verbose: bool = False) -> int:
|
||||
"""Run standalone test scripts, honouring the 0 pass / 2 skip / 1 fail
|
||||
convention that ledmatrix-plugins' own runner established.
|
||||
|
||||
Scripts opt into skipping by printing "SKIP: <reason>" and exiting 2 --
|
||||
a script that needs a tty or an LED matrix is not a regression.
|
||||
"""
|
||||
env = dict(os.environ)
|
||||
env.setdefault("PYTHONPATH", str(PROJECT_ROOT))
|
||||
env.setdefault("LEDMATRIX_CORE", str(PROJECT_ROOT))
|
||||
|
||||
passed = skipped = failed = 0
|
||||
failures = []
|
||||
for path in test_files:
|
||||
try:
|
||||
proc = subprocess.run([sys.executable, str(path)], cwd=str(Path(path).parent),
|
||||
capture_output=True, text=True, env=env,
|
||||
stdin=subprocess.DEVNULL, timeout=300)
|
||||
rc = proc.returncode
|
||||
tail = " | ".join((proc.stdout or proc.stderr or "").strip().splitlines()[-2:])[:200]
|
||||
except subprocess.TimeoutExpired:
|
||||
rc, tail = 1, "timed out after 300s"
|
||||
if rc == 0:
|
||||
passed += 1
|
||||
label = "pass"
|
||||
elif rc == 2:
|
||||
skipped += 1
|
||||
label = "SKIP"
|
||||
else:
|
||||
failed += 1
|
||||
label = "FAIL"
|
||||
failures.append(f"{Path(path).name}: exit {rc} | {tail}")
|
||||
if verbose or rc != 0:
|
||||
print(f" [{label}] {Path(path).name}" + (f" -- {tail}" if rc != 0 else ""))
|
||||
|
||||
print(f"\n{passed} passed, {skipped} skipped, {failed} failed (scripts)")
|
||||
for f in failures:
|
||||
print(f" - {f}", file=sys.stderr)
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def run_unittest_tests(test_files: list, verbose: bool = False) -> int:
|
||||
"""
|
||||
Run tests using unittest.
|
||||
@@ -186,11 +248,16 @@ def main():
|
||||
print("No test files found in plugins directory")
|
||||
return 0
|
||||
|
||||
print(f"Found {len(test_files)} test file(s)")
|
||||
scripts = [f for f in test_files if _is_script_style(f)]
|
||||
modules = [f for f in test_files if f not in scripts]
|
||||
|
||||
print(f"Found {len(test_files)} test file(s)"
|
||||
+ (f" -- {len(modules)} collectable, {len(scripts)} standalone script(s)"
|
||||
if scripts else ""))
|
||||
for test_file in test_files:
|
||||
print(f" - {test_file}")
|
||||
print()
|
||||
|
||||
|
||||
# Determine runner
|
||||
runner = args.runner
|
||||
if runner == 'auto':
|
||||
@@ -199,12 +266,21 @@ def main():
|
||||
runner = 'pytest'
|
||||
except ImportError:
|
||||
runner = 'unittest'
|
||||
|
||||
# Run tests
|
||||
if runner == 'pytest':
|
||||
return run_pytest_tests(test_files, args.verbose, args.coverage)
|
||||
else:
|
||||
return run_unittest_tests(test_files, args.verbose)
|
||||
|
||||
# Standalone scripts cannot be collected by pytest or unittest -- run them
|
||||
# as the scripts they are. Doing this rather than silently collecting zero
|
||||
# items is the whole point: this runner used to report success having
|
||||
# executed nothing.
|
||||
rc = 0
|
||||
if scripts:
|
||||
rc |= run_script_tests(scripts, args.verbose)
|
||||
|
||||
if modules:
|
||||
if runner == 'pytest':
|
||||
rc |= run_pytest_tests(modules, args.verbose, args.coverage)
|
||||
else:
|
||||
rc |= run_unittest_tests(modules, args.verbose)
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user