mirror of
https://github.com/ChuckBuilds/LEDMatrix.git
synced 2026-10-18 04:49:13 +00:00
fix(scripts): run standalone plugin tests instead of collecting nothing
run_plugin_tests.py discovered every plugin test file and handed the lot to
pytest. Most plugin tests are standalone scripts -- module-level main() plus an
`if __name__ == "__main__"` guard, signalling through an exit code -- and pytest
collects zero items from those. The run printed how many files it had *found*,
then "no tests ran", and exited without executing any of them. On a rig with all
44 first-party plugins that is 151 of 248 files.
Files are now classified and each kind runs under the right runner: pytest for
real test modules, subprocess for scripts, honouring the 0 pass / 2 skip / 1
fail convention ledmatrix-plugins' own runner established (a script that wants a
tty or an LED matrix is a skip, not a regression).
Before:
$ python3 scripts/run_plugin_tests.py -p countdown -d ~/LEDMatrix/plugin-repos
Found 1 test file(s)
collected 0 items
no tests ran in 0.31s rc=0
After:
Found 1 test file(s) -- 0 collectable, 1 standalone script(s)
1 passed, 0 skipped, 0 failed (scripts) rc=0
Verified across three shapes: countdown (1 script), jellyfin-now-playing and
pomodoro-timer (pytest only, 16 and 42 tests), and ledmatrix-flights (11 files
split 4 collectable / 7 scripts, all seven of which had never run).
Closes #532.
Running the flights scripts for the first time also surfaced four genuinely
failing tests there, hidden by the mirror-image bug in the plugins repo's own
runner -- filed as ChuckBuilds/ledmatrix-plugins#464 and #465.
This commit is contained in:
@@ -6,6 +6,9 @@ Discovers and runs tests for LEDMatrix plugins.
|
|||||||
Supports both unittest and pytest.
|
Supports both unittest and pytest.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import argparse
|
import argparse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -68,6 +71,65 @@ def _find_tests_in_dir(directory: Path) -> list:
|
|||||||
return sorted(set(test_files))
|
return sorted(set(test_files))
|
||||||
|
|
||||||
|
|
||||||
|
def _is_script_style(path) -> bool:
|
||||||
|
"""True when a test file is a standalone script, not a pytest module.
|
||||||
|
|
||||||
|
Most plugin tests are written as `def main()` plus an `if __name__ ==
|
||||||
|
"__main__"` guard and signal through an exit code. pytest collects zero
|
||||||
|
items from those, so handing them to pytest printed "no tests ran" and this
|
||||||
|
runner reported success over work it had not done -- 151 of 248 files on a
|
||||||
|
fully populated rig.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
src = Path(path).read_text(encoding="utf-8", errors="replace")
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
has_pytest_items = re.search(r"^\s*(def test_|class Test|async def test_)", src, re.M)
|
||||||
|
has_main_guard = "__main__" in src and "__name__" in src
|
||||||
|
return bool(has_main_guard and not has_pytest_items)
|
||||||
|
|
||||||
|
|
||||||
|
def run_script_tests(test_files: list, verbose: bool = False) -> int:
|
||||||
|
"""Run standalone test scripts, honouring the 0 pass / 2 skip / 1 fail
|
||||||
|
convention that ledmatrix-plugins' own runner established.
|
||||||
|
|
||||||
|
Scripts opt into skipping by printing "SKIP: <reason>" and exiting 2 --
|
||||||
|
a script that needs a tty or an LED matrix is not a regression.
|
||||||
|
"""
|
||||||
|
env = dict(os.environ)
|
||||||
|
env.setdefault("PYTHONPATH", str(PROJECT_ROOT))
|
||||||
|
env.setdefault("LEDMATRIX_CORE", str(PROJECT_ROOT))
|
||||||
|
|
||||||
|
passed = skipped = failed = 0
|
||||||
|
failures = []
|
||||||
|
for path in test_files:
|
||||||
|
try:
|
||||||
|
proc = subprocess.run([sys.executable, str(path)], cwd=str(Path(path).parent),
|
||||||
|
capture_output=True, text=True, env=env,
|
||||||
|
stdin=subprocess.DEVNULL, timeout=300)
|
||||||
|
rc = proc.returncode
|
||||||
|
tail = " | ".join((proc.stdout or proc.stderr or "").strip().splitlines()[-2:])[:200]
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
rc, tail = 1, "timed out after 300s"
|
||||||
|
if rc == 0:
|
||||||
|
passed += 1
|
||||||
|
label = "pass"
|
||||||
|
elif rc == 2:
|
||||||
|
skipped += 1
|
||||||
|
label = "SKIP"
|
||||||
|
else:
|
||||||
|
failed += 1
|
||||||
|
label = "FAIL"
|
||||||
|
failures.append(f"{Path(path).name}: exit {rc} | {tail}")
|
||||||
|
if verbose or rc != 0:
|
||||||
|
print(f" [{label}] {Path(path).name}" + (f" -- {tail}" if rc != 0 else ""))
|
||||||
|
|
||||||
|
print(f"\n{passed} passed, {skipped} skipped, {failed} failed (scripts)")
|
||||||
|
for f in failures:
|
||||||
|
print(f" - {f}", file=sys.stderr)
|
||||||
|
return 1 if failed else 0
|
||||||
|
|
||||||
|
|
||||||
def run_unittest_tests(test_files: list, verbose: bool = False) -> int:
|
def run_unittest_tests(test_files: list, verbose: bool = False) -> int:
|
||||||
"""
|
"""
|
||||||
Run tests using unittest.
|
Run tests using unittest.
|
||||||
@@ -186,7 +248,12 @@ def main():
|
|||||||
print("No test files found in plugins directory")
|
print("No test files found in plugins directory")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
print(f"Found {len(test_files)} test file(s)")
|
scripts = [f for f in test_files if _is_script_style(f)]
|
||||||
|
modules = [f for f in test_files if f not in scripts]
|
||||||
|
|
||||||
|
print(f"Found {len(test_files)} test file(s)"
|
||||||
|
+ (f" -- {len(modules)} collectable, {len(scripts)} standalone script(s)"
|
||||||
|
if scripts else ""))
|
||||||
for test_file in test_files:
|
for test_file in test_files:
|
||||||
print(f" - {test_file}")
|
print(f" - {test_file}")
|
||||||
print()
|
print()
|
||||||
@@ -200,11 +267,20 @@ def main():
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
runner = 'unittest'
|
runner = 'unittest'
|
||||||
|
|
||||||
# Run tests
|
# Standalone scripts cannot be collected by pytest or unittest -- run them
|
||||||
if runner == 'pytest':
|
# as the scripts they are. Doing this rather than silently collecting zero
|
||||||
return run_pytest_tests(test_files, args.verbose, args.coverage)
|
# items is the whole point: this runner used to report success having
|
||||||
else:
|
# executed nothing.
|
||||||
return run_unittest_tests(test_files, args.verbose)
|
rc = 0
|
||||||
|
if scripts:
|
||||||
|
rc |= run_script_tests(scripts, args.verbose)
|
||||||
|
|
||||||
|
if modules:
|
||||||
|
if runner == 'pytest':
|
||||||
|
rc |= run_pytest_tests(modules, args.verbose, args.coverage)
|
||||||
|
else:
|
||||||
|
rc |= run_unittest_tests(modules, args.verbose)
|
||||||
|
return rc
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|||||||
Reference in New Issue
Block a user