Coverage for src/governed_harness/orchestration/run_outcome.py: 96%
82 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""What a failing test run tells the differential verification about the tests that ran (#190).
3A test run that was interrupted before any test ran (pytest stopped at collection, a usage
4error, no test collected) reports problems, but none of them is a test outcome: the collection
5errors hide every other result, so a regression the change introduced cannot show up as a new
6problem. The differential comparison treats such a run as not comparable and keeps the failure.
8The tests that ran are also read from the output, so the comparison can check that the test
9files the ChangeSet added or modified ran at all. pytest prints the node ids of the tests that
10failed and, with ``-rA`` or ``-v``, of those that passed; the list is complete when it holds as
11many tests as the summary line counts."""
13from __future__ import annotations
15import re
16from dataclasses import dataclass
17from pathlib import PurePosixPath
19from governed_harness.checks.model import is_test_path
21__all__ = ["RunOutcome", "changed_test_files", "not_run", "read_test_run", "runner_of"]
23# pytest exit codes: 2 interrupted (collection errors, KeyboardInterrupt), 3 internal error,
24# 4 usage error, 5 no tests collected. None of them comes with a test outcome.
25_PYTEST_NO_OUTCOME = {
26 2: "pytest was interrupted (exit code 2), for example by an error during collection",
27 3: "pytest stopped with an internal error (exit code 3)",
28 4: "pytest was called wrongly (exit code 4)",
29 5: "pytest collected no test (exit code 5)",
30}
31_SUMMARY_NAME = re.compile(r" (passed|failed|xfailed|xpassed|errors?|skipped|deselected)\b")
32_SUMMARY_TIME = re.compile(r"\bin \d+(?:\.\d+)?s\b")
33_SHORT_NODE = re.compile(r"^(?:PASSED|FAILED|XFAIL|XPASS) (?P<node>[^\s:]+::\S+)")
34_VERBOSE_NODE = re.compile(r"^(?P<node>[^\s:]+::\S+) (?:PASSED|FAILED|XFAIL|XPASS)\b")
35_RAN = ("passed", "failed", "xfailed", "xpassed")
36_PYTEST_TEST_FILE = re.compile(r"^(test_.*|.*_test)\.py$")
37_TEST_TOKEN = re.compile(r"(?i)test\w*")
40@dataclass(frozen=True)
41class RunOutcome:
42 """What a test run's output says about the tests that ran."""
44 ran: bool
45 """At least one test passed or failed."""
46 reason: str
47 """Why the run has no test outcome (empty when ``ran``)."""
48 executed: frozenset[str] | None
49 """The workspace-relative paths of the test files with a test that ran; ``None`` when the
50 output does not list every test that ran."""
53def runner_of(validator_id: str, argv: tuple[str, ...]) -> str | None:
54 """``pytest`` for a pytest command, ``other`` for another test runner, ``None`` for a
55 validator that does not run tests (a linter, a type checker)."""
56 names = [PurePosixPath(item.replace("\\", "/")).name.lower() for item in argv]
57 if any(name.removesuffix(".exe") in {"pytest", "py.test"} for name in names):
58 return "pytest"
59 last = validator_id.rsplit(".", 1)[-1].lower()
60 if "test" in last or last == "unit" or any(_TEST_TOKEN.fullmatch(item) for item in argv):
61 return "other"
62 return None
65def read_test_run(
66 runner: str,
67 exit_code: int | None,
68 stdout: str,
69 stderr: str,
70 keys: tuple[str, ...],
71) -> RunOutcome:
72 """The outcome of a failing test run, from its exit code, its output and its parsed
73 problem keys (``rule|path|message``)."""
74 failed_keys = [key for key in keys if key.startswith("test-failed|")]
75 if runner != "pytest":
76 if failed_keys:
77 return RunOutcome(True, "", None)
78 return RunOutcome(False, "no test outcome could be read from the output", None)
79 if exit_code in _PYTEST_NO_OUTCOME:
80 return RunOutcome(False, _PYTEST_NO_OUTCOME[exit_code], frozenset())
81 lines = f"{stdout}\n{stderr}".splitlines()
82 counts = _summary_counts(lines)
83 ran = sum(counts.get(name, 0) for name in _RAN) if counts is not None else None
84 if ran == 0 or (ran is None and not failed_keys):
85 return RunOutcome(
86 False, "the pytest output reports no test that passed or failed", frozenset()
87 )
88 nodes = {
89 found["node"]
90 for line in lines
91 if (found := _SHORT_NODE.match(line) or _VERBOSE_NODE.match(line))
92 }
93 files = frozenset(node.split("::", 1)[0].replace("\\", "/") for node in nodes)
94 complete = ran is not None and len(nodes) >= ran
95 return RunOutcome(True, "", files if complete else None)
98def changed_test_files(runner: str, changes: list[tuple[str, str]]) -> list[str]:
99 """The test files the ChangeSet added or modified (``(path, status)`` pairs) that the
100 runner is expected to run."""
101 found = []
102 for path, status in changes:
103 normalized = path.replace("\\", "/")
104 if status == "DELETED" or not is_test_path(normalized):
105 continue
106 if runner == "pytest" and not _PYTEST_TEST_FILE.match(PurePosixPath(normalized).name):
107 continue
108 found.append(normalized)
109 return sorted(set(found))
112def not_run(changed: list[str], executed: frozenset[str]) -> list[str]:
113 """The changed test files none of whose tests ran (a node path may be relative to a
114 rootdir below the workspace)."""
115 return [
116 path
117 for path in changed
118 if not any(path == item or path.endswith(f"/{item}") for item in executed)
119 ]
122def _summary_items(line: str) -> list[tuple[str, str]]:
123 """The ``(number, name)`` pairs of a summary line, such as ``("2", "passed")``, in order.
125 A pair is a run of decimal digits, a space and a count name not followed by a word
126 character. Each name is found by an expression without a leading repetition and its number is
127 read backwards from it, so the time is linear in the length of ``line``.
128 """
129 found: list[tuple[str, str]] = []
130 for name in _SUMMARY_NAME.finditer(line):
131 end = start = name.start()
132 while start > 0 and line[start - 1].isdecimal():
133 start -= 1
134 if start < end:
135 found.append((line[start:end], name.group(1)))
136 return found
139def _summary_counts(lines: list[str]) -> dict[str, int] | None:
140 """The counts of the last pytest summary line (``1 failed, 2 passed in 0.12s``)."""
141 for line in reversed(lines):
142 if not _SUMMARY_TIME.search(line):
143 continue
144 found = _summary_items(line)
145 if found: 145 ↛ 151line 145 didn't jump to line 151 because the condition on line 145 was always true
146 counts: dict[str, int] = {}
147 for number, name in found:
148 key = "errors" if name == "error" else name
149 counts[key] = counts.get(key, 0) + int(number)
150 return counts
151 if "no tests ran" in line:
152 return {}
153 return None