Coverage for src/governed_harness/orchestration/run_outcome.py: 96%

82 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""What a failing test run tells the differential verification about the tests that ran (#190). 

2 

3A test run that was interrupted before any test ran (pytest stopped at collection, a usage 

4error, no test collected) reports problems, but none of them is a test outcome: the collection 

5errors hide every other result, so a regression the change introduced cannot show up as a new 

6problem. The differential comparison treats such a run as not comparable and keeps the failure. 

7 

8The tests that ran are also read from the output, so the comparison can check that the test 

9files the ChangeSet added or modified ran at all. pytest prints the node ids of the tests that 

10failed and, with ``-rA`` or ``-v``, of those that passed; the list is complete when it holds as 

11many tests as the summary line counts.""" 

12 

13from __future__ import annotations 

14 

15import re 

16from dataclasses import dataclass 

17from pathlib import PurePosixPath 

18 

19from governed_harness.checks.model import is_test_path 

20 

21__all__ = ["RunOutcome", "changed_test_files", "not_run", "read_test_run", "runner_of"] 

22 

23# pytest exit codes: 2 interrupted (collection errors, KeyboardInterrupt), 3 internal error, 

24# 4 usage error, 5 no tests collected. None of them comes with a test outcome. 

25_PYTEST_NO_OUTCOME = { 

26 2: "pytest was interrupted (exit code 2), for example by an error during collection", 

27 3: "pytest stopped with an internal error (exit code 3)", 

28 4: "pytest was called wrongly (exit code 4)", 

29 5: "pytest collected no test (exit code 5)", 

30} 

31_SUMMARY_NAME = re.compile(r" (passed|failed|xfailed|xpassed|errors?|skipped|deselected)\b") 

32_SUMMARY_TIME = re.compile(r"\bin \d+(?:\.\d+)?s\b") 

33_SHORT_NODE = re.compile(r"^(?:PASSED|FAILED|XFAIL|XPASS) (?P<node>[^\s:]+::\S+)") 

34_VERBOSE_NODE = re.compile(r"^(?P<node>[^\s:]+::\S+) (?:PASSED|FAILED|XFAIL|XPASS)\b") 

35_RAN = ("passed", "failed", "xfailed", "xpassed") 

36_PYTEST_TEST_FILE = re.compile(r"^(test_.*|.*_test)\.py$") 

37_TEST_TOKEN = re.compile(r"(?i)test\w*") 

38 

39 

40@dataclass(frozen=True) 

41class RunOutcome: 

42 """What a test run's output says about the tests that ran.""" 

43 

44 ran: bool 

45 """At least one test passed or failed.""" 

46 reason: str 

47 """Why the run has no test outcome (empty when ``ran``).""" 

48 executed: frozenset[str] | None 

49 """The workspace-relative paths of the test files with a test that ran; ``None`` when the 

50 output does not list every test that ran.""" 

51 

52 

53def runner_of(validator_id: str, argv: tuple[str, ...]) -> str | None: 

54 """``pytest`` for a pytest command, ``other`` for another test runner, ``None`` for a 

55 validator that does not run tests (a linter, a type checker).""" 

56 names = [PurePosixPath(item.replace("\\", "/")).name.lower() for item in argv] 

57 if any(name.removesuffix(".exe") in {"pytest", "py.test"} for name in names): 

58 return "pytest" 

59 last = validator_id.rsplit(".", 1)[-1].lower() 

60 if "test" in last or last == "unit" or any(_TEST_TOKEN.fullmatch(item) for item in argv): 

61 return "other" 

62 return None 

63 

64 

65def read_test_run( 

66 runner: str, 

67 exit_code: int | None, 

68 stdout: str, 

69 stderr: str, 

70 keys: tuple[str, ...], 

71) -> RunOutcome: 

72 """The outcome of a failing test run, from its exit code, its output and its parsed 

73 problem keys (``rule|path|message``).""" 

74 failed_keys = [key for key in keys if key.startswith("test-failed|")] 

75 if runner != "pytest": 

76 if failed_keys: 

77 return RunOutcome(True, "", None) 

78 return RunOutcome(False, "no test outcome could be read from the output", None) 

79 if exit_code in _PYTEST_NO_OUTCOME: 

80 return RunOutcome(False, _PYTEST_NO_OUTCOME[exit_code], frozenset()) 

81 lines = f"{stdout}\n{stderr}".splitlines() 

82 counts = _summary_counts(lines) 

83 ran = sum(counts.get(name, 0) for name in _RAN) if counts is not None else None 

84 if ran == 0 or (ran is None and not failed_keys): 

85 return RunOutcome( 

86 False, "the pytest output reports no test that passed or failed", frozenset() 

87 ) 

88 nodes = { 

89 found["node"] 

90 for line in lines 

91 if (found := _SHORT_NODE.match(line) or _VERBOSE_NODE.match(line)) 

92 } 

93 files = frozenset(node.split("::", 1)[0].replace("\\", "/") for node in nodes) 

94 complete = ran is not None and len(nodes) >= ran 

95 return RunOutcome(True, "", files if complete else None) 

96 

97 

98def changed_test_files(runner: str, changes: list[tuple[str, str]]) -> list[str]: 

99 """The test files the ChangeSet added or modified (``(path, status)`` pairs) that the 

100 runner is expected to run.""" 

101 found = [] 

102 for path, status in changes: 

103 normalized = path.replace("\\", "/") 

104 if status == "DELETED" or not is_test_path(normalized): 

105 continue 

106 if runner == "pytest" and not _PYTEST_TEST_FILE.match(PurePosixPath(normalized).name): 

107 continue 

108 found.append(normalized) 

109 return sorted(set(found)) 

110 

111 

112def not_run(changed: list[str], executed: frozenset[str]) -> list[str]: 

113 """The changed test files none of whose tests ran (a node path may be relative to a 

114 rootdir below the workspace).""" 

115 return [ 

116 path 

117 for path in changed 

118 if not any(path == item or path.endswith(f"/{item}") for item in executed) 

119 ] 

120 

121 

122def _summary_items(line: str) -> list[tuple[str, str]]: 

123 """The ``(number, name)`` pairs of a summary line, such as ``("2", "passed")``, in order. 

124 

125 A pair is a run of decimal digits, a space and a count name not followed by a word 

126 character. Each name is found by an expression without a leading repetition and its number is 

127 read backwards from it, so the time is linear in the length of ``line``. 

128 """ 

129 found: list[tuple[str, str]] = [] 

130 for name in _SUMMARY_NAME.finditer(line): 

131 end = start = name.start() 

132 while start > 0 and line[start - 1].isdecimal(): 

133 start -= 1 

134 if start < end: 

135 found.append((line[start:end], name.group(1))) 

136 return found 

137 

138 

139def _summary_counts(lines: list[str]) -> dict[str, int] | None: 

140 """The counts of the last pytest summary line (``1 failed, 2 passed in 0.12s``).""" 

141 for line in reversed(lines): 

142 if not _SUMMARY_TIME.search(line): 

143 continue 

144 found = _summary_items(line) 

145 if found: 145 ↛ 151line 145 didn't jump to line 151 because the condition on line 145 was always true

146 counts: dict[str, int] = {} 

147 for number, name in found: 

148 key = "errors" if name == "error" else name 

149 counts[key] = counts.get(key, 0) + int(number) 

150 return counts 

151 if "no tests ran" in line: 

152 return {} 

153 return None