Coverage for src/governed_harness/checks/suite_quality.py: 96%
131 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Checks of test quality: tests that cannot fail, interface methods no test calls, and changed
2lines no test executes.
4A green suite only means something when its tests assert, reach the code under change and
5exercise the contract the task declared. These checks read sources, a diff and the JSON report of
6``coverage json``; they never run the tests themselves."""
8from __future__ import annotations
10import ast
11import re
12from collections.abc import Iterator, Mapping, Sequence
13from typing import Any
15from governed_harness.domain.enums import FindingSeverity
17from .model import DiffFile, Issue, is_test_path
19CATEGORY = "tests"
20_MAX_LISTED_LINES = 15
21_ASSERTING_CALLS = frozenset(
22 {"pytest.fail", "pytest.raises", "pytest.warns", "raises", "warns", "self.fail"}
23)
26# ----- assertion-free tests ------------------------------------------------------------------
27def _dotted(node: ast.expr) -> str:
28 if isinstance(node, ast.Name):
29 return node.id
30 if isinstance(node, ast.Attribute): 30 ↛ 33line 30 didn't jump to line 33 because the condition on line 30 was always true
31 base = _dotted(node.value)
32 return f"{base}.{node.attr}" if base else node.attr
33 return ""
36def _call_name(node: ast.Call) -> str:
37 if isinstance(node.func, ast.Attribute):
38 return node.func.attr
39 if isinstance(node.func, ast.Name): 39 ↛ 41line 39 didn't jump to line 41 because the condition on line 39 was always true
40 return node.func.id
41 return ""
44def _asserts(function: ast.FunctionDef | ast.AsyncFunctionDef) -> bool:
45 """Whether the test body can fail on purpose: an ``assert``, an ``assert*`` call (unittest,
46 mock), or a ``pytest.raises``/``pytest.warns``/``pytest.fail`` call or context."""
47 for node in ast.walk(function):
48 if isinstance(node, ast.Assert):
49 return True
50 if isinstance(node, ast.Call):
51 if "assert" in _call_name(node).lower():
52 return True
53 if _dotted(node.func) in _ASSERTING_CALLS:
54 return True
55 return False
58def _tests(body: Sequence[ast.stmt]) -> Iterator[ast.FunctionDef | ast.AsyncFunctionDef]:
59 for node in body:
60 if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name.startswith(
61 "test_"
62 ):
63 yield node
64 elif isinstance(node, ast.ClassDef) and node.name.startswith("Test"):
65 yield from _tests(node.body)
68def assertion_free_tests(files: Mapping[str, str], *, severity: FindingSeverity) -> list[Issue]:
69 """One issue per ``test_*`` function or ``Test*`` method that asserts nothing.
71 Unparsable test files yield an INFO ``tests.unparsable`` issue: the suite would fail to
72 collect them anyway, and a check must not crash on agent output."""
73 issues: list[Issue] = []
74 for path in sorted(files):
75 if not path.endswith(".py") or not is_test_path(path):
76 continue
77 try:
78 tree = ast.parse(files[path], filename=path)
79 except (SyntaxError, ValueError) as error:
80 issues.append(
81 Issue(
82 rule_id="tests.unparsable",
83 severity=FindingSeverity.INFO,
84 message=f"{path} could not be parsed: {error.__class__.__name__}",
85 path=path,
86 line=getattr(error, "lineno", None),
87 category=CATEGORY,
88 )
89 )
90 continue
91 for function in _tests(tree.body):
92 if not _asserts(function):
93 issues.append(
94 Issue(
95 rule_id="tests.assertion-free",
96 severity=severity,
97 message=f"Test {function.name!r} has no assertion; it can only fail by "
98 "raising.",
99 path=path,
100 line=function.lineno,
101 category=CATEGORY,
102 recommendation="Assert on the observable result of the behavior under "
103 "test.",
104 )
105 )
106 return issues
109# ----- interface methods ---------------------------------------------------------------------
110def untested_interface_methods(
111 names: Sequence[str], test_sources: Mapping[str, str], *, severity: FindingSeverity
112) -> list[Issue]:
113 """One issue per declared interface method whose name no test calls or reads.
115 The match is lexical (``advance(`` or ``.advance``): it proves a test mentions the method,
116 not that it covers it, which is why it complements changed-line coverage."""
117 issues: list[Issue] = []
118 seen: set[str] = set()
119 for name in names:
120 if not name or name in seen:
121 continue
122 seen.add(name)
123 last = name.rsplit(".", 1)[-1]
124 if not last: 124 ↛ 125line 124 didn't jump to line 125 because the condition on line 124 was never true
125 continue
126 escaped = re.escape(last)
127 pattern = re.compile(rf"(?<![\w.]){escaped}\s*\(|\.{escaped}\b")
128 if any(pattern.search(source) for source in test_sources.values()):
129 continue
130 issues.append(
131 Issue(
132 rule_id="tests.interface-untested",
133 severity=severity,
134 message=f"Interface method {name!r} is not called by any test.",
135 category=CATEGORY,
136 recommendation=f"Add a test that exercises {name}.",
137 )
138 )
139 return issues
142# ----- changed-line coverage -----------------------------------------------------------------
143def _coverage_entry(path: str, files: Mapping[str, Any]) -> Mapping[str, Any] | None:
144 for key, entry in files.items():
145 normalized = str(key).replace("\\", "/").removeprefix("./")
146 if (normalized == path or normalized.endswith("/" + path)) and isinstance(entry, Mapping):
147 return entry
148 return None
151def _line_numbers(value: Any) -> set[int]:
152 if not isinstance(value, list):
153 return set()
154 return {item for item in value if isinstance(item, int) and not isinstance(item, bool)}
157def _code(text: str) -> bool:
158 stripped = text.strip()
159 return bool(stripped) and not stripped.startswith("#")
162def changed_line_coverage(
163 diff: Sequence[DiffFile],
164 coverage: Mapping[str, Any],
165 *,
166 minimum_percent: float,
167 severity: FindingSeverity,
168) -> tuple[float | None, list[Issue]]:
169 """The percentage of executable added lines (non-test ``.py`` files) that the tests ran,
170 and the issues when it is below ``minimum_percent``.
172 Whole-project coverage hides an untested change behind well-tested old code; measuring only
173 the changed lines does not. A changed file absent from the report was never imported by the
174 tests, which is reported on its own whatever the percentage."""
175 files = coverage.get("files")
176 measured: Mapping[str, Any] = files if isinstance(files, Mapping) else {}
177 executed_total = missing_total = 0
178 uncovered: list[tuple[str, list[int]]] = []
179 issues: list[Issue] = []
180 for file in diff:
181 found = _changed_code_lines(file)
182 if found is None:
183 continue
184 path, changed = found
185 entry = _coverage_entry(path, measured)
186 if entry is None:
187 issues.append(
188 Issue(
189 rule_id="tests.changed-file-not-executed",
190 severity=severity,
191 message=f"{path} changed ({len(changed)} line(s)) but the tests never "
192 "executed it.",
193 path=path,
194 category=CATEGORY,
195 recommendation="Add tests that import and exercise the changed module.",
196 )
197 )
198 continue
199 executed = changed & _line_numbers(entry.get("executed_lines"))
200 missing = changed & _line_numbers(entry.get("missing_lines"))
201 executed_total += len(executed)
202 missing_total += len(missing)
203 if missing:
204 uncovered.append((path, sorted(missing)))
205 total = executed_total + missing_total
206 percent = executed_total / total * 100 if total else None
207 if percent is not None and percent < minimum_percent:
208 for path, lines in uncovered:
209 issues.append(_uncovered_issue(path, lines, percent, minimum_percent, severity))
210 return percent, issues
213def _changed_code_lines(file: DiffFile) -> tuple[str, set[int]] | None:
214 """The path and added code line numbers of a changed non-test ``.py`` file; ``None`` for
215 any other file or a change without code lines."""
216 path = (file.new_path or "").replace("\\", "/").removeprefix("./")
217 if file.is_deleted or not path.endswith(".py") or is_test_path(path):
218 return None
219 changed = {line.number for line in file.added if _code(line.text)}
220 if not changed: 220 ↛ 221line 220 didn't jump to line 221 because the condition on line 220 was never true
221 return None
222 return path, changed
225def _uncovered_issue(
226 path: str,
227 lines: list[int],
228 percent: float,
229 minimum_percent: float,
230 severity: FindingSeverity,
231) -> Issue:
232 listed = ", ".join(str(number) for number in lines[:_MAX_LISTED_LINES])
233 more = ", …" if len(lines) > _MAX_LISTED_LINES else ""
234 return Issue(
235 rule_id="tests.changed-lines-uncovered",
236 severity=severity,
237 message=f"{len(lines)} changed line(s) of {path} are not executed by the "
238 f"tests (lines {listed}{more}); changed-line coverage {percent:.1f}% is "
239 f"below {minimum_percent:g}%.",
240 path=path,
241 line=lines[0],
242 category=CATEGORY,
243 recommendation="Add tests that reach the listed changed lines.",
244 )
247# ----- pytest node ids -----------------------------------------------------------------------
248_SUMMARY_ID = re.compile(r"^(?:FAILED|ERROR)\s+(\S+::\S+)", re.MULTILINE)
249_VERBOSE_ID = re.compile(r"^(?=\S+::\S)(\S+)\s+(?:FAILED|ERROR)\b", re.MULTILINE)
252def pytest_node_ids(text: str) -> list[str]:
253 """The failing node ids (``path::name``) of a pytest output, in order and without repeats,
254 from the short summary (``FAILED path::name - ...``) and from verbose result lines
255 (``path::name FAILED``), so two runs can be compared test by test."""
256 found: list[tuple[int, str]] = [
257 (match.start(), match.group(1)) for match in _SUMMARY_ID.finditer(text)
258 ]
259 found.extend((match.start(), match.group(1)) for match in _VERBOSE_ID.finditer(text))
260 ids: list[str] = []
261 for _, node_id in sorted(found):
262 if node_id not in ids:
263 ids.append(node_id)
264 return ids