Coverage for src/governed_harness/checks/suite_quality.py: 96%

131 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Checks of test quality: tests that cannot fail, interface methods no test calls, and changed 

2lines no test executes. 

3 

4A green suite only means something when its tests assert, reach the code under change and 

5exercise the contract the task declared. These checks read sources, a diff and the JSON report of 

6``coverage json``; they never run the tests themselves.""" 

7 

8from __future__ import annotations 

9 

10import ast 

11import re 

12from collections.abc import Iterator, Mapping, Sequence 

13from typing import Any 

14 

15from governed_harness.domain.enums import FindingSeverity 

16 

17from .model import DiffFile, Issue, is_test_path 

18 

19CATEGORY = "tests" 

20_MAX_LISTED_LINES = 15 

21_ASSERTING_CALLS = frozenset( 

22 {"pytest.fail", "pytest.raises", "pytest.warns", "raises", "warns", "self.fail"} 

23) 

24 

25 

26# ----- assertion-free tests ------------------------------------------------------------------ 

27def _dotted(node: ast.expr) -> str: 

28 if isinstance(node, ast.Name): 

29 return node.id 

30 if isinstance(node, ast.Attribute): 30 ↛ 33line 30 didn't jump to line 33 because the condition on line 30 was always true

31 base = _dotted(node.value) 

32 return f"{base}.{node.attr}" if base else node.attr 

33 return "" 

34 

35 

36def _call_name(node: ast.Call) -> str: 

37 if isinstance(node.func, ast.Attribute): 

38 return node.func.attr 

39 if isinstance(node.func, ast.Name): 39 ↛ 41line 39 didn't jump to line 41 because the condition on line 39 was always true

40 return node.func.id 

41 return "" 

42 

43 

44def _asserts(function: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: 

45 """Whether the test body can fail on purpose: an ``assert``, an ``assert*`` call (unittest, 

46 mock), or a ``pytest.raises``/``pytest.warns``/``pytest.fail`` call or context.""" 

47 for node in ast.walk(function): 

48 if isinstance(node, ast.Assert): 

49 return True 

50 if isinstance(node, ast.Call): 

51 if "assert" in _call_name(node).lower(): 

52 return True 

53 if _dotted(node.func) in _ASSERTING_CALLS: 

54 return True 

55 return False 

56 

57 

58def _tests(body: Sequence[ast.stmt]) -> Iterator[ast.FunctionDef | ast.AsyncFunctionDef]: 

59 for node in body: 

60 if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name.startswith( 

61 "test_" 

62 ): 

63 yield node 

64 elif isinstance(node, ast.ClassDef) and node.name.startswith("Test"): 

65 yield from _tests(node.body) 

66 

67 

68def assertion_free_tests(files: Mapping[str, str], *, severity: FindingSeverity) -> list[Issue]: 

69 """One issue per ``test_*`` function or ``Test*`` method that asserts nothing. 

70 

71 Unparsable test files yield an INFO ``tests.unparsable`` issue: the suite would fail to 

72 collect them anyway, and a check must not crash on agent output.""" 

73 issues: list[Issue] = [] 

74 for path in sorted(files): 

75 if not path.endswith(".py") or not is_test_path(path): 

76 continue 

77 try: 

78 tree = ast.parse(files[path], filename=path) 

79 except (SyntaxError, ValueError) as error: 

80 issues.append( 

81 Issue( 

82 rule_id="tests.unparsable", 

83 severity=FindingSeverity.INFO, 

84 message=f"{path} could not be parsed: {error.__class__.__name__}", 

85 path=path, 

86 line=getattr(error, "lineno", None), 

87 category=CATEGORY, 

88 ) 

89 ) 

90 continue 

91 for function in _tests(tree.body): 

92 if not _asserts(function): 

93 issues.append( 

94 Issue( 

95 rule_id="tests.assertion-free", 

96 severity=severity, 

97 message=f"Test {function.name!r} has no assertion; it can only fail by " 

98 "raising.", 

99 path=path, 

100 line=function.lineno, 

101 category=CATEGORY, 

102 recommendation="Assert on the observable result of the behavior under " 

103 "test.", 

104 ) 

105 ) 

106 return issues 

107 

108 

109# ----- interface methods --------------------------------------------------------------------- 

110def untested_interface_methods( 

111 names: Sequence[str], test_sources: Mapping[str, str], *, severity: FindingSeverity 

112) -> list[Issue]: 

113 """One issue per declared interface method whose name no test calls or reads. 

114 

115 The match is lexical (``advance(`` or ``.advance``): it proves a test mentions the method, 

116 not that it covers it, which is why it complements changed-line coverage.""" 

117 issues: list[Issue] = [] 

118 seen: set[str] = set() 

119 for name in names: 

120 if not name or name in seen: 

121 continue 

122 seen.add(name) 

123 last = name.rsplit(".", 1)[-1] 

124 if not last: 124 ↛ 125line 124 didn't jump to line 125 because the condition on line 124 was never true

125 continue 

126 escaped = re.escape(last) 

127 pattern = re.compile(rf"(?<![\w.]){escaped}\s*\(|\.{escaped}\b") 

128 if any(pattern.search(source) for source in test_sources.values()): 

129 continue 

130 issues.append( 

131 Issue( 

132 rule_id="tests.interface-untested", 

133 severity=severity, 

134 message=f"Interface method {name!r} is not called by any test.", 

135 category=CATEGORY, 

136 recommendation=f"Add a test that exercises {name}.", 

137 ) 

138 ) 

139 return issues 

140 

141 

142# ----- changed-line coverage ----------------------------------------------------------------- 

143def _coverage_entry(path: str, files: Mapping[str, Any]) -> Mapping[str, Any] | None: 

144 for key, entry in files.items(): 

145 normalized = str(key).replace("\\", "/").removeprefix("./") 

146 if (normalized == path or normalized.endswith("/" + path)) and isinstance(entry, Mapping): 

147 return entry 

148 return None 

149 

150 

151def _line_numbers(value: Any) -> set[int]: 

152 if not isinstance(value, list): 

153 return set() 

154 return {item for item in value if isinstance(item, int) and not isinstance(item, bool)} 

155 

156 

157def _code(text: str) -> bool: 

158 stripped = text.strip() 

159 return bool(stripped) and not stripped.startswith("#") 

160 

161 

162def changed_line_coverage( 

163 diff: Sequence[DiffFile], 

164 coverage: Mapping[str, Any], 

165 *, 

166 minimum_percent: float, 

167 severity: FindingSeverity, 

168) -> tuple[float | None, list[Issue]]: 

169 """The percentage of executable added lines (non-test ``.py`` files) that the tests ran, 

170 and the issues when it is below ``minimum_percent``. 

171 

172 Whole-project coverage hides an untested change behind well-tested old code; measuring only 

173 the changed lines does not. A changed file absent from the report was never imported by the 

174 tests, which is reported on its own whatever the percentage.""" 

175 files = coverage.get("files") 

176 measured: Mapping[str, Any] = files if isinstance(files, Mapping) else {} 

177 executed_total = missing_total = 0 

178 uncovered: list[tuple[str, list[int]]] = [] 

179 issues: list[Issue] = [] 

180 for file in diff: 

181 found = _changed_code_lines(file) 

182 if found is None: 

183 continue 

184 path, changed = found 

185 entry = _coverage_entry(path, measured) 

186 if entry is None: 

187 issues.append( 

188 Issue( 

189 rule_id="tests.changed-file-not-executed", 

190 severity=severity, 

191 message=f"{path} changed ({len(changed)} line(s)) but the tests never " 

192 "executed it.", 

193 path=path, 

194 category=CATEGORY, 

195 recommendation="Add tests that import and exercise the changed module.", 

196 ) 

197 ) 

198 continue 

199 executed = changed & _line_numbers(entry.get("executed_lines")) 

200 missing = changed & _line_numbers(entry.get("missing_lines")) 

201 executed_total += len(executed) 

202 missing_total += len(missing) 

203 if missing: 

204 uncovered.append((path, sorted(missing))) 

205 total = executed_total + missing_total 

206 percent = executed_total / total * 100 if total else None 

207 if percent is not None and percent < minimum_percent: 

208 for path, lines in uncovered: 

209 issues.append(_uncovered_issue(path, lines, percent, minimum_percent, severity)) 

210 return percent, issues 

211 

212 

213def _changed_code_lines(file: DiffFile) -> tuple[str, set[int]] | None: 

214 """The path and added code line numbers of a changed non-test ``.py`` file; ``None`` for 

215 any other file or a change without code lines.""" 

216 path = (file.new_path or "").replace("\\", "/").removeprefix("./") 

217 if file.is_deleted or not path.endswith(".py") or is_test_path(path): 

218 return None 

219 changed = {line.number for line in file.added if _code(line.text)} 

220 if not changed: 220 ↛ 221line 220 didn't jump to line 221 because the condition on line 220 was never true

221 return None 

222 return path, changed 

223 

224 

225def _uncovered_issue( 

226 path: str, 

227 lines: list[int], 

228 percent: float, 

229 minimum_percent: float, 

230 severity: FindingSeverity, 

231) -> Issue: 

232 listed = ", ".join(str(number) for number in lines[:_MAX_LISTED_LINES]) 

233 more = ", …" if len(lines) > _MAX_LISTED_LINES else "" 

234 return Issue( 

235 rule_id="tests.changed-lines-uncovered", 

236 severity=severity, 

237 message=f"{len(lines)} changed line(s) of {path} are not executed by the " 

238 f"tests (lines {listed}{more}); changed-line coverage {percent:.1f}% is " 

239 f"below {minimum_percent:g}%.", 

240 path=path, 

241 line=lines[0], 

242 category=CATEGORY, 

243 recommendation="Add tests that reach the listed changed lines.", 

244 ) 

245 

246 

247# ----- pytest node ids ----------------------------------------------------------------------- 

248_SUMMARY_ID = re.compile(r"^(?:FAILED|ERROR)\s+(\S+::\S+)", re.MULTILINE) 

249_VERBOSE_ID = re.compile(r"^(?=\S+::\S)(\S+)\s+(?:FAILED|ERROR)\b", re.MULTILINE) 

250 

251 

252def pytest_node_ids(text: str) -> list[str]: 

253 """The failing node ids (``path::name``) of a pytest output, in order and without repeats, 

254 from the short summary (``FAILED path::name - ...``) and from verbose result lines 

255 (``path::name FAILED``), so two runs can be compared test by test.""" 

256 found: list[tuple[int, str]] = [ 

257 (match.start(), match.group(1)) for match in _SUMMARY_ID.finditer(text) 

258 ] 

259 found.extend((match.start(), match.group(1)) for match in _VERBOSE_ID.finditer(text)) 

260 ids: list[str] = [] 

261 for _, node_id in sorted(found): 

262 if node_id not in ids: 

263 ids.append(node_id) 

264 return ids