Coverage for src/governed_harness/checks/weakened.py: 99%

176 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Detector of weakened controls in a ChangeSet diff. 

2 

3An agent asked to make a build green can do so by removing what makes it red: deleting tests, 

4dropping assertions, skipping cases, silencing linters, lowering coverage thresholds or turning a 

5failing command into a successful one. None of that is visible in a passing test run, so this 

6check reads the diff itself. It is a heuristic over added and removed lines (the diff carries no 

7context lines), tuned to point a reviewer at the line, not to prove intent. 

8 

9Documentation files (``.md``, ``.rst``, ``.adoc``) are skipped by the line-pattern rules: prose 

10that *mentions* ``|| true`` or ``# noqa`` weakens nothing.""" 

11 

12from __future__ import annotations 

13 

14import re 

15from collections.abc import Iterable, Sequence 

16from dataclasses import replace 

17from pathlib import PurePosixPath 

18from typing import Literal 

19 

20from governed_harness.domain.enums import FindingSeverity 

21 

22from .model import DiffFile, DiffLine, Issue, is_test_path 

23 

24CATEGORY = "weakened-controls" 

25_QUOTE_CHARS = 160 

26_DOC_SUFFIXES = (".md", ".rst", ".adoc") 

27_TEST_DELETED = "weakened.test-deleted" 

28_THRESHOLD_CHANGED = "weakened.threshold-changed" 

29 

30_RECOMMENDATIONS = { 

31 _TEST_DELETED: "Restore the test or justify its removal in the ChangeSet rationale.", 

32 "weakened.assert-removed": "Keep the assertions; a test without them cannot fail.", 

33 "weakened.skip-added": "Fix the failing case instead of skipping it, or record an approved exception.", 

34 "weakened.suppression-added": "Fix the reported problem instead of suppressing the tool.", 

35 _THRESHOLD_CHANGED: "Keep the quality threshold or strictness; lowering it needs a human decision.", 

36 "weakened.broad-except": "Catch the specific exception and handle or re-raise it.", 

37 "weakened.error-to-success": "Let the failure propagate; do not convert an error into success.", 

38} 

39 

40# ----- tests --------------------------------------------------------------------------------- 

41_PY_TEST_DEF = re.compile(r"^\s*(?:async\s+)?def\s+(test_\w*)\s*\(") 

42_JS_TEST = re.compile(r"(?<![\w.])(?:it|test)\s*\(\s*([\"'`])(.+?)\1") 

43_ASSERTION = re.compile( 

44 r"\bassert\s|\bassert\(|self\.assert|assert_called|pytest\.raises|\bexpect\(" 

45) 

46_SKIP = re.compile( 

47 r"pytest\.mark\.skip\b|pytest\.mark\.skipif\b|pytest\.mark\.xfail\b|pytest\.skip\(" 

48 r"|@unittest\.skip|unittest\.expectedFailure" 

49 r"|(?<![\w.])(?:it|test|describe)\.skip\(|(?<![\w.])xit\(|(?<![\w.])xdescribe\(" 

50) 

51_SUPPRESSION = re.compile( 

52 r"#\s*noqa\b|#\s*nosec\b|#\s*type:\s*ignore\b|#\s*pragma:\s*no\s+cover\b" 

53 r"|eslint-disable|@ts-ignore\b|@ts-expect-error\b|#\s*pylint:\s*disable\b" 

54) 

55 

56# ----- configuration ------------------------------------------------------------------------- 

57_CONFIG_NAMES = frozenset( 

58 { 

59 "pyproject.toml", 

60 "setup.cfg", 

61 "tox.ini", 

62 ".coveragerc", 

63 "mypy.ini", 

64 "ruff.toml", 

65 ".ruff.toml", 

66 "pytest.ini", 

67 "package.json", 

68 } 

69) 

70_THRESHOLD = re.compile( 

71 r"(?P<key>--cov-fail-under|fail[_-]under|minimumPercent|coverageThreshold|threshold" 

72 r"|max[_-]complexity|branches|lines|statements|functions)(?![\w-])" 

73 r"[\"']?\s*[:=]?\s*[\"']?(?P<number>\d+(?:\.\d+)?)" 

74) 

75_STRICT = re.compile(r"[\"']?\bstrict[\"']?\s*[:=]\s*true\b|--strict(?:-[a-z]+)*\b", re.IGNORECASE) 

76_CHECK_TOOL = re.compile(r"\b(pytest|ruff|mypy|eslint|tsc|bandit)\b|\bnpm\s+(?:run\s+)?test\b") 

77_NOT_A_RUN = ( 

78 re.compile(r"^\s*(?:(?:-\s*)?(?:name|uses)\s*:|#|(?:echo|cat|printf)\b)"), 

79 re.compile(r"\binstall\b|\bnpm\s+ci\b"), 

80) 

81 

82# ----- error handling ------------------------------------------------------------------------ 

83_EXCEPT = re.compile( 

84 r"^\s*except\s*(?::|(?:\(\s*)?(?:Exception|BaseException)\b[^:]*:)(?P<rest>.*)$" 

85) 

86_SWALLOW = re.compile(r"^(?:pass|\.\.\.|continue|return|return\s+None)\s*(?:#.*)?$") 

87_ERROR_TO_SUCCESS = re.compile( 

88 r"\|\|\s*true\b|\|\|\s*exit\s+0\b|continue-on-error:\s*true\b|--exit-zero\b" 

89 r"|\bcheck\s*=\s*False\b|\bignore_errors\s*[=:]\s*true\b|\breturncode\s*=(?!=)\s*0\b", 

90 re.IGNORECASE, 

91) 

92_EXIT_ZERO = re.compile(r"(?<![\w.])(?:sys\.)?exit\(\s*0\s*\)") 

93 

94 

95def _quote(text: str) -> str: 

96 value = text.strip() 

97 return value if len(value) <= _QUOTE_CHARS else value[: _QUOTE_CHARS - 1] + "…" 

98 

99 

100def _issue( 

101 rule: str, severity: FindingSeverity, message: str, path: str, line: int | None 

102) -> Issue: 

103 return Issue( 

104 rule_id=rule, 

105 severity=severity, 

106 message=message, 

107 path=path, 

108 line=line, 

109 category=CATEGORY, 

110 recommendation=_RECOMMENDATIONS[rule], 

111 ) 

112 

113 

114def _is_doc(path: str) -> bool: 

115 return path.lower().endswith(_DOC_SUFFIXES) 

116 

117 

118def _is_config(path: str) -> bool: 

119 normalized = path.replace("\\", "/") 

120 name = PurePosixPath(normalized).name 

121 return ( 

122 name in _CONFIG_NAMES 

123 or name.startswith((".eslintrc", "jest.config.")) 

124 or name.endswith((".yml", ".yaml")) 

125 ) 

126 

127 

128def _is_workflow(path: str) -> bool: 

129 normalized = path.replace("\\", "/") 

130 return ".github/workflows/" in normalized or normalized.endswith(".gitlab-ci.yml") 

131 

132 

133def _test_names(lines: Iterable[DiffLine]) -> list[tuple[str, DiffLine]]: 

134 names: list[tuple[str, DiffLine]] = [] 

135 for line in lines: 

136 python = _PY_TEST_DEF.match(line.text) 

137 if python: 

138 names.append((python.group(1), line)) 

139 continue 

140 javascript = _JS_TEST.search(line.text) 

141 if javascript: 

142 names.append((javascript.group(2), line)) 

143 return names 

144 

145 

146def _deleted_tests(file: DiffFile) -> list[Issue]: 

147 path = file.path 

148 if not is_test_path(path): 

149 return [] 

150 if file.is_deleted: 

151 return [ 

152 _issue( 

153 _TEST_DELETED, 

154 FindingSeverity.HIGH, 

155 f"Test file {path} was deleted.", 

156 path, 

157 None, 

158 ) 

159 ] 

160 added = {name for name, _ in _test_names(file.added)} 

161 return [ 

162 _issue( 

163 _TEST_DELETED, 

164 FindingSeverity.HIGH, 

165 f"Test {name!r} was removed and not re-added: {_quote(line.text)!r}", 

166 path, 

167 line.number, 

168 ) 

169 for name, line in _test_names(file.removed) 

170 if name not in added 

171 ] 

172 

173 

174def _is_comment(text: str) -> bool: 

175 return text.lstrip().startswith(("#", "//")) 

176 

177 

178def _removed_assertions(file: DiffFile) -> list[Issue]: 

179 if file.is_deleted or not is_test_path(file.path): 

180 return [] 

181 removed = [ 

182 line for line in file.removed if not _is_comment(line.text) and _ASSERTION.search(line.text) 

183 ] 

184 added = [ 

185 line for line in file.added if not _is_comment(line.text) and _ASSERTION.search(line.text) 

186 ] 

187 if len(removed) <= len(added): 

188 return [] 

189 first = removed[0] 

190 return [ 

191 _issue( 

192 "weakened.assert-removed", 

193 FindingSeverity.HIGH, 

194 f"{len(removed)} assertion line(s) removed and {len(added)} added in {file.path}; " 

195 f"first removed: {_quote(first.text)!r}", 

196 file.path, 

197 first.number, 

198 ) 

199 ] 

200 

201 

202def _pattern_lines( 

203 file: DiffFile, pattern: re.Pattern[str], rule: str, severity: FindingSeverity, what: str 

204) -> list[Issue]: 

205 return [ 

206 _issue(rule, severity, f"{what}: {_quote(line.text)!r}", file.path, line.number) 

207 for line in file.added 

208 if pattern.search(line.text) 

209 ] 

210 

211 

212def _threshold_key(raw: str) -> str: 

213 key = raw.lower().replace("_", "-") 

214 return "fail-under" if key == "--cov-fail-under" else key 

215 

216 

217def _thresholds(lines: Iterable[DiffLine]) -> list[tuple[str, float, DiffLine]]: 

218 found: list[tuple[str, float, DiffLine]] = [] 

219 for line in lines: 

220 if _is_comment(line.text): 220 ↛ 221line 220 didn't jump to line 221 because the condition on line 220 was never true

221 continue 

222 for match in _THRESHOLD.finditer(line.text): 

223 found.append((_threshold_key(match.group("key")), float(match.group("number")), line)) 

224 return found 

225 

226 

227def _threshold_changes(file: DiffFile) -> list[Issue]: 

228 path = file.path 

229 if not _is_config(path): 

230 return [] 

231 issues = _lowered_thresholds(file, path) 

232 issues.extend(_strict_removed(file, path)) 

233 if _is_workflow(path): 

234 issues.extend(_workflow_checks_removed(file, path)) 

235 return issues 

236 

237 

238def _lowered_thresholds(file: DiffFile, path: str) -> list[Issue]: 

239 """Removed thresholds that no added threshold of the same key replaces at least as high.""" 

240 issues: list[Issue] = [] 

241 added_by_key: dict[str, list[float]] = {} 

242 for key, number, _ in _thresholds(file.added): 

243 added_by_key.setdefault(key, []).append(number) 

244 seen: dict[str, int] = {} 

245 for key, number, line in _thresholds(file.removed): 

246 position = seen.get(key, 0) 

247 seen[key] = position + 1 

248 counterparts = added_by_key.get(key, []) 

249 if position >= len(counterparts): 

250 reason = f"threshold {key!r} ({number:g}) removed without replacement" 

251 elif counterparts[position] < number: 

252 reason = f"threshold {key!r} lowered from {number:g} to {counterparts[position]:g}" 

253 else: 

254 continue 

255 issues.append( 

256 _issue( 

257 _THRESHOLD_CHANGED, 

258 FindingSeverity.HIGH, 

259 f"{reason}: {_quote(line.text)!r}", 

260 path, 

261 line.number, 

262 ) 

263 ) 

264 return issues 

265 

266 

267def _strict_removed(file: DiffFile, path: str) -> list[Issue]: 

268 """Removed strict-mode settings that the added lines do not restore.""" 

269 issues: list[Issue] = [] 

270 added_strict = { 

271 match.group(0).lower().replace(" ", "") 

272 for line in file.added 

273 for match in _STRICT.finditer(line.text) 

274 } 

275 for line in file.removed: 

276 match = _STRICT.search(line.text) 

277 if match and match.group(0).lower().replace(" ", "") not in added_strict: 

278 issues.append( 

279 _issue( 

280 _THRESHOLD_CHANGED, 

281 FindingSeverity.HIGH, 

282 f"strict mode removed: {_quote(line.text)!r}", 

283 path, 

284 line.number, 

285 ) 

286 ) 

287 return issues 

288 

289 

290def _workflow_checks_removed(file: DiffFile, path: str) -> list[Issue]: 

291 """Removed workflow lines whose check tools no added line runs any more.""" 

292 issues: list[Issue] = [] 

293 added_tools = {_tool(match) for line in file.added for match in _CHECK_TOOL.finditer(line.text)} 

294 for line in file.removed: 

295 if any(pattern.search(line.text) for pattern in _NOT_A_RUN): 

296 continue 

297 tools = {_tool(match) for match in _CHECK_TOOL.finditer(line.text)} 

298 missing = sorted(tools - added_tools) 

299 if missing: 

300 issues.append( 

301 _issue( 

302 _THRESHOLD_CHANGED, 

303 FindingSeverity.HIGH, 

304 f"workflow no longer runs {', '.join(missing)}: {_quote(line.text)!r}", 

305 path, 

306 line.number, 

307 ) 

308 ) 

309 return issues 

310 

311 

312def _tool(match: re.Match[str]) -> str: 

313 return match.group(1) or "npm test" 

314 

315 

316def _contiguous(lines: Sequence[DiffLine]) -> Iterable[tuple[DiffLine, DiffLine | None]]: 

317 """Each added line with the next added line when it directly follows it in the new file 

318 (the same hunk), else ``None``.""" 

319 for index, line in enumerate(lines): 

320 following = lines[index + 1] if index + 1 < len(lines) else None 

321 yield line, following if following and following.number == line.number + 1 else None 

322 

323 

324def _broad_excepts(file: DiffFile) -> list[Issue]: 

325 issues: list[Issue] = [] 

326 for line, following in _contiguous(file.added): 

327 match = _EXCEPT.match(line.text) 

328 if not match: 

329 continue 

330 rest = match.group("rest").strip() 

331 if rest and not rest.startswith("#"): 

332 swallowed = _SWALLOW.match(rest) is not None 

333 else: 

334 swallowed = following is not None and _SWALLOW.match(following.text.strip()) is not None 

335 if swallowed: 

336 issues.append( 

337 _issue( 

338 "weakened.broad-except", 

339 FindingSeverity.MEDIUM, 

340 f"broad exception handler swallows errors: {_quote(line.text)!r}", 

341 file.path, 

342 line.number, 

343 ) 

344 ) 

345 return issues 

346 

347 

348def _error_to_success(file: DiffFile) -> list[Issue]: 

349 issues: list[Issue] = [] 

350 previous_except: int | None = None 

351 for line in file.added: 

352 exits_after_except = previous_except is not None and line.number == previous_except + 1 

353 # A comment that explains a flag is not the flag; ``rmtree(..., ignore_errors=True)`` 

354 # is cleanup, not a check whose failure is hidden. 

355 if _is_comment(line.text) or "rmtree(" in line.text: 

356 previous_except = None 

357 continue 

358 if _ERROR_TO_SUCCESS.search(line.text) or ( 

359 exits_after_except and _EXIT_ZERO.search(line.text) 

360 ): 

361 issues.append( 

362 _issue( 

363 "weakened.error-to-success", 

364 FindingSeverity.HIGH, 

365 f"an error is turned into success: {_quote(line.text)!r}", 

366 file.path, 

367 line.number, 

368 ) 

369 ) 

370 previous_except = line.number if re.match(r"^\s*except\b", line.text) else None 

371 return issues 

372 

373 

374def check_weakened_controls( 

375 diff: Sequence[DiffFile], policy: Literal["enforce", "warn"] 

376) -> list[Issue]: 

377 """Issues for every weakened control the diff introduces, in diff order. 

378 

379 Under ``warn`` every issue is LOW: the evidence is still recorded, but it does not block.""" 

380 issues: list[Issue] = [] 

381 for file in diff: 

382 issues.extend(_deleted_tests(file)) 

383 issues.extend(_removed_assertions(file)) 

384 if not _is_doc(file.path): 

385 issues.extend( 

386 _pattern_lines( 

387 file, 

388 _SKIP, 

389 "weakened.skip-added", 

390 FindingSeverity.HIGH, 

391 "test skip or expected-failure marker added", 

392 ) 

393 ) 

394 issues.extend( 

395 _pattern_lines( 

396 file, 

397 _SUPPRESSION, 

398 "weakened.suppression-added", 

399 FindingSeverity.MEDIUM, 

400 "tool suppression added", 

401 ) 

402 ) 

403 issues.extend(_threshold_changes(file)) 

404 if not _is_doc(file.path): 

405 issues.extend(_broad_excepts(file)) 

406 issues.extend(_error_to_success(file)) 

407 if policy == "warn": 

408 return [replace(issue, severity=FindingSeverity.LOW) for issue in issues] 

409 return issues