Coverage for src/governed_harness/checks/weakened.py: 99%
176 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Detector of weakened controls in a ChangeSet diff.
3An agent asked to make a build green can do so by removing what makes it red: deleting tests,
4dropping assertions, skipping cases, silencing linters, lowering coverage thresholds or turning a
5failing command into a successful one. None of that is visible in a passing test run, so this
6check reads the diff itself. It is a heuristic over added and removed lines (the diff carries no
7context lines), tuned to point a reviewer at the line, not to prove intent.
9Documentation files (``.md``, ``.rst``, ``.adoc``) are skipped by the line-pattern rules: prose
10that *mentions* ``|| true`` or ``# noqa`` weakens nothing."""
12from __future__ import annotations
14import re
15from collections.abc import Iterable, Sequence
16from dataclasses import replace
17from pathlib import PurePosixPath
18from typing import Literal
20from governed_harness.domain.enums import FindingSeverity
22from .model import DiffFile, DiffLine, Issue, is_test_path
24CATEGORY = "weakened-controls"
25_QUOTE_CHARS = 160
26_DOC_SUFFIXES = (".md", ".rst", ".adoc")
27_TEST_DELETED = "weakened.test-deleted"
28_THRESHOLD_CHANGED = "weakened.threshold-changed"
30_RECOMMENDATIONS = {
31 _TEST_DELETED: "Restore the test or justify its removal in the ChangeSet rationale.",
32 "weakened.assert-removed": "Keep the assertions; a test without them cannot fail.",
33 "weakened.skip-added": "Fix the failing case instead of skipping it, or record an approved exception.",
34 "weakened.suppression-added": "Fix the reported problem instead of suppressing the tool.",
35 _THRESHOLD_CHANGED: "Keep the quality threshold or strictness; lowering it needs a human decision.",
36 "weakened.broad-except": "Catch the specific exception and handle or re-raise it.",
37 "weakened.error-to-success": "Let the failure propagate; do not convert an error into success.",
38}
40# ----- tests ---------------------------------------------------------------------------------
41_PY_TEST_DEF = re.compile(r"^\s*(?:async\s+)?def\s+(test_\w*)\s*\(")
42_JS_TEST = re.compile(r"(?<![\w.])(?:it|test)\s*\(\s*([\"'`])(.+?)\1")
43_ASSERTION = re.compile(
44 r"\bassert\s|\bassert\(|self\.assert|assert_called|pytest\.raises|\bexpect\("
45)
46_SKIP = re.compile(
47 r"pytest\.mark\.skip\b|pytest\.mark\.skipif\b|pytest\.mark\.xfail\b|pytest\.skip\("
48 r"|@unittest\.skip|unittest\.expectedFailure"
49 r"|(?<![\w.])(?:it|test|describe)\.skip\(|(?<![\w.])xit\(|(?<![\w.])xdescribe\("
50)
51_SUPPRESSION = re.compile(
52 r"#\s*noqa\b|#\s*nosec\b|#\s*type:\s*ignore\b|#\s*pragma:\s*no\s+cover\b"
53 r"|eslint-disable|@ts-ignore\b|@ts-expect-error\b|#\s*pylint:\s*disable\b"
54)
56# ----- configuration -------------------------------------------------------------------------
57_CONFIG_NAMES = frozenset(
58 {
59 "pyproject.toml",
60 "setup.cfg",
61 "tox.ini",
62 ".coveragerc",
63 "mypy.ini",
64 "ruff.toml",
65 ".ruff.toml",
66 "pytest.ini",
67 "package.json",
68 }
69)
70_THRESHOLD = re.compile(
71 r"(?P<key>--cov-fail-under|fail[_-]under|minimumPercent|coverageThreshold|threshold"
72 r"|max[_-]complexity|branches|lines|statements|functions)(?![\w-])"
73 r"[\"']?\s*[:=]?\s*[\"']?(?P<number>\d+(?:\.\d+)?)"
74)
75_STRICT = re.compile(r"[\"']?\bstrict[\"']?\s*[:=]\s*true\b|--strict(?:-[a-z]+)*\b", re.IGNORECASE)
76_CHECK_TOOL = re.compile(r"\b(pytest|ruff|mypy|eslint|tsc|bandit)\b|\bnpm\s+(?:run\s+)?test\b")
77_NOT_A_RUN = (
78 re.compile(r"^\s*(?:(?:-\s*)?(?:name|uses)\s*:|#|(?:echo|cat|printf)\b)"),
79 re.compile(r"\binstall\b|\bnpm\s+ci\b"),
80)
82# ----- error handling ------------------------------------------------------------------------
83_EXCEPT = re.compile(
84 r"^\s*except\s*(?::|(?:\(\s*)?(?:Exception|BaseException)\b[^:]*:)(?P<rest>.*)$"
85)
86_SWALLOW = re.compile(r"^(?:pass|\.\.\.|continue|return|return\s+None)\s*(?:#.*)?$")
87_ERROR_TO_SUCCESS = re.compile(
88 r"\|\|\s*true\b|\|\|\s*exit\s+0\b|continue-on-error:\s*true\b|--exit-zero\b"
89 r"|\bcheck\s*=\s*False\b|\bignore_errors\s*[=:]\s*true\b|\breturncode\s*=(?!=)\s*0\b",
90 re.IGNORECASE,
91)
92_EXIT_ZERO = re.compile(r"(?<![\w.])(?:sys\.)?exit\(\s*0\s*\)")
95def _quote(text: str) -> str:
96 value = text.strip()
97 return value if len(value) <= _QUOTE_CHARS else value[: _QUOTE_CHARS - 1] + "…"
100def _issue(
101 rule: str, severity: FindingSeverity, message: str, path: str, line: int | None
102) -> Issue:
103 return Issue(
104 rule_id=rule,
105 severity=severity,
106 message=message,
107 path=path,
108 line=line,
109 category=CATEGORY,
110 recommendation=_RECOMMENDATIONS[rule],
111 )
114def _is_doc(path: str) -> bool:
115 return path.lower().endswith(_DOC_SUFFIXES)
118def _is_config(path: str) -> bool:
119 normalized = path.replace("\\", "/")
120 name = PurePosixPath(normalized).name
121 return (
122 name in _CONFIG_NAMES
123 or name.startswith((".eslintrc", "jest.config."))
124 or name.endswith((".yml", ".yaml"))
125 )
128def _is_workflow(path: str) -> bool:
129 normalized = path.replace("\\", "/")
130 return ".github/workflows/" in normalized or normalized.endswith(".gitlab-ci.yml")
133def _test_names(lines: Iterable[DiffLine]) -> list[tuple[str, DiffLine]]:
134 names: list[tuple[str, DiffLine]] = []
135 for line in lines:
136 python = _PY_TEST_DEF.match(line.text)
137 if python:
138 names.append((python.group(1), line))
139 continue
140 javascript = _JS_TEST.search(line.text)
141 if javascript:
142 names.append((javascript.group(2), line))
143 return names
146def _deleted_tests(file: DiffFile) -> list[Issue]:
147 path = file.path
148 if not is_test_path(path):
149 return []
150 if file.is_deleted:
151 return [
152 _issue(
153 _TEST_DELETED,
154 FindingSeverity.HIGH,
155 f"Test file {path} was deleted.",
156 path,
157 None,
158 )
159 ]
160 added = {name for name, _ in _test_names(file.added)}
161 return [
162 _issue(
163 _TEST_DELETED,
164 FindingSeverity.HIGH,
165 f"Test {name!r} was removed and not re-added: {_quote(line.text)!r}",
166 path,
167 line.number,
168 )
169 for name, line in _test_names(file.removed)
170 if name not in added
171 ]
174def _is_comment(text: str) -> bool:
175 return text.lstrip().startswith(("#", "//"))
178def _removed_assertions(file: DiffFile) -> list[Issue]:
179 if file.is_deleted or not is_test_path(file.path):
180 return []
181 removed = [
182 line for line in file.removed if not _is_comment(line.text) and _ASSERTION.search(line.text)
183 ]
184 added = [
185 line for line in file.added if not _is_comment(line.text) and _ASSERTION.search(line.text)
186 ]
187 if len(removed) <= len(added):
188 return []
189 first = removed[0]
190 return [
191 _issue(
192 "weakened.assert-removed",
193 FindingSeverity.HIGH,
194 f"{len(removed)} assertion line(s) removed and {len(added)} added in {file.path}; "
195 f"first removed: {_quote(first.text)!r}",
196 file.path,
197 first.number,
198 )
199 ]
202def _pattern_lines(
203 file: DiffFile, pattern: re.Pattern[str], rule: str, severity: FindingSeverity, what: str
204) -> list[Issue]:
205 return [
206 _issue(rule, severity, f"{what}: {_quote(line.text)!r}", file.path, line.number)
207 for line in file.added
208 if pattern.search(line.text)
209 ]
212def _threshold_key(raw: str) -> str:
213 key = raw.lower().replace("_", "-")
214 return "fail-under" if key == "--cov-fail-under" else key
217def _thresholds(lines: Iterable[DiffLine]) -> list[tuple[str, float, DiffLine]]:
218 found: list[tuple[str, float, DiffLine]] = []
219 for line in lines:
220 if _is_comment(line.text): 220 ↛ 221line 220 didn't jump to line 221 because the condition on line 220 was never true
221 continue
222 for match in _THRESHOLD.finditer(line.text):
223 found.append((_threshold_key(match.group("key")), float(match.group("number")), line))
224 return found
227def _threshold_changes(file: DiffFile) -> list[Issue]:
228 path = file.path
229 if not _is_config(path):
230 return []
231 issues = _lowered_thresholds(file, path)
232 issues.extend(_strict_removed(file, path))
233 if _is_workflow(path):
234 issues.extend(_workflow_checks_removed(file, path))
235 return issues
238def _lowered_thresholds(file: DiffFile, path: str) -> list[Issue]:
239 """Removed thresholds that no added threshold of the same key replaces at least as high."""
240 issues: list[Issue] = []
241 added_by_key: dict[str, list[float]] = {}
242 for key, number, _ in _thresholds(file.added):
243 added_by_key.setdefault(key, []).append(number)
244 seen: dict[str, int] = {}
245 for key, number, line in _thresholds(file.removed):
246 position = seen.get(key, 0)
247 seen[key] = position + 1
248 counterparts = added_by_key.get(key, [])
249 if position >= len(counterparts):
250 reason = f"threshold {key!r} ({number:g}) removed without replacement"
251 elif counterparts[position] < number:
252 reason = f"threshold {key!r} lowered from {number:g} to {counterparts[position]:g}"
253 else:
254 continue
255 issues.append(
256 _issue(
257 _THRESHOLD_CHANGED,
258 FindingSeverity.HIGH,
259 f"{reason}: {_quote(line.text)!r}",
260 path,
261 line.number,
262 )
263 )
264 return issues
267def _strict_removed(file: DiffFile, path: str) -> list[Issue]:
268 """Removed strict-mode settings that the added lines do not restore."""
269 issues: list[Issue] = []
270 added_strict = {
271 match.group(0).lower().replace(" ", "")
272 for line in file.added
273 for match in _STRICT.finditer(line.text)
274 }
275 for line in file.removed:
276 match = _STRICT.search(line.text)
277 if match and match.group(0).lower().replace(" ", "") not in added_strict:
278 issues.append(
279 _issue(
280 _THRESHOLD_CHANGED,
281 FindingSeverity.HIGH,
282 f"strict mode removed: {_quote(line.text)!r}",
283 path,
284 line.number,
285 )
286 )
287 return issues
290def _workflow_checks_removed(file: DiffFile, path: str) -> list[Issue]:
291 """Removed workflow lines whose check tools no added line runs any more."""
292 issues: list[Issue] = []
293 added_tools = {_tool(match) for line in file.added for match in _CHECK_TOOL.finditer(line.text)}
294 for line in file.removed:
295 if any(pattern.search(line.text) for pattern in _NOT_A_RUN):
296 continue
297 tools = {_tool(match) for match in _CHECK_TOOL.finditer(line.text)}
298 missing = sorted(tools - added_tools)
299 if missing:
300 issues.append(
301 _issue(
302 _THRESHOLD_CHANGED,
303 FindingSeverity.HIGH,
304 f"workflow no longer runs {', '.join(missing)}: {_quote(line.text)!r}",
305 path,
306 line.number,
307 )
308 )
309 return issues
312def _tool(match: re.Match[str]) -> str:
313 return match.group(1) or "npm test"
316def _contiguous(lines: Sequence[DiffLine]) -> Iterable[tuple[DiffLine, DiffLine | None]]:
317 """Each added line with the next added line when it directly follows it in the new file
318 (the same hunk), else ``None``."""
319 for index, line in enumerate(lines):
320 following = lines[index + 1] if index + 1 < len(lines) else None
321 yield line, following if following and following.number == line.number + 1 else None
324def _broad_excepts(file: DiffFile) -> list[Issue]:
325 issues: list[Issue] = []
326 for line, following in _contiguous(file.added):
327 match = _EXCEPT.match(line.text)
328 if not match:
329 continue
330 rest = match.group("rest").strip()
331 if rest and not rest.startswith("#"):
332 swallowed = _SWALLOW.match(rest) is not None
333 else:
334 swallowed = following is not None and _SWALLOW.match(following.text.strip()) is not None
335 if swallowed:
336 issues.append(
337 _issue(
338 "weakened.broad-except",
339 FindingSeverity.MEDIUM,
340 f"broad exception handler swallows errors: {_quote(line.text)!r}",
341 file.path,
342 line.number,
343 )
344 )
345 return issues
348def _error_to_success(file: DiffFile) -> list[Issue]:
349 issues: list[Issue] = []
350 previous_except: int | None = None
351 for line in file.added:
352 exits_after_except = previous_except is not None and line.number == previous_except + 1
353 # A comment that explains a flag is not the flag; ``rmtree(..., ignore_errors=True)``
354 # is cleanup, not a check whose failure is hidden.
355 if _is_comment(line.text) or "rmtree(" in line.text:
356 previous_except = None
357 continue
358 if _ERROR_TO_SUCCESS.search(line.text) or (
359 exits_after_except and _EXIT_ZERO.search(line.text)
360 ):
361 issues.append(
362 _issue(
363 "weakened.error-to-success",
364 FindingSeverity.HIGH,
365 f"an error is turned into success: {_quote(line.text)!r}",
366 file.path,
367 line.number,
368 )
369 )
370 previous_except = line.number if re.match(r"^\s*except\b", line.text) else None
371 return issues
374def check_weakened_controls(
375 diff: Sequence[DiffFile], policy: Literal["enforce", "warn"]
376) -> list[Issue]:
377 """Issues for every weakened control the diff introduces, in diff order.
379 Under ``warn`` every issue is LOW: the evidence is still recorded, but it does not block."""
380 issues: list[Issue] = []
381 for file in diff:
382 issues.extend(_deleted_tests(file))
383 issues.extend(_removed_assertions(file))
384 if not _is_doc(file.path):
385 issues.extend(
386 _pattern_lines(
387 file,
388 _SKIP,
389 "weakened.skip-added",
390 FindingSeverity.HIGH,
391 "test skip or expected-failure marker added",
392 )
393 )
394 issues.extend(
395 _pattern_lines(
396 file,
397 _SUPPRESSION,
398 "weakened.suppression-added",
399 FindingSeverity.MEDIUM,
400 "tool suppression added",
401 )
402 )
403 issues.extend(_threshold_changes(file))
404 if not _is_doc(file.path):
405 issues.extend(_broad_excepts(file))
406 issues.extend(_error_to_success(file))
407 if policy == "warn":
408 return [replace(issue, severity=FindingSeverity.LOW) for issue in issues]
409 return issues