Coverage for src/governed_harness/orchestration/ladder_mutation.py: 81%
184 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Discriminating evidence and light mutation (#55, item 5; ``verification.mutation``).
3A passing test suite says little when the new tests would pass without the change, or when a
4changed line is never exercised. After the mandatory validators passed, VERIFICATION:
61. **classifies the new or changed test files**: each runs on a scratch copy of the workspace in
7 which every changed source file is put back to its baseline content while the tests keep
8 their new content. A test file that fails there and passes on the change is
9 *discriminating*; one that passes before and after is *weak* (``tests.weak``); one that fails
10 after the change is *broken* (``tests.broken``);
112. **reverts each changed hunk** of the source, one at a time, in a scratch copy and runs the
12 relevant tests (the changed test files, else the mandatory test command). A hunk whose
13 reversion no test notices is a ``tests.change-not-exercised`` finding at its line.
15The order is deterministic (path, then position), blank and comment-only hunks are not
16mutated, and the number of hunks (``maxHunks``) and the time (``maxSeconds``) are capped; what
17the cap left out is recorded. The test command comes from the profile's ``mutationCommand``
18(``{paths}`` is replaced by the test files); a profile without one is skipped and the record
19says so. Under ``enforce`` the findings are HIGH and the validation is mandatory; under
20``warn`` they are LOW."""
22from __future__ import annotations
24import time
25from pathlib import Path, PurePosixPath
26from typing import TYPE_CHECKING, Any
28from governed_harness.capabilities import grants_from_rules
29from governed_harness.domain.enums import (
30 ActorType,
31 FindingSeverity,
32 PhaseId,
33 ResultStatus,
34 ValidationKind,
35)
36from governed_harness.domain.models import Actor, ChangeSet, Execution, Finding
37from governed_harness.ladder.hunks import Hunk, cosmetic, hunks, revert
38from governed_harness.orchestration.workspace_ops import Contents, materialized
39from governed_harness.runtime import CancellationToken, SafeProcessRunner
40from governed_harness.runtime.process_runner import CommandSpec
41from governed_harness.validators import ValidatorOutput
42from governed_harness.validators.traceability import is_test_file
44if TYPE_CHECKING:
45 from governed_harness.orchestration.ladder_host import LadderHost
47MUTATION_ID = "harness.mutation"
48NOT_EXERCISED_RULE = "tests.change-not-exercised"
49WEAK_RULE = "tests.weak"
50BROKEN_RULE = "tests.broken"
51MAX_CLASSIFIED_FILES = 10
52_TEST_SEGMENTS = frozenset({"test", "tests", "__tests__", "spec", "specs"})
55def _single_file(data: bytes) -> Contents:
56 """The recorded content of the one path a materialized copy changes."""
58 def content(_path: str) -> bytes | None:
59 return data
61 return Contents(lambda _path: True, content)
64def looks_like_test(path: str, technologies: list[str]) -> bool:
65 relative = PurePosixPath(path)
66 if is_test_file(relative, technologies):
67 return True
68 lowered = relative.name.lower()
69 return (
70 any(part.lower() in _TEST_SEGMENTS for part in relative.parts[:-1])
71 or lowered.startswith("test")
72 or lowered.endswith(("_test.go", "test.java", "tests.swift", "test.kt"))
73 )
76def _test_class(before: ResultStatus | None, after: ResultStatus | None) -> str:
77 """``broken`` when the test fails on the change, ``discriminating`` when it failed on the
78 baseline sources, ``weak`` when it passed there too, else ``unknown``."""
79 if after is not ResultStatus.PASSED: 79 ↛ 80line 79 didn't jump to line 80 because the condition on line 79 was never true
80 return "broken"
81 if before is ResultStatus.FAILED:
82 return "discriminating"
83 if before is ResultStatus.PASSED: 83 ↛ 85line 83 didn't jump to line 85 because the condition on line 83 was always true
84 return "weak"
85 return "unknown"
88def _texts(contents: Contents, path: str, target: Path) -> tuple[str, str] | None:
89 """The baseline and current text of a changed source, when both are readable text."""
90 data = contents.content(path) if contents.existed(path) else b""
91 if data is None or not target.is_file(): 91 ↛ 92line 91 didn't jump to line 92 because the condition on line 91 was never true
92 return None
93 try:
94 return data.decode("utf-8"), target.read_text(encoding="utf-8")
95 except (UnicodeDecodeError, OSError):
96 return None
99class Mutation:
100 def __init__(self, ladder: LadderHost) -> None:
101 self.ladder = ladder
103 @property
104 def config(self) -> Any:
105 verification = self.ladder.project.verification
106 return verification.mutation if verification else None
108 @property
109 def enabled(self) -> bool:
110 return bool(self.config and self.config.enabled)
112 def _command(self) -> tuple[str, ...] | None:
113 for _profile, verification in self.ladder.profile_verifications(): 113 ↛ 116line 113 didn't jump to line 116 because the loop on line 113 didn't complete
114 if verification.mutation_command: 114 ↛ 113line 114 didn't jump to line 113 because the condition on line 114 was always true
115 return tuple(verification.mutation_command)
116 return None
118 def _suite(self) -> tuple[str, ...] | None:
119 for definition in self.ladder.s.resolved.effective_validators:
120 if definition.mandatory and definition.command:
121 return tuple(definition.command)
122 return None
124 def run(
125 self,
126 execution: Execution,
127 change_set: ChangeSet,
128 outputs: list[Any],
129 ) -> ValidatorOutput | None:
130 if not self.enabled:
131 return None
132 results = self.ladder.engine.results
133 mandatory_failed = [
134 item.result.validator_id
135 for item in outputs
136 if item.result.mandatory and item.result.status is not ResultStatus.PASSED
137 ]
138 template = self._command()
139 record: dict[str, Any] = {
140 "changeSetDigest": change_set.digest,
141 "mode": self.config.mode,
142 "command": list(template or ()),
143 }
144 if mandatory_failed: 144 ↛ 145line 144 didn't jump to line 145 because the condition on line 144 was never true
145 return self._skipped(
146 execution,
147 change_set,
148 record,
149 f"not run: mandatory validator(s) did not pass ({', '.join(mandatory_failed)})",
150 )
151 if template is None: 151 ↛ 152line 151 didn't jump to line 152 because the condition on line 151 was never true
152 return self._skipped(
153 execution,
154 change_set,
155 record,
156 "not run: no selected profile declares a mutationCommand",
157 )
158 technologies = [profile.technology for profile in self.ladder.s.resolved.profiles]
159 contents = results.baseline_contents(execution)
160 changed = [item for item in change_set.files if item.status != "DELETED"]
161 tests = sorted(item.path for item in changed if looks_like_test(item.path, technologies))
162 sources = sorted(
163 item.path for item in changed if not looks_like_test(item.path, technologies)
164 )
165 severity = FindingSeverity.HIGH if self.config.mode == "enforce" else FindingSeverity.LOW
166 started = time.monotonic()
167 classification: list[dict[str, Any]] = []
168 findings: list[Finding] = []
169 if tests and contents is not None: 169 ↛ 176line 169 didn't jump to line 176 because the condition on line 169 was always true
170 classification = self._classify(execution, template, tests, sources, contents)
171 findings.extend(
172 self._test_finding(execution, item, severity)
173 for item in classification
174 if item["class"] in {"weak", "broken"}
175 )
176 candidates = self._candidates(sources, contents) if contents is not None else []
177 mutated, skipped_hunks = self._mutate(
178 execution,
179 candidates,
180 (template, tests) if tests else (self._suite(), []),
181 deadline=started + self.config.time_limit,
182 )
183 findings.extend(
184 self._not_exercised(execution, hunk, severity)
185 for hunk, item in mutated
186 if item["status"] == ResultStatus.PASSED.value
187 )
188 record.update(
189 {
190 "testFiles": classification,
191 "hunks": [item for _, item in mutated],
192 "hunksLeftOut": skipped_hunks,
193 "maxHunks": self.config.hunk_limit,
194 "maxSeconds": self.config.time_limit,
195 "elapsedSeconds": round(time.monotonic() - started, 3),
196 }
197 )
198 return self._report(execution, change_set, record, classification, findings)
200 def _skipped(
201 self, execution: Execution, change_set: ChangeSet, record: dict[str, Any], reason: str
202 ) -> ValidatorOutput:
203 """The record of a mutation run that did not start, and its NOT_APPLICABLE result."""
204 results = self.ladder.engine.results
205 record["skipped"] = reason
206 ref = results.record_json(
207 execution, PhaseId.VERIFICATION, record, kind="mutation-report", summary=reason
208 )
209 result = results.record_validation(
210 execution,
211 validator_id=MUTATION_ID,
212 digest=change_set.digest,
213 status=ResultStatus.NOT_APPLICABLE,
214 kind=ValidationKind.SUCCESS,
215 mandatory=False,
216 summary=reason,
217 evidence_refs=(ref,),
218 )
219 return ValidatorOutput(result, ())
221 def _classify(
222 self,
223 execution: Execution,
224 template: tuple[str, ...],
225 tests: list[str],
226 sources: list[str],
227 contents: Contents,
228 ) -> list[dict[str, Any]]:
229 """Each new or changed test file on the baseline sources and on the change."""
230 hub = self.ladder
231 workspace = hub.s.paths.workspace
232 classification: list[dict[str, Any]] = []
233 with materialized(workspace, hub.s.paths.harness_dir / "tmp", contents, sources) as before:
234 for path in tests[:MAX_CLASSIFIED_FILES]:
235 status_before = self._run(execution, template, [path], before) if before else None
236 classification.append({"path": path, "before": status_before})
237 current = self._run(execution, template, tests[:MAX_CLASSIFIED_FILES], workspace)
238 for item in classification:
239 before_status = item["before"]
240 item["class"] = _test_class(before_status, current)
241 item["before"] = before_status.value if before_status else None
242 item["after"] = current.value if current else None
243 return classification
245 def _test_finding(
246 self, execution: Execution, item: dict[str, Any], severity: FindingSeverity
247 ) -> Finding:
248 weak = item["class"] == "weak"
249 if weak: 249 ↛ 253line 249 didn't jump to line 253 because the condition on line 249 was always true
250 message = f"{item['path']} passes on the baseline too: it does not show the change"
251 recommendation = "Add an assertion that fails without the change."
252 else:
253 message = f"{item['path']} fails on the change"
254 recommendation = "Fix the test or the change; the suite may not collect it."
255 return self.ladder.engine.results.record_finding(
256 execution,
257 validator_id=MUTATION_ID,
258 rule_id=WEAK_RULE if weak else BROKEN_RULE,
259 category="test-quality",
260 severity=FindingSeverity.LOW if weak else severity,
261 message=message,
262 path=item["path"],
263 recommendation=recommendation,
264 )
266 def _candidates(self, sources: list[str], contents: Contents) -> list[tuple[Hunk, str, str]]:
267 """The hunks of the changed sources that are not blank or comment-only, in order."""
268 workspace = self.ladder.s.paths.workspace
269 candidates: list[tuple[Hunk, str, str]] = []
270 for path in sources:
271 texts = _texts(contents, path, workspace / path)
272 if texts is None: 272 ↛ 273line 272 didn't jump to line 273 because the condition on line 272 was never true
273 continue
274 before_text, after_text = texts
275 candidates.extend(
276 (hunk, before_text, after_text)
277 for hunk in hunks(path, before_text, after_text)
278 if not cosmetic(hunk, before_text, after_text)
279 )
280 return candidates
282 def _mutate(
283 self,
284 execution: Execution,
285 candidates: list[tuple[Hunk, str, str]],
286 command: tuple[tuple[str, ...] | None, list[str]],
287 *,
288 deadline: float,
289 ) -> tuple[list[tuple[Hunk, dict[str, Any]]], list[str]]:
290 """Revert each hunk alone in a scratch copy and run the relevant tests (the changed test
291 files, else the mandatory test command), within ``maxHunks`` and ``maxSeconds``."""
292 hub = self.ladder
293 workspace = hub.s.paths.workspace
294 scratch = hub.s.paths.harness_dir / "tmp"
295 template, paths = command
296 mutated: list[tuple[Hunk, dict[str, Any]]] = []
297 skipped: list[str] = []
298 for hunk, before_text, after_text in candidates:
299 if len(mutated) >= self.config.hunk_limit or time.monotonic() >= deadline: 299 ↛ 300line 299 didn't jump to line 300 because the condition on line 299 was never true
300 skipped.append(hunk.label)
301 continue
302 reverted = revert(hunk, before_text, after_text).encode("utf-8")
303 with materialized(workspace, scratch, _single_file(reverted), [hunk.path]) as copy:
304 status = None
305 if copy is not None and template: 305 ↛ 307line 305 didn't jump to line 307
306 status = self._run(execution, template, paths, copy)
307 item = {
308 **hunk.as_dict(),
309 "status": status.value if status else None,
310 "exercised": status is ResultStatus.FAILED,
311 }
312 mutated.append((hunk, item))
313 return mutated, skipped
315 def _not_exercised(
316 self, execution: Execution, hunk: Hunk, severity: FindingSeverity
317 ) -> Finding:
318 return self.ladder.engine.results.record_finding(
319 execution,
320 validator_id=MUTATION_ID,
321 rule_id=NOT_EXERCISED_RULE,
322 category="test-quality",
323 severity=severity,
324 message=(
325 f"Reverting the change at {hunk.path}:{hunk.after_start + 1} "
326 "leaves every relevant test passing: no test exercises it"
327 ),
328 path=hunk.path,
329 line=hunk.after_start + 1,
330 recommendation="Add a test that fails without this change.",
331 )
333 def _report(
334 self,
335 execution: Execution,
336 change_set: ChangeSet,
337 record: dict[str, Any],
338 classification: list[dict[str, Any]],
339 findings: list[Finding],
340 ) -> ValidatorOutput:
341 results = self.ladder.engine.results
342 mutated = record["hunks"]
343 skipped_hunks = record["hunksLeftOut"]
344 not_exercised = [item for item in mutated if item["status"] == ResultStatus.PASSED.value]
345 discriminating = sum(1 for item in classification if item.get("class") == "discriminating")
346 summary = (
347 f"{len(mutated)} hunk(s) reverted, {len(not_exercised)} not exercised; "
348 f"{discriminating} of {len(classification)} new test file(s) discriminating"
349 )
350 if skipped_hunks: 350 ↛ 351line 350 didn't jump to line 351 because the condition on line 350 was never true
351 summary += f"; {len(skipped_hunks)} hunk(s) left out by the limits"
352 ref = results.record_json(
353 execution, PhaseId.VERIFICATION, record, kind="mutation-report", summary=summary
354 )
355 enforce = self.config.mode == "enforce"
356 blocking = enforce and any(item.severity is FindingSeverity.HIGH for item in findings)
357 result = results.record_validation(
358 execution,
359 validator_id=MUTATION_ID,
360 digest=change_set.digest,
361 status=ResultStatus.FAILED if blocking else ResultStatus.PASSED,
362 kind=ValidationKind.VALIDATION_FAILURE if blocking else ValidationKind.SUCCESS,
363 mandatory=enforce,
364 summary=summary,
365 findings=tuple(findings),
366 evidence_refs=(ref,),
367 )
368 return ValidatorOutput(result, tuple(findings))
370 def _run(
371 self,
372 execution: Execution,
373 template: tuple[str, ...],
374 paths: list[str],
375 root: Path,
376 ) -> ResultStatus | None:
377 argv: list[str] = []
378 for item in template:
379 if item == "{paths}":
380 argv.extend(paths)
381 else:
382 argv.append(item)
383 hub = self.ladder
384 engine = hub.engine
385 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{MUTATION_ID}", version="1")
386 try:
387 result = SafeProcessRunner(root).run(
388 CommandSpec(
389 argv=tuple(argv),
390 cwd=root,
391 timeout_seconds=float(engine._bounded_timeout(600)),
392 max_output_bytes=hub.project.runtime.max_output_bytes,
393 ),
394 actor=actor,
395 grants=grants_from_rules(
396 execution.execution_id, actor, hub.s.resolved.effective_capabilities
397 ),
398 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)),
399 )
400 except Exception: # noqa: BLE001 # a command that cannot start gives no evidence
401 return None
402 if result.status in {ResultStatus.PASSED, ResultStatus.FAILED}: 402 ↛ 404line 402 didn't jump to line 404 because the condition on line 402 was always true
403 return result.status
404 return None
407__all__ = [
408 "BROKEN_RULE",
409 "MUTATION_ID",
410 "NOT_EXERCISED_RULE",
411 "WEAK_RULE",
412 "Mutation",
413 "looks_like_test",
414]