Coverage for src/governed_harness/orchestration/ladder_mutation.py: 81%

184 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Discriminating evidence and light mutation (#55, item 5; ``verification.mutation``). 

2 

3A passing test suite says little when the new tests would pass without the change, or when a 

4changed line is never exercised. After the mandatory validators passed, VERIFICATION: 

5 

61. **classifies the new or changed test files**: each runs on a scratch copy of the workspace in 

7 which every changed source file is put back to its baseline content while the tests keep 

8 their new content. A test file that fails there and passes on the change is 

9 *discriminating*; one that passes before and after is *weak* (``tests.weak``); one that fails 

10 after the change is *broken* (``tests.broken``); 

112. **reverts each changed hunk** of the source, one at a time, in a scratch copy and runs the 

12 relevant tests (the changed test files, else the mandatory test command). A hunk whose 

13 reversion no test notices is a ``tests.change-not-exercised`` finding at its line. 

14 

15The order is deterministic (path, then position), blank and comment-only hunks are not 

16mutated, and the number of hunks (``maxHunks``) and the time (``maxSeconds``) are capped; what 

17the cap left out is recorded. The test command comes from the profile's ``mutationCommand`` 

18(``{paths}`` is replaced by the test files); a profile without one is skipped and the record 

19says so. Under ``enforce`` the findings are HIGH and the validation is mandatory; under 

20``warn`` they are LOW.""" 

21 

22from __future__ import annotations 

23 

24import time 

25from pathlib import Path, PurePosixPath 

26from typing import TYPE_CHECKING, Any 

27 

28from governed_harness.capabilities import grants_from_rules 

29from governed_harness.domain.enums import ( 

30 ActorType, 

31 FindingSeverity, 

32 PhaseId, 

33 ResultStatus, 

34 ValidationKind, 

35) 

36from governed_harness.domain.models import Actor, ChangeSet, Execution, Finding 

37from governed_harness.ladder.hunks import Hunk, cosmetic, hunks, revert 

38from governed_harness.orchestration.workspace_ops import Contents, materialized 

39from governed_harness.runtime import CancellationToken, SafeProcessRunner 

40from governed_harness.runtime.process_runner import CommandSpec 

41from governed_harness.validators import ValidatorOutput 

42from governed_harness.validators.traceability import is_test_file 

43 

44if TYPE_CHECKING: 

45 from governed_harness.orchestration.ladder_host import LadderHost 

46 

47MUTATION_ID = "harness.mutation" 

48NOT_EXERCISED_RULE = "tests.change-not-exercised" 

49WEAK_RULE = "tests.weak" 

50BROKEN_RULE = "tests.broken" 

51MAX_CLASSIFIED_FILES = 10 

52_TEST_SEGMENTS = frozenset({"test", "tests", "__tests__", "spec", "specs"}) 

53 

54 

55def _single_file(data: bytes) -> Contents: 

56 """The recorded content of the one path a materialized copy changes.""" 

57 

58 def content(_path: str) -> bytes | None: 

59 return data 

60 

61 return Contents(lambda _path: True, content) 

62 

63 

64def looks_like_test(path: str, technologies: list[str]) -> bool: 

65 relative = PurePosixPath(path) 

66 if is_test_file(relative, technologies): 

67 return True 

68 lowered = relative.name.lower() 

69 return ( 

70 any(part.lower() in _TEST_SEGMENTS for part in relative.parts[:-1]) 

71 or lowered.startswith("test") 

72 or lowered.endswith(("_test.go", "test.java", "tests.swift", "test.kt")) 

73 ) 

74 

75 

76def _test_class(before: ResultStatus | None, after: ResultStatus | None) -> str: 

77 """``broken`` when the test fails on the change, ``discriminating`` when it failed on the 

78 baseline sources, ``weak`` when it passed there too, else ``unknown``.""" 

79 if after is not ResultStatus.PASSED: 79 ↛ 80line 79 didn't jump to line 80 because the condition on line 79 was never true

80 return "broken" 

81 if before is ResultStatus.FAILED: 

82 return "discriminating" 

83 if before is ResultStatus.PASSED: 83 ↛ 85line 83 didn't jump to line 85 because the condition on line 83 was always true

84 return "weak" 

85 return "unknown" 

86 

87 

88def _texts(contents: Contents, path: str, target: Path) -> tuple[str, str] | None: 

89 """The baseline and current text of a changed source, when both are readable text.""" 

90 data = contents.content(path) if contents.existed(path) else b"" 

91 if data is None or not target.is_file(): 91 ↛ 92line 91 didn't jump to line 92 because the condition on line 91 was never true

92 return None 

93 try: 

94 return data.decode("utf-8"), target.read_text(encoding="utf-8") 

95 except (UnicodeDecodeError, OSError): 

96 return None 

97 

98 

99class Mutation: 

100 def __init__(self, ladder: LadderHost) -> None: 

101 self.ladder = ladder 

102 

103 @property 

104 def config(self) -> Any: 

105 verification = self.ladder.project.verification 

106 return verification.mutation if verification else None 

107 

108 @property 

109 def enabled(self) -> bool: 

110 return bool(self.config and self.config.enabled) 

111 

112 def _command(self) -> tuple[str, ...] | None: 

113 for _profile, verification in self.ladder.profile_verifications(): 113 ↛ 116line 113 didn't jump to line 116 because the loop on line 113 didn't complete

114 if verification.mutation_command: 114 ↛ 113line 114 didn't jump to line 113 because the condition on line 114 was always true

115 return tuple(verification.mutation_command) 

116 return None 

117 

118 def _suite(self) -> tuple[str, ...] | None: 

119 for definition in self.ladder.s.resolved.effective_validators: 

120 if definition.mandatory and definition.command: 

121 return tuple(definition.command) 

122 return None 

123 

124 def run( 

125 self, 

126 execution: Execution, 

127 change_set: ChangeSet, 

128 outputs: list[Any], 

129 ) -> ValidatorOutput | None: 

130 if not self.enabled: 

131 return None 

132 results = self.ladder.engine.results 

133 mandatory_failed = [ 

134 item.result.validator_id 

135 for item in outputs 

136 if item.result.mandatory and item.result.status is not ResultStatus.PASSED 

137 ] 

138 template = self._command() 

139 record: dict[str, Any] = { 

140 "changeSetDigest": change_set.digest, 

141 "mode": self.config.mode, 

142 "command": list(template or ()), 

143 } 

144 if mandatory_failed: 144 ↛ 145line 144 didn't jump to line 145 because the condition on line 144 was never true

145 return self._skipped( 

146 execution, 

147 change_set, 

148 record, 

149 f"not run: mandatory validator(s) did not pass ({', '.join(mandatory_failed)})", 

150 ) 

151 if template is None: 151 ↛ 152line 151 didn't jump to line 152 because the condition on line 151 was never true

152 return self._skipped( 

153 execution, 

154 change_set, 

155 record, 

156 "not run: no selected profile declares a mutationCommand", 

157 ) 

158 technologies = [profile.technology for profile in self.ladder.s.resolved.profiles] 

159 contents = results.baseline_contents(execution) 

160 changed = [item for item in change_set.files if item.status != "DELETED"] 

161 tests = sorted(item.path for item in changed if looks_like_test(item.path, technologies)) 

162 sources = sorted( 

163 item.path for item in changed if not looks_like_test(item.path, technologies) 

164 ) 

165 severity = FindingSeverity.HIGH if self.config.mode == "enforce" else FindingSeverity.LOW 

166 started = time.monotonic() 

167 classification: list[dict[str, Any]] = [] 

168 findings: list[Finding] = [] 

169 if tests and contents is not None: 169 ↛ 176line 169 didn't jump to line 176 because the condition on line 169 was always true

170 classification = self._classify(execution, template, tests, sources, contents) 

171 findings.extend( 

172 self._test_finding(execution, item, severity) 

173 for item in classification 

174 if item["class"] in {"weak", "broken"} 

175 ) 

176 candidates = self._candidates(sources, contents) if contents is not None else [] 

177 mutated, skipped_hunks = self._mutate( 

178 execution, 

179 candidates, 

180 (template, tests) if tests else (self._suite(), []), 

181 deadline=started + self.config.time_limit, 

182 ) 

183 findings.extend( 

184 self._not_exercised(execution, hunk, severity) 

185 for hunk, item in mutated 

186 if item["status"] == ResultStatus.PASSED.value 

187 ) 

188 record.update( 

189 { 

190 "testFiles": classification, 

191 "hunks": [item for _, item in mutated], 

192 "hunksLeftOut": skipped_hunks, 

193 "maxHunks": self.config.hunk_limit, 

194 "maxSeconds": self.config.time_limit, 

195 "elapsedSeconds": round(time.monotonic() - started, 3), 

196 } 

197 ) 

198 return self._report(execution, change_set, record, classification, findings) 

199 

200 def _skipped( 

201 self, execution: Execution, change_set: ChangeSet, record: dict[str, Any], reason: str 

202 ) -> ValidatorOutput: 

203 """The record of a mutation run that did not start, and its NOT_APPLICABLE result.""" 

204 results = self.ladder.engine.results 

205 record["skipped"] = reason 

206 ref = results.record_json( 

207 execution, PhaseId.VERIFICATION, record, kind="mutation-report", summary=reason 

208 ) 

209 result = results.record_validation( 

210 execution, 

211 validator_id=MUTATION_ID, 

212 digest=change_set.digest, 

213 status=ResultStatus.NOT_APPLICABLE, 

214 kind=ValidationKind.SUCCESS, 

215 mandatory=False, 

216 summary=reason, 

217 evidence_refs=(ref,), 

218 ) 

219 return ValidatorOutput(result, ()) 

220 

221 def _classify( 

222 self, 

223 execution: Execution, 

224 template: tuple[str, ...], 

225 tests: list[str], 

226 sources: list[str], 

227 contents: Contents, 

228 ) -> list[dict[str, Any]]: 

229 """Each new or changed test file on the baseline sources and on the change.""" 

230 hub = self.ladder 

231 workspace = hub.s.paths.workspace 

232 classification: list[dict[str, Any]] = [] 

233 with materialized(workspace, hub.s.paths.harness_dir / "tmp", contents, sources) as before: 

234 for path in tests[:MAX_CLASSIFIED_FILES]: 

235 status_before = self._run(execution, template, [path], before) if before else None 

236 classification.append({"path": path, "before": status_before}) 

237 current = self._run(execution, template, tests[:MAX_CLASSIFIED_FILES], workspace) 

238 for item in classification: 

239 before_status = item["before"] 

240 item["class"] = _test_class(before_status, current) 

241 item["before"] = before_status.value if before_status else None 

242 item["after"] = current.value if current else None 

243 return classification 

244 

245 def _test_finding( 

246 self, execution: Execution, item: dict[str, Any], severity: FindingSeverity 

247 ) -> Finding: 

248 weak = item["class"] == "weak" 

249 if weak: 249 ↛ 253line 249 didn't jump to line 253 because the condition on line 249 was always true

250 message = f"{item['path']} passes on the baseline too: it does not show the change" 

251 recommendation = "Add an assertion that fails without the change." 

252 else: 

253 message = f"{item['path']} fails on the change" 

254 recommendation = "Fix the test or the change; the suite may not collect it." 

255 return self.ladder.engine.results.record_finding( 

256 execution, 

257 validator_id=MUTATION_ID, 

258 rule_id=WEAK_RULE if weak else BROKEN_RULE, 

259 category="test-quality", 

260 severity=FindingSeverity.LOW if weak else severity, 

261 message=message, 

262 path=item["path"], 

263 recommendation=recommendation, 

264 ) 

265 

266 def _candidates(self, sources: list[str], contents: Contents) -> list[tuple[Hunk, str, str]]: 

267 """The hunks of the changed sources that are not blank or comment-only, in order.""" 

268 workspace = self.ladder.s.paths.workspace 

269 candidates: list[tuple[Hunk, str, str]] = [] 

270 for path in sources: 

271 texts = _texts(contents, path, workspace / path) 

272 if texts is None: 272 ↛ 273line 272 didn't jump to line 273 because the condition on line 272 was never true

273 continue 

274 before_text, after_text = texts 

275 candidates.extend( 

276 (hunk, before_text, after_text) 

277 for hunk in hunks(path, before_text, after_text) 

278 if not cosmetic(hunk, before_text, after_text) 

279 ) 

280 return candidates 

281 

282 def _mutate( 

283 self, 

284 execution: Execution, 

285 candidates: list[tuple[Hunk, str, str]], 

286 command: tuple[tuple[str, ...] | None, list[str]], 

287 *, 

288 deadline: float, 

289 ) -> tuple[list[tuple[Hunk, dict[str, Any]]], list[str]]: 

290 """Revert each hunk alone in a scratch copy and run the relevant tests (the changed test 

291 files, else the mandatory test command), within ``maxHunks`` and ``maxSeconds``.""" 

292 hub = self.ladder 

293 workspace = hub.s.paths.workspace 

294 scratch = hub.s.paths.harness_dir / "tmp" 

295 template, paths = command 

296 mutated: list[tuple[Hunk, dict[str, Any]]] = [] 

297 skipped: list[str] = [] 

298 for hunk, before_text, after_text in candidates: 

299 if len(mutated) >= self.config.hunk_limit or time.monotonic() >= deadline: 299 ↛ 300line 299 didn't jump to line 300 because the condition on line 299 was never true

300 skipped.append(hunk.label) 

301 continue 

302 reverted = revert(hunk, before_text, after_text).encode("utf-8") 

303 with materialized(workspace, scratch, _single_file(reverted), [hunk.path]) as copy: 

304 status = None 

305 if copy is not None and template: 305 ↛ 307line 305 didn't jump to line 307

306 status = self._run(execution, template, paths, copy) 

307 item = { 

308 **hunk.as_dict(), 

309 "status": status.value if status else None, 

310 "exercised": status is ResultStatus.FAILED, 

311 } 

312 mutated.append((hunk, item)) 

313 return mutated, skipped 

314 

315 def _not_exercised( 

316 self, execution: Execution, hunk: Hunk, severity: FindingSeverity 

317 ) -> Finding: 

318 return self.ladder.engine.results.record_finding( 

319 execution, 

320 validator_id=MUTATION_ID, 

321 rule_id=NOT_EXERCISED_RULE, 

322 category="test-quality", 

323 severity=severity, 

324 message=( 

325 f"Reverting the change at {hunk.path}:{hunk.after_start + 1} " 

326 "leaves every relevant test passing: no test exercises it" 

327 ), 

328 path=hunk.path, 

329 line=hunk.after_start + 1, 

330 recommendation="Add a test that fails without this change.", 

331 ) 

332 

333 def _report( 

334 self, 

335 execution: Execution, 

336 change_set: ChangeSet, 

337 record: dict[str, Any], 

338 classification: list[dict[str, Any]], 

339 findings: list[Finding], 

340 ) -> ValidatorOutput: 

341 results = self.ladder.engine.results 

342 mutated = record["hunks"] 

343 skipped_hunks = record["hunksLeftOut"] 

344 not_exercised = [item for item in mutated if item["status"] == ResultStatus.PASSED.value] 

345 discriminating = sum(1 for item in classification if item.get("class") == "discriminating") 

346 summary = ( 

347 f"{len(mutated)} hunk(s) reverted, {len(not_exercised)} not exercised; " 

348 f"{discriminating} of {len(classification)} new test file(s) discriminating" 

349 ) 

350 if skipped_hunks: 350 ↛ 351line 350 didn't jump to line 351 because the condition on line 350 was never true

351 summary += f"; {len(skipped_hunks)} hunk(s) left out by the limits" 

352 ref = results.record_json( 

353 execution, PhaseId.VERIFICATION, record, kind="mutation-report", summary=summary 

354 ) 

355 enforce = self.config.mode == "enforce" 

356 blocking = enforce and any(item.severity is FindingSeverity.HIGH for item in findings) 

357 result = results.record_validation( 

358 execution, 

359 validator_id=MUTATION_ID, 

360 digest=change_set.digest, 

361 status=ResultStatus.FAILED if blocking else ResultStatus.PASSED, 

362 kind=ValidationKind.VALIDATION_FAILURE if blocking else ValidationKind.SUCCESS, 

363 mandatory=enforce, 

364 summary=summary, 

365 findings=tuple(findings), 

366 evidence_refs=(ref,), 

367 ) 

368 return ValidatorOutput(result, tuple(findings)) 

369 

370 def _run( 

371 self, 

372 execution: Execution, 

373 template: tuple[str, ...], 

374 paths: list[str], 

375 root: Path, 

376 ) -> ResultStatus | None: 

377 argv: list[str] = [] 

378 for item in template: 

379 if item == "{paths}": 

380 argv.extend(paths) 

381 else: 

382 argv.append(item) 

383 hub = self.ladder 

384 engine = hub.engine 

385 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{MUTATION_ID}", version="1") 

386 try: 

387 result = SafeProcessRunner(root).run( 

388 CommandSpec( 

389 argv=tuple(argv), 

390 cwd=root, 

391 timeout_seconds=float(engine._bounded_timeout(600)), 

392 max_output_bytes=hub.project.runtime.max_output_bytes, 

393 ), 

394 actor=actor, 

395 grants=grants_from_rules( 

396 execution.execution_id, actor, hub.s.resolved.effective_capabilities 

397 ), 

398 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)), 

399 ) 

400 except Exception: # noqa: BLE001 # a command that cannot start gives no evidence 

401 return None 

402 if result.status in {ResultStatus.PASSED, ResultStatus.FAILED}: 402 ↛ 404line 402 didn't jump to line 404 because the condition on line 402 was always true

403 return result.status 

404 return None 

405 

406 

407__all__ = [ 

408 "BROKEN_RULE", 

409 "MUTATION_ID", 

410 "NOT_EXERCISED_RULE", 

411 "WEAK_RULE", 

412 "Mutation", 

413 "looks_like_test", 

414]