Coverage for src/governed_harness/orchestration/differential.py: 85%
234 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Differential verification against the baseline (#7, #52) and the Ruff/Mypy ratchet.
3Before 2.0 a mandatory validator that failed blocked the gate whether or not the change caused
4the failure: Mypy failed on every governed brownfield run because of an error the repository
5already had, and a step that inherited a red test from an earlier step was blamed for it.
6Under ``verification.differential`` the harness runs each failing mandatory command validator
7on the baseline (a temporary copy of the workspace with every change since DISCOVERY reverted)
8and classifies the failure:
10* the baseline passes: the failure is introduced (``INTRODUCED_ERROR``) and blocks;
11* the baseline fails with the same problems: the failure is pre-existing
12 (``PREEXISTING_ERROR``): the validator's latest result for the ChangeSet passes, with a LOW
13 finding that says what already failed;
14* the baseline fails with fewer problems: only the new problems are findings and block;
15* the outputs cannot be compared (no problem could be parsed, the baseline could not be
16 built or the validator did not run on it): the failure stays as it was (fail closed).
18A test run that produced no test outcome (pytest interrupted at collection, exit codes 2 to 5,
19or another runner whose output shows no failed test) is never comparable (#190): its problems
20hide every other result. A pre-existing test failure passes only when every test file the
21ChangeSet added or modified ran (``tests.changed-not-run``, HIGH; INFO when the output does not
22list the tests that ran).
24Problems are compared by tool rule, path and message (line numbers and other digits removed),
25as the output parsers report them. A baseline run is cached by baseline digest and command.
27Under ``verification.ratchet`` an optional validator (Ruff, Mypy) that fails is compared the
28same way: more problems than on the baseline is a ``ratchet.regressed`` finding; never worse
29than the baseline, without blocking on the debt the repository already had. The problems in a
30frozen acceptance test still as approved are left out (#183): the file is the harness's, approved
31by a person and write-protected, so the agent cannot fix them."""
33from __future__ import annotations
35import json
36import re
37from dataclasses import dataclass
38from datetime import UTC, datetime
39from typing import TYPE_CHECKING, Any
41from governed_harness.capabilities import grants_from_rules
42from governed_harness.domain.enums import (
43 ActorType,
44 FindingSeverity,
45 PhaseId,
46 ResultStatus,
47 ValidationKind,
48)
49from governed_harness.domain.models import (
50 Actor,
51 ChangeSet,
52 Execution,
53 Finding,
54 ValidationResult,
55)
56from governed_harness.evidence import sha256_json
57from governed_harness.orchestration.run_outcome import (
58 RunOutcome,
59 changed_test_files,
60 not_run,
61 read_test_run,
62 runner_of,
63)
64from governed_harness.orchestration.workspace_ops import materialized
65from governed_harness.runtime import CancellationToken, SafeProcessRunner
66from governed_harness.validators import CommandValidator, ValidationContext, ValidatorOutput
67from governed_harness.validators.parsers import parse_output
69if TYPE_CHECKING:
70 from pathlib import Path
72 from governed_harness.orchestration.hosts import ResultsHost
74RATCHET_ID = "harness.ratchet"
75_DIGITS = re.compile(r"\d+")
78@dataclass(frozen=True)
79class RunSummary:
80 status: ResultStatus
81 keys: tuple[str, ...]
82 evidence_refs: tuple[str, ...]
85def issue_keys(stdout: str, stderr: str, workspace: Path) -> tuple[str, ...]:
86 """Comparable identities of the problems a validator reported."""
87 return tuple(
88 sorted(
89 f"{item.rule}|{item.path or ''}|{_DIGITS.sub('#', item.message)}"
90 for item in parse_output(stdout, stderr, workspace)
91 )
92 )
95class Differential:
96 def __init__(self, results: ResultsHost) -> None:
97 self.results = results
99 @property
100 def enabled(self) -> bool:
101 config = self.results.project.verification
102 return bool(config and config.differential)
104 @property
105 def ratchet(self) -> str:
106 config = self.results.project.verification
107 return (config.ratchet if config else None) or "off"
109 def reclassify(
110 self,
111 execution: Execution,
112 change_set: ChangeSet,
113 outputs: list[Any],
114 ) -> list[Any]:
115 if not self.enabled and self.ratchet == "off":
116 return outputs
117 definitions = {
118 item.validator_id: item for item in self.results.s.resolved.effective_validators
119 }
120 failing_mandatory, failing_optional = self._failing(outputs, definitions)
121 if not failing_mandatory and not failing_optional:
122 return outputs
123 baseline = self.results.baseline_snapshot(execution)
124 if baseline is None: 124 ↛ 125line 124 didn't jump to line 125 because the condition on line 124 was never true
125 return outputs
126 validator_ids = [
127 outputs[index].result.validator_id for index in failing_mandatory + failing_optional
128 ]
129 runs = self._baseline_runs(execution, change_set, baseline, validator_ids)
130 updated = list(outputs)
131 workspace = self.results.s.paths.workspace
132 self._reclassify_mandatory(
133 execution, change_set, outputs, failing_mandatory, runs, workspace, updated
134 )
135 if failing_optional:
136 updated.append(
137 self._ratchet_validation(
138 execution, change_set, outputs, failing_optional, runs, workspace
139 )
140 )
141 return updated
143 def _failing(
144 self, outputs: list[Any], definitions: dict[str, Any]
145 ) -> tuple[list[int], list[int]]:
146 """The indices of the failed mandatory validators the differential mode may
147 reclassify, and of the failed optional validators the ratchet compares."""
148 failing_mandatory = [
149 index
150 for index, output in enumerate(outputs)
151 if self.enabled
152 and output.result.mandatory
153 and output.result.status is ResultStatus.FAILED
154 and output.result.validator_id in definitions
155 and output.result.tool_invocation_ids
156 ]
157 failing_optional = [
158 index
159 for index, output in enumerate(outputs)
160 if self.ratchet != "off"
161 and not output.result.mandatory
162 and output.result.status is ResultStatus.FAILED
163 and output.result.validator_id in definitions
164 and output.result.tool_invocation_ids
165 ]
166 return failing_mandatory, failing_optional
168 def _reclassify_mandatory(
169 self,
170 execution: Execution,
171 change_set: ChangeSet,
172 outputs: list[Any],
173 failing_mandatory: list[int],
174 runs: dict[str, RunSummary],
175 workspace: Path,
176 updated: list[Any],
177 ) -> None:
178 """Replace in ``updated`` the failed mandatory outputs the baseline explains."""
179 for index in failing_mandatory:
180 output = outputs[index]
181 run = runs.get(output.result.validator_id)
182 if run is None: 182 ↛ 183line 182 didn't jump to line 183 because the condition on line 182 was never true
183 continue
184 candidate = self._candidate_keys(output.result, workspace)
185 replacement = self._classify(execution, change_set, output, candidate, run)
186 if replacement is not None:
187 updated[index] = replacement
189 def _ratchet_validation(
190 self,
191 execution: Execution,
192 change_set: ChangeSet,
193 outputs: list[Any],
194 failing_optional: list[int],
195 runs: dict[str, RunSummary],
196 workspace: Path,
197 ) -> ValidatorOutput:
198 """The ratchet's validation: the optional validators that report more problems than
199 on the baseline."""
200 regressions: list[Finding] = []
201 refs: list[str] = []
202 for index in failing_optional:
203 output = outputs[index]
204 run = runs.get(output.result.validator_id)
205 if run is None: 205 ↛ 206line 205 didn't jump to line 206 because the condition on line 205 was never true
206 continue
207 candidate = self._candidate_keys(output.result, workspace)
208 finding = self._ratchet(execution, output, candidate, run)
209 refs.extend((*output.result.evidence_refs[:1], *run.evidence_refs[:1]))
210 if finding is not None:
211 regressions.append(finding)
212 enforce = self.ratchet == "enforce"
213 blocking = enforce and bool(regressions)
214 ratchet = self.results.record_validation(
215 execution,
216 validator_id=RATCHET_ID,
217 digest=change_set.digest,
218 status=ResultStatus.FAILED if blocking else ResultStatus.PASSED,
219 kind=ValidationKind.INTRODUCED_ERROR if regressions else ValidationKind.SUCCESS,
220 mandatory=enforce,
221 summary=(
222 f"{len(regressions)} optional validator(s) report more problems than on "
223 "the baseline"
224 if regressions
225 else "No optional validator got worse than on the baseline"
226 ),
227 findings=tuple(regressions),
228 evidence_refs=tuple(dict.fromkeys(refs)) or (change_set.diff_ref,),
229 )
230 return ValidatorOutput(ratchet, tuple(regressions))
232 # ----- baseline runs ---------------------------------------------------------------------
233 def _baseline_runs(
234 self,
235 execution: Execution,
236 change_set: ChangeSet,
237 baseline: Any,
238 validator_ids: list[str],
239 ) -> dict[str, RunSummary]:
240 results = self.results
241 engine = results.engine
242 definitions = {item.validator_id: item for item in results.s.resolved.effective_validators}
243 runs: dict[str, RunSummary] = {}
244 pending: list[str] = []
245 for validator_id in validator_ids:
246 cached = results.flag_json(self._cache_key(baseline.digest, definitions[validator_id]))
247 if isinstance(cached, dict):
248 runs[validator_id] = RunSummary(
249 ResultStatus(cached["status"]),
250 tuple(cached["keys"]),
251 tuple(cached["evidenceRefs"]),
252 )
253 else:
254 pending.append(validator_id)
255 if not pending:
256 return runs
257 workspace = results.s.paths.workspace
258 diff = results.baseline_changes(execution)
259 contents = results.baseline_contents(execution)
260 if diff is None or contents is None: 260 ↛ 261line 260 didn't jump to line 261 because the condition on line 260 was never true
261 return runs
262 paths = [item.path for item in diff.changes]
263 scratch = results.s.paths.harness_dir / "tmp"
264 with materialized(workspace, scratch, contents, paths) as copy:
265 if copy is None: 265 ↛ 266line 265 didn't jump to line 266 because the condition on line 265 was never true
266 results.s.events.append(
267 execution.execution_id,
268 "verification.baseline.unavailable",
269 {"reason": "a changed path has no recorded text in the baseline"},
270 )
271 return runs
272 for validator_id in pending:
273 definition = engine._bounded_definition(definitions[validator_id])
274 actor = Actor(
275 actor_type=ActorType.TOOL,
276 actor_id=f"validator.{validator_id}",
277 version="1",
278 )
279 output = CommandValidator(validator_id).execute(
280 ValidationContext(
281 execution_id=execution.execution_id,
282 workspace=copy,
283 task=engine.run_task(execution),
284 change_set=change_set,
285 definition=definition,
286 grants=grants_from_rules(
287 execution.execution_id, actor, results.s.resolved.effective_capabilities
288 ),
289 artifact_store=results.s.artifacts,
290 process_runner=SafeProcessRunner(copy),
291 provenance=engine._provenance(execution).model_copy(
292 update={"actor": actor}
293 ),
294 cancellation=CancellationToken(
295 lambda: engine.is_cancelled(execution.execution_id)
296 ),
297 max_output_bytes=results.project.runtime.max_output_bytes,
298 )
299 )
300 status = output.result.status
301 keys = self._candidate_keys(output.result, copy)
302 summary = RunSummary(status, keys, output.result.evidence_refs)
303 runs[validator_id] = summary
304 record = {
305 "validatorId": validator_id,
306 "baselineDigest": baseline.digest,
307 "status": status.value,
308 "keys": list(keys),
309 "evidenceRefs": list(output.result.evidence_refs),
310 }
311 ref = results.record_json(
312 execution,
313 PhaseId.VERIFICATION,
314 record,
315 kind="baseline-validation",
316 summary=f"{validator_id} on the baseline: {status.value}",
317 )
318 record["ref"] = ref
319 if status in {ResultStatus.PASSED, ResultStatus.FAILED}: 319 ↛ 323line 319 didn't jump to line 323 because the condition on line 319 was always true
320 results.set_flag_json(
321 self._cache_key(baseline.digest, definitions[validator_id]), record
322 )
323 results.s.events.append(
324 execution.execution_id,
325 "verification.baseline.completed",
326 {
327 "validatorId": validator_id,
328 "status": status.value,
329 "problems": len(keys),
330 "evidenceRef": ref,
331 },
332 )
333 return runs
335 @staticmethod
336 def _cache_key(digest: str, definition: Any) -> str:
337 command = sha256_json(list(definition.command or ()))
338 return f"baselinerun:{digest}:{definition.validator_id}:{command}"
340 def _candidate_keys(self, result: ValidationResult, workspace: Path) -> tuple[str, ...]:
341 report = self._report(result)
342 if report is None: 342 ↛ 343line 342 didn't jump to line 343 because the condition on line 342 was never true
343 return ()
344 _, stdout, stderr = report
345 return issue_keys(stdout, stderr, workspace)
347 def _report(self, result: ValidationResult) -> tuple[dict[str, Any], str, str] | None:
348 """The command report of a validation and its output streams."""
349 artifacts = self.results.s.artifacts
350 try:
351 report = json.loads(artifacts.get(result.evidence_refs[0]))
352 stdout = artifacts.get(report["stdoutRef"]).decode("utf-8", "replace")
353 stderr = artifacts.get(report["stderrRef"]).decode("utf-8", "replace")
354 except (OSError, ValueError, KeyError, IndexError, TypeError):
355 return None
356 if not isinstance(report, dict): 356 ↛ 357line 356 didn't jump to line 357 because the condition on line 356 was never true
357 return None
358 return report, stdout, stderr
360 def _test_run(
361 self, output: ValidatorOutput, candidate: tuple[str, ...]
362 ) -> tuple[str | None, RunOutcome | None]:
363 """The runner and outcome of a failing test validator; ``(None, None)`` for a
364 validator that does not run tests."""
365 definitions = {
366 item.validator_id: item for item in self.results.s.resolved.effective_validators
367 }
368 definition = definitions.get(output.result.validator_id)
369 argv = tuple(definition.command or ()) if definition is not None else ()
370 runner = runner_of(output.result.validator_id, argv)
371 if runner is None: 371 ↛ 372line 371 didn't jump to line 372 because the condition on line 371 was never true
372 return None, None
373 report = self._report(output.result)
374 if report is None: 374 ↛ 375line 374 didn't jump to line 375 because the condition on line 374 was never true
375 return runner, read_test_run(runner, None, "", "", ())
376 data, stdout, stderr = report
377 exit_code = data.get("exitCode")
378 return runner, read_test_run(
379 runner, exit_code if isinstance(exit_code, int) else None, stdout, stderr, candidate
380 )
382 # ----- classification --------------------------------------------------------------------
383 def _classify(
384 self,
385 execution: Execution,
386 change_set: ChangeSet,
387 output: ValidatorOutput,
388 candidate: tuple[str, ...],
389 baseline: RunSummary,
390 ) -> ValidatorOutput | None:
391 results = self.results
392 validator_id = output.result.validator_id
393 refs = tuple(dict.fromkeys((*output.result.evidence_refs, *baseline.evidence_refs)))
394 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{validator_id}", version="1")
395 started = datetime.now(UTC)
396 if baseline.status is ResultStatus.PASSED: 396 ↛ 397line 396 didn't jump to line 397 because the condition on line 396 was never true
397 finding = results.record_finding(
398 execution,
399 validator_id=validator_id,
400 rule_id="differential.introduced",
401 category="validation",
402 severity=FindingSeverity.HIGH,
403 message=f"{validator_id} fails on the ChangeSet and passes on the baseline",
404 evidence_refs=refs,
405 recommendation="The change introduced this failure; fix it before approval.",
406 introduced=True,
407 actor=actor,
408 )
409 result = results.record_validation(
410 execution,
411 validator_id=validator_id,
412 digest=change_set.digest,
413 status=ResultStatus.FAILED,
414 kind=ValidationKind.INTRODUCED_ERROR,
415 mandatory=True,
416 summary=f"{validator_id} failed; the baseline passes (introduced)",
417 findings=(*_original(results, output), finding),
418 evidence_refs=refs,
419 started_at=started,
420 )
421 return ValidatorOutput(result, ())
422 runner, outcome = self._test_run(output, candidate)
423 reason = ""
424 if baseline.status is not ResultStatus.FAILED: 424 ↛ 425line 424 didn't jump to line 425 because the condition on line 424 was never true
425 reason = "the baseline run did not pass or fail"
426 elif not candidate: 426 ↛ 427line 426 didn't jump to line 427 because the condition on line 426 was never true
427 reason = "no problem could be parsed from the output"
428 elif outcome is not None and not outcome.ran:
429 # No test ran (#190): the problems hide every other result, so a regression the
430 # change introduced cannot show up as a new problem.
431 reason = outcome.reason
432 if reason:
433 # Not comparable: keep the failure (fail closed) and say why.
434 results.s.events.append(
435 execution.execution_id,
436 "verification.differential.inconclusive",
437 {
438 "validatorId": validator_id,
439 "baselineStatus": baseline.status.value,
440 "parsedProblems": len(candidate),
441 "reason": reason,
442 },
443 )
444 return None
445 known = set(baseline.keys)
446 introduced = [key for key in candidate if key not in known]
447 preexisting = [key for key in candidate if key in known]
448 note = results.record_finding(
449 execution,
450 validator_id=validator_id,
451 rule_id="differential.preexisting",
452 category="validation",
453 severity=FindingSeverity.LOW,
454 message=(
455 f"{len(preexisting)} problem(s) reported by {validator_id} also fail on the "
456 "baseline (pre-existing): " + "; ".join(_shown(preexisting))
457 ),
458 evidence_refs=refs,
459 recommendation="Pre-existing problems do not block this ChangeSet; track them apart.",
460 introduced=False,
461 actor=actor,
462 )
463 if not introduced:
464 unrun = (
465 self._changed_not_run(
466 execution, change_set, validator_id, runner, outcome, refs, actor
467 )
468 if runner is not None
469 else None
470 )
471 if unrun is not None and unrun.severity is FindingSeverity.HIGH:
472 # A changed test file did not run: the comparison says nothing about it, so the
473 # failure stays (fail closed) with the reason as a blocking finding.
474 result = results.record_validation(
475 execution,
476 validator_id=validator_id,
477 digest=change_set.digest,
478 status=ResultStatus.FAILED,
479 kind=output.result.kind,
480 mandatory=True,
481 summary=(
482 f"{validator_id} failed; a test file the ChangeSet changed did not run"
483 ),
484 findings=(*_original(results, output), note, unrun),
485 evidence_refs=refs,
486 started_at=started,
487 )
488 return ValidatorOutput(result, ())
489 result = results.record_validation(
490 execution,
491 validator_id=validator_id,
492 digest=change_set.digest,
493 status=ResultStatus.PASSED,
494 kind=ValidationKind.PREEXISTING_ERROR,
495 mandatory=True,
496 summary=(
497 f"{validator_id} failed only with problems the baseline already had "
498 f"({len(preexisting)} pre-existing)"
499 ),
500 findings=(note,) if unrun is None else (note, unrun),
501 evidence_refs=refs,
502 started_at=started,
503 )
504 return ValidatorOutput(result, ())
505 new = [
506 results.record_finding(
507 execution,
508 validator_id=validator_id,
509 rule_id="differential.introduced",
510 category="validation",
511 severity=FindingSeverity.HIGH,
512 message=f"Introduced by the change: {_describe(key)}",
513 path=key.split("|")[1] or None,
514 evidence_refs=refs,
515 recommendation="Fix this problem; it does not occur on the baseline.",
516 introduced=True,
517 actor=actor,
518 )
519 for key in introduced[:50]
520 ]
521 result = results.record_validation(
522 execution,
523 validator_id=validator_id,
524 digest=change_set.digest,
525 status=ResultStatus.FAILED,
526 kind=ValidationKind.INTRODUCED_ERROR,
527 mandatory=True,
528 summary=(
529 f"{validator_id}: {len(introduced)} introduced and {len(preexisting)} "
530 "pre-existing problem(s)"
531 ),
532 findings=(*new, note),
533 evidence_refs=refs,
534 started_at=started,
535 )
536 return ValidatorOutput(result, ())
538 def _changed_not_run(
539 self,
540 execution: Execution,
541 change_set: ChangeSet,
542 validator_id: str,
543 runner: str,
544 outcome: RunOutcome | None,
545 refs: tuple[str, ...],
546 actor: Actor,
547 ) -> Finding | None:
548 """``tests.changed-not-run`` when a test file the ChangeSet added or modified has no
549 test that ran (HIGH), or when the output does not say which tests ran (INFO)."""
550 changed = changed_test_files(
551 runner, [(item.path, item.status) for item in change_set.files]
552 )
553 if not changed:
554 return None
555 executed = outcome.executed if outcome is not None else None
556 if executed is None:
557 return self.results.record_finding(
558 execution,
559 validator_id=validator_id,
560 rule_id="tests.changed-not-run",
561 category="validation",
562 severity=FindingSeverity.INFO,
563 message=(
564 "Could not tell from the output whether the tests of the changed test "
565 "file(s) ran: " + ", ".join(changed[:5])
566 ),
567 evidence_refs=refs,
568 recommendation=(
569 "Check the tests of these files ran (pytest prints every test with -rA)."
570 ),
571 introduced=False,
572 actor=actor,
573 )
574 missing = not_run(changed, executed)
575 if not missing: 575 ↛ 576line 575 didn't jump to line 576 because the condition on line 575 was never true
576 return None
577 return self.results.record_finding(
578 execution,
579 validator_id=validator_id,
580 rule_id="tests.changed-not-run",
581 category="validation",
582 severity=FindingSeverity.HIGH,
583 message=(
584 "No test of these test file(s) the ChangeSet added or modified ran: "
585 + ", ".join(missing[:5])
586 ),
587 path=missing[0],
588 evidence_refs=refs,
589 recommendation=(
590 "Make the changed tests run (fix their collection or environment); a failure "
591 "the baseline shares does not stand in for them."
592 ),
593 introduced=True,
594 actor=actor,
595 )
597 def _ratchet(
598 self,
599 execution: Execution,
600 output: ValidatorOutput,
601 candidate: tuple[str, ...],
602 baseline: RunSummary,
603 ) -> Finding | None:
604 validator_id = output.result.validator_id
605 known = set(baseline.keys)
606 # The frozen acceptance tests still as approved are the harness's files, approved by a
607 # person and write-protected (#81, #94): their problems are not the agent's to fix (#183).
608 harness_written = self.results.acceptance.untouched(execution)
609 own = [key for key in candidate if _key_path(key) not in harness_written]
610 if baseline.status is ResultStatus.FAILED: 610 ↛ 612line 610 didn't jump to line 612 because the condition on line 610 was always true
611 worse = [key for key in own if key not in known]
612 elif own:
613 worse = own
614 elif candidate:
615 worse = []
616 else:
617 worse = ["(failed; the baseline passes)"]
618 if baseline.status is ResultStatus.FAILED and not candidate: 618 ↛ 619line 618 didn't jump to line 619 because the condition on line 618 was never true
619 return None
620 if not worse:
621 return None
622 severity = FindingSeverity.HIGH if self.ratchet == "enforce" else FindingSeverity.LOW
623 return self.results.record_finding(
624 execution,
625 validator_id=RATCHET_ID,
626 rule_id="ratchet.regressed",
627 category="validation",
628 severity=severity,
629 message=(
630 f"{validator_id} reports {len(worse)} problem(s) the baseline does not have: "
631 + "; ".join(_shown(worse))
632 ),
633 evidence_refs=tuple(
634 dict.fromkeys((*output.result.evidence_refs, *baseline.evidence_refs))
635 ),
636 recommendation=(
637 f"Fix the new {validator_id} problems; the project must not get worse than "
638 "its baseline."
639 ),
640 introduced=True,
641 )
644def _original(results: ResultsHost, output: ValidatorOutput) -> tuple[Finding, ...]:
645 return tuple(
646 results.s.state.get("finding", item, Finding) for item in output.result.finding_ids
647 )
650def _describe(key: str) -> str:
651 rule, path, message = (key.split("|", 2) + ["", ""])[:3]
652 location = f" at {path}" if path else ""
653 return f"[{rule}]{location}: {message}"
656def _key_path(key: str) -> str:
657 """The workspace-relative path of a problem key, with forward slashes."""
658 return (key.split("|", 2) + [""])[1].replace("\\", "/")
661def _shown(keys: list[str], limit: int = 5) -> list[str]:
662 shown = [_describe(key) for key in keys[:limit]]
663 if len(keys) > limit: 663 ↛ 664line 663 didn't jump to line 664 because the condition on line 663 was never true
664 shown.append(f"and {len(keys) - limit} more")
665 return shown