Coverage for src/governed_harness/orchestration/ladder.py: 93%
496 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""The verification ladder, certification and delivery hygiene of a run (#55), wired into the
2engine's phases.
4``RunEngine`` delegates every step of these settings to this class, as it does for the
5agent-results settings: a project without any of them runs exactly the earlier code path
6(``active`` is false and the engine never calls in). What each phase does:
8* INTENT: localisation of M/L tasks, the operational contract and the interruption budget
9 (``ladder_intake``);
10* DISCOVERY: the environment preflight (``ladder_environment``);
11* PLANNING: the verification plan (the rung each criterion requires, the rungs the profiles can
12 reach here and why) and the preflight on the baseline: probes and frozen acceptance tests run
13 before the change and the run is READY, PARTIAL or UNAVAILABLE; UNAVAILABLE stops the run
14 until a person decides to continue uncertified (``harness verification decide``) or fixes the
15 environment;
16* VERIFICATION: the probes after the change, discriminating evidence and light mutation
17 (``ladder_mutation``) and the certification of the ChangeSet;
18* DECISION: the manual checklist, ticked by the person who decides; the gate's reason codes
19 carry the certification;
20* CLOSURE: deferred verifications are bound to the closure commit, and the change is staged,
21 pushed, proposed as a pull request and commented as the contract authorises
22 (``ladder_delivery``)."""
24from __future__ import annotations
26import fnmatch
27import os
28from dataclasses import dataclass
29from datetime import timedelta
30from pathlib import Path, PurePosixPath
31from typing import TYPE_CHECKING, Any
33from governed_harness.domain.enums import (
34 DecisionKind,
35 EvidenceKind,
36 FindingSeverity,
37 PhaseId,
38 ResultStatus,
39 ValidationKind,
40 VerificationLevel,
41)
42from governed_harness.domain.errors import ConfigurationError, PolicyViolationError
43from governed_harness.domain.ids import new_id
44from governed_harness.domain.models import (
45 AcceptanceCriterion,
46 CertificationRecord,
47 ChangeSet,
48 CriterionVerification,
49 DeferredVerification,
50 Execution,
51 Finding,
52 HumanAttachment,
53 HumanDecision,
54 PhaseExecution,
55 ProbeDefinition,
56 Task,
57 ValidationResult,
58 utc_now,
59)
60from governed_harness.ladder.capabilities import (
61 CapabilityStatus,
62 capability_statuses,
63 load_catalog,
64 profile_verification,
65)
66from governed_harness.ladder.certification import (
67 CertificationInputs,
68 NamedTest,
69 ProbeOutcome,
70 certify,
71)
72from governed_harness.ladder.probes import ProbeEvaluation
73from governed_harness.orchestration.engine_types import PhaseOutcome
74from governed_harness.orchestration.ladder_delivery import LadderDelivery
75from governed_harness.orchestration.ladder_environment import EnvironmentPreflight
76from governed_harness.orchestration.ladder_intake import IntentResult, LadderIntake
77from governed_harness.orchestration.ladder_mutation import Mutation
78from governed_harness.orchestration.ladder_probes import ProbeRun, run_probe
79from governed_harness.orchestration.workspace_ops import Contents, materialized
80from governed_harness.runtime import SafeProcessRunner
81from governed_harness.validators import ValidatorOutput
82from governed_harness.validators.traceability import (
83 TestCorpus,
84 load_test_corpus,
85 tests_naming,
86 word_pattern,
87)
89if TYPE_CHECKING:
90 from governed_harness.configuration.ladder import LadderConfig, ProfileVerification
91 from governed_harness.configuration.models import ProjectConfiguration
92 from governed_harness.orchestration.engine_types import EngineServices
93 from governed_harness.orchestration.hosts import EngineHost
95CERTIFICATION_ID = "harness.certification"
96CONTRACT_ID = "harness.contract"
97SCOPE_RULE = "contract.scope-contradiction"
98PREFLIGHT_DECISION_FLAG = "preflightdecision"
99LEVEL_NOT_REACHED_RULE = "certification.level-not-reached"
100PROBE_FAILED_RULE = "probe.assertion-failed"
101PROBE_UNAVAILABLE_RULE = "probe.unavailable"
102_GENERIC_TEST_SUFFIXES = (
103 ".py",
104 ".js",
105 ".ts",
106 ".go",
107 ".rs",
108 ".java",
109 ".kt",
110 ".swift",
111 ".rb",
112 ".cs",
113 ".mjs",
114 ".cjs",
115)
116_MAX_GENERIC_TEST_BYTES = 2_000_000
117_SKIPPED_TEST_DIRECTORIES = frozenset({"node_modules", "build", "target", "vendor", "dist"})
119_PROBE_KINDS = {
120 ResultStatus.PASSED: ValidationKind.SUCCESS,
121 ResultStatus.FAILED: ValidationKind.VALIDATION_FAILURE,
122 ResultStatus.BLOCKED: ValidationKind.CONFIGURATION_ERROR,
123}
124"""The validation kind of a probe's status."""
127@dataclass(frozen=True)
128class PreflightResult:
129 status: str # READY, PARTIAL, UNAVAILABLE
130 reasons: tuple[str, ...]
131 ref: str
134def _probe_readiness(probe_runs: dict[str, ProbeRun]) -> tuple[list[str], list[str]]:
135 """What the probes on the baseline say about the run: reasons it cannot be certified and
136 reasons it is only partly verifiable."""
137 reasons: list[str] = []
138 partial: list[str] = []
139 for probe_id, run in sorted(probe_runs.items()):
140 if run.evaluation.readiness == "UNAVAILABLE":
141 reasons.append(
142 f"probe {probe_id} cannot run on the baseline: "
143 + "; ".join(run.evaluation.problems[:2])
144 )
145 elif run.evaluation.passed:
146 partial.append(
147 f"probe {probe_id} already passes on the baseline (it does not show the change)"
148 )
149 return reasons, partial
152def _plan_readiness(plan: list[dict[str, Any]]) -> tuple[list[str], list[str]]:
153 """The same for the declared criteria of the verification plan."""
154 reasons: list[str] = []
155 partial: list[str] = []
156 for item in plan:
157 if not item["declared"]:
158 continue
159 if item["route"] == "unreachable":
160 reasons.append(
161 f"criterion {item['criterionId']} requires {item['required']}: {item['why']}"
162 )
163 elif item["route"] in {"deferred", "manual"}:
164 when = "after the run" if item["route"] == "deferred" else "with a person"
165 partial.append(
166 f"criterion {item['criterionId']} reaches {item['required']} only {when}"
167 )
168 return reasons, partial
171def _reachable(
172 capabilities: list[CapabilityStatus],
173 linked: list[ProbeDefinition],
174 probe_runs: dict[str, ProbeRun],
175) -> dict[VerificationLevel, str]:
176 """The rungs a criterion reaches here and what reaches each one."""
177 reachable: dict[VerificationLevel, str] = {VerificationLevel.L0: "the mandatory validators"}
178 for item in capabilities:
179 level = VerificationLevel(item.level)
180 if item.available and not item.probes:
181 reachable.setdefault(level, f"{item.provides} ({item.profile_id})")
182 for probe in linked:
183 run = probe_runs.get(probe.probe_id)
184 if run is None or run.evaluation.readiness == "READY":
185 reachable.setdefault(probe.level, f"probe {probe.probe_id}")
186 return reachable
189def _unreachable_reason(
190 required: VerificationLevel,
191 linked: list[ProbeDefinition],
192 probe_runs: dict[str, ProbeRun],
193 capabilities: list[CapabilityStatus],
194) -> str:
195 """Why no rung at or above ``required`` is reachable here."""
196 broken = [
197 f"probe {probe.probe_id} cannot run here ("
198 + "; ".join(probe_runs[probe.probe_id].evaluation.problems[:2])
199 + ")"
200 for probe in linked
201 if probe.probe_id in probe_runs
202 and probe_runs[probe.probe_id].evaluation.readiness == "UNAVAILABLE"
203 ]
204 if broken:
205 return "; ".join(broken[:2])
206 missing_probe = (
207 required.rank >= VerificationLevel.L3.rank
208 and required is not VerificationLevel.L5
209 and not linked
210 )
211 if missing_probe: 211 ↛ 212line 211 didn't jump to line 212 because the condition on line 211 was never true
212 return "no probe is linked to it"
213 unavailable = [
214 f"{item.level} {item.provides} needs "
215 + ", ".join(d.description for d in item.detections if not d.available)
216 for item in capabilities
217 if not item.available
218 and not item.probes
219 and VerificationLevel(item.level).rank >= required.rank
220 ]
221 return "; ".join(unavailable[:2]) or f"no capability reaches {required.value} here"
224def _recorded_evaluations(
225 validations: list[ValidationResult], definitions: dict[str, ProbeDefinition]
226) -> dict[str, ProbeEvaluation]:
227 """The probes' evaluations as their recorded validations say (a decision after the run)."""
228 evaluations: dict[str, ProbeEvaluation] = {}
229 for item in validations:
230 probe_id = item.validator_id[6:]
231 if item.validator_id.startswith("probe.") and probe_id in definitions:
232 evaluations[probe_id] = ProbeEvaluation(
233 probe_id,
234 "UNAVAILABLE" if item.status is ResultStatus.BLOCKED else "READY",
235 item.status is ResultStatus.PASSED,
236 )
237 return evaluations
240class VerificationLadder:
241 def __init__(self, engine: EngineHost) -> None:
242 self.engine = engine
243 self.intake = LadderIntake(self)
244 self.environment = EnvironmentPreflight(self)
245 self.mutation = Mutation(self)
246 self.delivery = LadderDelivery(self)
247 self._capabilities: dict[str, list[CapabilityStatus]] = {}
248 self._catalog: dict[str, ProfileVerification] | None = None
250 # ----- configuration ------------------------------------------------------------------------
251 @property
252 def s(self) -> EngineServices:
253 return self.engine.s
255 @property
256 def project(self) -> ProjectConfiguration:
257 return self.s.resolved.project
259 @property
260 def config(self) -> LadderConfig | None:
261 verification = self.project.verification
262 ladder = verification.ladder if verification else None
263 return ladder if ladder is not None and ladder.enabled else None
265 def enabled_config(self) -> LadderConfig:
266 """The ladder's configuration, for the steps that run only when it is enabled."""
267 config = self.config
268 if config is None: # not reached: the engine calls these steps only when enabled 268 ↛ 269line 268 didn't jump to line 269 because the condition on line 268 was never true
269 raise RuntimeError("verification.ladder is not enabled")
270 return config
272 @property
273 def manual_checklist(self) -> bool:
274 return bool(self.project.review and self.project.review.manual_checklist)
276 @property
277 def active(self) -> bool:
278 """Whether any setting of #55 that acts on a run is configured; without one the engine
279 runs the earlier path and never calls into this class."""
280 project = self.project
281 intake = project.intake
282 verification = project.verification
283 delivery = project.delivery
284 context = project.context
285 return any(
286 (
287 self.config is not None,
288 bool(verification and verification.probes),
289 self.mutation.enabled,
290 self.manual_checklist,
291 bool(intake and (intake.operational_contract or intake.interruptions)),
292 project.environment is not None,
293 bool(context and context.locate and context.locate.enabled),
294 bool(
295 delivery
296 and (
297 delivery.stage is not None
298 or delivery.push is not None
299 or delivery.pull_request is not None
300 or delivery.comment is not None
301 )
302 ),
303 )
304 )
306 def profile_verifications(self) -> list[tuple[str, ProfileVerification]]:
307 if self._catalog is None:
308 self._catalog = load_catalog()
309 return profile_verification(self.s.resolved.profiles, self._catalog)
311 def capabilities(self, execution: Execution) -> list[CapabilityStatus]:
312 """The profiles' capabilities and their availability here, detected once per run."""
313 key = execution.execution_id
314 if key not in self._capabilities:
315 config = self.project.verification.ladder if self.project.verification else None
316 self._capabilities[key] = capability_statuses(
317 self.profile_verifications(),
318 self.s.paths.workspace,
319 {item.validator_id for item in self.s.resolved.effective_validators},
320 run_detections=bool(config and config.capability_detection),
321 timeout=config.detection_timeout if config else 10,
322 )
323 return self._capabilities[key]
325 def level_validators(self, execution: Execution) -> dict[VerificationLevel, set[str]]:
326 """Validators that establish L1 and L2, from the available capabilities; without a
327 declared capability, every mandatory validator of the profiles establishes L1."""
328 found: dict[VerificationLevel, set[str]] = {}
329 for item in self.capabilities(execution):
330 if item.available and item.validators:
331 level = VerificationLevel(item.level)
332 found.setdefault(level, set()).update(item.validators)
333 if VerificationLevel.L1 not in found: 333 ↛ 334line 333 didn't jump to line 334 because the condition on line 333 was never true
334 found[VerificationLevel.L1] = {
335 item.validator_id for item in self.s.resolved.effective_validators if item.mandatory
336 }
337 return found
339 def probes(self, task: Task) -> list[ProbeDefinition]:
340 """The project's probes and the task's (a task probe replaces a project probe with the
341 same id)."""
342 verification = self.project.verification
343 merged: dict[str, ProbeDefinition] = {
344 item.probe_id: item for item in (verification.probes if verification else None) or ()
345 }
346 merged.update({item.probe_id: item for item in task.probes})
347 return list(merged.values())
349 def manual_items(self, task: Task) -> list[dict[str, str]]:
350 """What only a person can verify: the ``manual`` declaration of each criterion and the
351 task's checklist."""
352 if not self.manual_checklist:
353 return []
354 items = [
355 {"itemId": criterion.criterion_id, "text": criterion.verification.manual}
356 for criterion in task.acceptance_criteria
357 if criterion.verification and criterion.verification.manual
358 ]
359 items.extend({"itemId": item.item_id, "text": item.text} for item in task.checklist)
360 return items
362 # ----- INTENT and DISCOVERY -----------------------------------------------------------------
363 def intent(
364 self,
365 execution: Execution,
366 phase: PhaseExecution,
367 task: Task,
368 questions: tuple[Any, ...],
369 ) -> IntentResult:
370 return self.intake.intent(execution, phase, task, questions)
372 def discovery(
373 self, execution: Execution, phase: PhaseExecution, baseline_digest: str
374 ) -> PhaseOutcome | None:
375 return self.environment.run(execution, phase, baseline_digest)
377 # ----- PLANNING: the verification plan and the preflight ------------------------------------
378 def planning(
379 self, execution: Execution, phase: PhaseExecution, task: Task
380 ) -> PhaseOutcome | None:
381 if self.config is None:
382 return None
383 preflight = self.preflight(execution, phase, task)
384 if preflight.status != "UNAVAILABLE":
385 return None
386 decision = self.preflight_decision(execution)
387 if decision is not None:
388 return None
389 if self.config.mode != "enforce":
390 # warn: the person sees it in the brief; nothing waits.
391 self.engine.results.record_finding(
392 execution,
393 validator_id=CERTIFICATION_ID,
394 rule_id="verification.preflight-unavailable",
395 category="verification-ladder",
396 severity=FindingSeverity.LOW,
397 message="Verification preflight UNAVAILABLE: " + "; ".join(preflight.reasons[:5]),
398 evidence_refs=(preflight.ref,),
399 )
400 return None
401 return PhaseOutcome(
402 ResultStatus.BLOCKED,
403 "Verification preflight UNAVAILABLE: "
404 + "; ".join(preflight.reasons[:5])
405 + f". Fix the environment and harness run continue --run {execution.execution_id}, "
406 f"or decide: harness verification decide --run {execution.execution_id} "
407 "--continue-uncertified --rationale '...'",
408 (preflight.ref,),
409 )
411 def preflight_decision(self, execution: Execution) -> dict[str, Any] | None:
412 value = self.engine.results.flag_json(f"{PREFLIGHT_DECISION_FLAG}:{execution.execution_id}")
413 return value if isinstance(value, dict) else None
415 def waived(self, execution: Execution) -> tuple[set[str], set[str]]:
416 """Criteria and probes a person decided to continue without (preflight)."""
417 decision = self.preflight_decision(execution)
418 if not decision:
419 return set(), set()
420 return set(decision.get("criteria") or []), set(decision.get("probes") or [])
422 def preflight(self, execution: Execution, phase: PhaseExecution, task: Task) -> PreflightResult:
423 """The verification plan of every criterion and, under ``preflight``, the probes and the
424 frozen acceptance tests on the baseline, classified READY, PARTIAL or UNAVAILABLE."""
425 config = self.enabled_config()
426 results = self.engine.results
427 capabilities = self.capabilities(execution)
428 probes = self.probes(task)
429 probe_runs: dict[str, ProbeRun] = {}
430 if config.preflight and probes and not self._fast_preflight(execution):
431 probe_runs = self._baseline_probes(execution, probes)
432 reasons, partial = _probe_readiness(probe_runs)
433 acceptance = results.acceptance.state(execution) or {}
434 before = (acceptance.get("failBefore") or {}).get("status")
435 if before in {"ERROR", "TIMED_OUT", "BLOCKED"}: 435 ↛ 436line 435 didn't jump to line 436 because the condition on line 435 was never true
436 reasons.append(f"the frozen acceptance tests cannot run on the baseline ({before})")
437 plan = [
438 self._criterion_plan(criterion, capabilities, probes, probe_runs, config)
439 for criterion in task.acceptance_criteria
440 ]
441 plan_reasons, plan_partial = _plan_readiness(plan)
442 reasons.extend(plan_reasons)
443 partial.extend(plan_partial)
444 if reasons:
445 status = "UNAVAILABLE"
446 elif partial:
447 status = "PARTIAL"
448 else:
449 status = "READY"
450 record = {
451 "status": status,
452 "reasons": reasons,
453 "partial": partial,
454 "criteria": plan,
455 "capabilities": [item.as_dict() for item in capabilities],
456 "probes": {key: value.as_dict() for key, value in sorted(probe_runs.items())},
457 "acceptanceFailBefore": before,
458 "preflight": bool(config.preflight),
459 }
460 ref = results.record_json(
461 execution,
462 PhaseId.PLANNING,
463 record,
464 kind="verification-plan",
465 summary=f"Verification plan and preflight: {status}",
466 evidence_kind=EvidenceKind.PLAN,
467 supports=tuple(item.criterion_id for item in task.acceptance_criteria),
468 )
469 results.set_flag_json(
470 f"preflight:{execution.execution_id}",
471 {"status": status, "reasons": reasons, "ref": ref},
472 )
473 self.s.events.append(
474 execution.execution_id,
475 "verification.preflight.completed",
476 {"status": status, "reasons": reasons, "partial": partial, "evidenceRef": ref},
477 phase_execution_id=phase.phase_execution_id,
478 )
479 return PreflightResult(status, tuple(reasons), ref)
481 def _baseline_probes(
482 self, execution: Execution, probes: list[ProbeDefinition]
483 ) -> dict[str, ProbeRun]:
484 results = self.engine.results
485 contents = results.baseline_contents(execution)
486 diff = results.baseline_changes(execution)
487 paths = [item.path for item in diff.changes] if diff is not None else []
488 recorded = contents or Contents(lambda _path: False, lambda _path: None)
489 runs: dict[str, ProbeRun] = {}
490 with materialized(
491 self.s.paths.workspace, self.s.paths.harness_dir / "tmp", recorded, paths
492 ) as copy:
493 for probe in probes:
494 if copy is None: 494 ↛ 495line 494 didn't jump to line 495 because the condition on line 494 was never true
495 continue
496 runs[probe.probe_id] = run_probe(
497 self.engine, execution, probe, copy, SafeProcessRunner(copy)
498 )
499 return runs
501 def _fast_preflight(self, execution: Execution) -> bool:
502 """Whether the fast lane of #58 leaves the probes on the baseline out (recorded)."""
503 friction = self.engine.friction
504 return friction.active and friction.skips(execution, "preflight", PhaseId.PLANNING)
506 def _criterion_plan(
507 self,
508 criterion: AcceptanceCriterion,
509 capabilities: list[CapabilityStatus],
510 probes: list[ProbeDefinition],
511 probe_runs: dict[str, ProbeRun],
512 config: LadderConfig,
513 ) -> dict[str, Any]:
514 """Which rungs a criterion can reach here, the highest one, and why."""
515 declaration = criterion.verification
516 required = declaration.level if declaration else config.required_default
517 linked = [
518 probe
519 for probe in probes
520 if (declaration and declaration.probe == probe.probe_id)
521 or criterion.criterion_id in probe.criteria
522 ]
523 reachable = _reachable(capabilities, linked, probe_runs)
524 highest = max(reachable, key=lambda level: level.rank)
525 routed = self._route(declaration, required, reachable)
526 if routed is None:
527 reason = _unreachable_reason(required, linked, probe_runs, capabilities)
528 routed = (
529 "unreachable",
530 f"{reason}; highest reachable here is {highest.value} ({reachable[highest]})",
531 )
532 route, why = routed
533 return {
534 "criterionId": criterion.criterion_id,
535 "declared": declaration is not None,
536 "required": required.value,
537 "highestReachable": highest.value,
538 "target": required.value if route != "unreachable" else highest.value,
539 "route": route,
540 "why": why,
541 "probes": [probe.probe_id for probe in linked],
542 }
544 def _route(
545 self,
546 declaration: CriterionVerification | None,
547 required: VerificationLevel,
548 reachable: dict[VerificationLevel, str],
549 ) -> tuple[str, str] | None:
550 """How a criterion reaches its rung: here, with a person or after the run; ``None``
551 when it cannot."""
552 enough = [level for level in reachable if level.rank >= required.rank]
553 if enough:
554 return "local", reachable[min(enough, key=lambda level: level.rank)]
555 if declaration and declaration.manual and required is VerificationLevel.L5:
556 if self.manual_checklist: 556 ↛ 558line 556 didn't jump to line 558 because the condition on line 556 was always true
557 return "manual", f"a person checks: {declaration.manual}"
558 return "unreachable", "manual checks need review.manualChecklist"
559 if declaration and declaration.deferred:
560 return "deferred", f"verified after the run: {declaration.deferred}"
561 return None
563 # ----- VERIFICATION -------------------------------------------------------------------------
564 def verification(
565 self,
566 execution: Execution,
567 change_set: ChangeSet,
568 outputs: list[Any],
569 ) -> list[ValidatorOutput]:
570 task = self.engine.run_task(execution)
571 added: list[ValidatorOutput] = []
572 scope = self._scope_check(execution, change_set, task)
573 if scope is not None:
574 added.append(scope)
575 probe_outputs, evaluations = self._verify_probes(execution, change_set, task)
576 added.extend(probe_outputs)
577 friction = self.engine.friction
578 mutation = (
579 None
580 if friction.active
581 and (
582 friction.skips(execution, "mutation", PhaseId.VERIFICATION)
583 or friction.tests_exempt(execution, change_set, "light mutation") is not None
584 )
585 else self.mutation.run(execution, change_set, [*outputs, *added])
586 )
587 if mutation is not None:
588 added.append(mutation)
589 if self.config is not None:
590 added.extend(
591 self._certification_validation(
592 execution, change_set, task, [*outputs, *added], evaluations
593 )
594 )
595 return added
597 def _scope_check(
598 self, execution: Execution, change_set: ChangeSet, task: Task
599 ) -> ValidatorOutput | None:
600 """A ChangeSet path outside the contract's ``scopePaths`` contradicts the confirmed
601 scope. Under the ``scope-contradiction`` stop condition (``intake.interruptions``) the
602 validation is ``BLOCKED``: the run stops for a person instead of a correction."""
603 contract = task.contract
604 if contract is None or not contract.scope_paths:
605 return None
606 outside = sorted(
607 item.path
608 for item in change_set.files
609 if not any(fnmatch.fnmatchcase(item.path, pattern) for pattern in contract.scope_paths)
610 )
611 results = self.engine.results
612 intake = self.project.intake
613 stops = intake.interruptions.conditions if intake and intake.interruptions else ()
614 findings: list[Finding] = []
615 if outside: 615 ↛ 641line 615 didn't jump to line 641 because the condition on line 615 was always true
616 findings.append(
617 results.record_finding(
618 execution,
619 validator_id=CONTRACT_ID,
620 rule_id=SCOPE_RULE,
621 category="operational-contract",
622 severity=FindingSeverity.HIGH,
623 message=(
624 f"{len(outside)} changed path(s) outside the contract's scope "
625 f"({', '.join(contract.scope_paths)}): {', '.join(outside[:10])}"
626 ),
627 path=outside[0],
628 recommendation="Revert the out-of-scope change or revise the contract.",
629 )
630 )
631 if "scope-contradiction" in stops: 631 ↛ 641line 631 didn't jump to line 641 because the condition on line 631 was always true
632 self.s.events.append(
633 execution.execution_id,
634 "stop.condition",
635 {
636 "condition": "scope-contradiction",
637 "detail": f"changed outside the contract's scope: {', '.join(outside[:10])}",
638 "changeSetDigest": change_set.digest,
639 },
640 )
641 if not outside: 641 ↛ 642line 641 didn't jump to line 642 because the condition on line 641 was never true
642 status = ResultStatus.PASSED
643 elif "scope-contradiction" in stops: 643 ↛ 646line 643 didn't jump to line 646 because the condition on line 643 was always true
644 status = ResultStatus.BLOCKED
645 else:
646 status = ResultStatus.FAILED
647 ref = results.record_json(
648 execution,
649 PhaseId.VERIFICATION,
650 {"scopePaths": list(contract.scope_paths), "outside": outside},
651 kind="contract-scope",
652 summary=f"Contract scope: {len(outside)} path(s) outside",
653 )
654 result = results.record_validation(
655 execution,
656 validator_id=CONTRACT_ID,
657 digest=change_set.digest,
658 status=status,
659 kind=ValidationKind.SUCCESS if not outside else ValidationKind.POLICY_VIOLATION,
660 mandatory=True,
661 summary=f"{len(outside)} changed path(s) outside the contract's scope",
662 findings=tuple(findings),
663 evidence_refs=(ref,),
664 )
665 return ValidatorOutput(result, tuple(findings))
667 def _verify_probes(
668 self, execution: Execution, change_set: ChangeSet, task: Task
669 ) -> tuple[list[ValidatorOutput], dict[str, ProbeEvaluation]]:
670 outputs: list[ValidatorOutput] = []
671 evaluations: dict[str, ProbeEvaluation] = {}
672 _, waived_probes = self.waived(execution)
673 enforce = self.config is not None and self.config.mode == "enforce"
674 for probe in self.probes(task):
675 if probe.probe_id in waived_probes:
676 outputs.append(self._waived_probe(execution, change_set, probe))
677 continue
678 output, evaluation = self._verify_probe(execution, change_set, probe, enforce)
679 evaluations[probe.probe_id] = evaluation
680 outputs.append(output)
681 return outputs, evaluations
683 def _waived_probe(
684 self, execution: Execution, change_set: ChangeSet, probe: ProbeDefinition
685 ) -> ValidatorOutput:
686 """The NOT_APPLICABLE result of a probe a person waived in preflight."""
687 results = self.engine.results
688 ref = results.record_json(
689 execution,
690 PhaseId.VERIFICATION,
691 {"probeId": probe.probe_id, "waived": True},
692 kind="probe-report",
693 summary=f"Probe {probe.probe_id} waived in preflight",
694 )
695 result = results.record_validation(
696 execution,
697 validator_id=f"probe.{probe.probe_id}",
698 digest=change_set.digest,
699 status=ResultStatus.NOT_APPLICABLE,
700 kind=ValidationKind.SUCCESS,
701 mandatory=False,
702 summary="waived: a person decided in preflight to continue without it",
703 evidence_refs=(ref,),
704 )
705 return ValidatorOutput(result, ())
707 def _verify_probe(
708 self, execution: Execution, change_set: ChangeSet, probe: ProbeDefinition, enforce: bool
709 ) -> tuple[ValidatorOutput, ProbeEvaluation]:
710 """Run one probe on the change and record its report, findings and validation."""
711 results = self.engine.results
712 validator_id = f"probe.{probe.probe_id}"
713 run = run_probe(
714 self.engine,
715 execution,
716 probe,
717 self.s.paths.workspace,
718 self.engine._runner(execution),
719 )
720 evaluation = run.evaluation
721 ref = results.record_json(
722 execution,
723 PhaseId.VERIFICATION,
724 run.as_dict(),
725 kind="probe-report",
726 summary=f"Probe {probe.probe_id}: {evaluation.readiness}, "
727 + ("passed" if evaluation.passed else "did not pass"),
728 evidence_kind=EvidenceKind.TEST_REPORT,
729 supports=probe.criteria,
730 )
731 if evaluation.readiness == "UNAVAILABLE": 731 ↛ 732line 731 didn't jump to line 732 because the condition on line 731 was never true
732 status = ResultStatus.BLOCKED
733 findings = [self._unavailable_probe(execution, probe, evaluation, ref)]
734 elif not evaluation.passed:
735 status = ResultStatus.FAILED
736 severity = FindingSeverity.HIGH if enforce else FindingSeverity.LOW
737 findings = [
738 results.record_finding(
739 execution,
740 validator_id=validator_id,
741 rule_id=PROBE_FAILED_RULE,
742 category="verification-ladder",
743 severity=severity,
744 message=(
745 f"Probe {probe.probe_id}"
746 + (f" variant {item.variant}" if item.variant else "")
747 + f", assertion {item.index} ({item.kind}): {item.detail}"
748 ),
749 evidence_refs=(ref,),
750 introduced=True,
751 )
752 for item in evaluation.results
753 if not item.passed
754 ]
755 else:
756 status = ResultStatus.PASSED
757 findings = []
758 outcome = "every assertion held" if evaluation.passed else f"{len(findings)} problem(s)"
759 result = results.record_validation(
760 execution,
761 validator_id=validator_id,
762 digest=change_set.digest,
763 status=status,
764 kind=_PROBE_KINDS[status],
765 mandatory=enforce or status is ResultStatus.BLOCKED,
766 summary=f"{len(evaluation.variants)} variant(s): {outcome}",
767 findings=tuple(findings),
768 evidence_refs=(ref,),
769 )
770 return ValidatorOutput(result, tuple(findings)), evaluation
772 def _unavailable_probe(
773 self, execution: Execution, probe: ProbeDefinition, evaluation: ProbeEvaluation, ref: str
774 ) -> Finding:
775 return self.engine.results.record_finding(
776 execution,
777 validator_id=f"probe.{probe.probe_id}",
778 rule_id=PROBE_UNAVAILABLE_RULE,
779 category="verification-ladder",
780 severity=FindingSeverity.HIGH,
781 message=f"Probe {probe.probe_id} could not run: " + "; ".join(evaluation.problems[:3]),
782 evidence_refs=(ref,),
783 recommendation="Make the probe's command available or waive it in "
784 "preflight (harness verification decide --continue-uncertified).",
785 )
787 # ----- certification --------------------------------------------------------------------------
788 def corpus(self) -> TestCorpus:
789 technologies = [profile.technology for profile in self.s.resolved.profiles]
790 return load_test_corpus(self.s.paths.workspace, technologies)
792 def _generic_tests(self, identifiers: list[str]) -> dict[str, list[NamedTest]]:
793 """For technologies the traceability corpus does not parse (Go, Rust, JVM, Swift),
794 files under a test directory or named after tests that mention the identifier."""
795 found: dict[str, list[NamedTest]] = {}
796 root = self.s.paths.workspace
797 patterns = {identifier: word_pattern(identifier) for identifier in identifiers}
798 for current, directories, files in os.walk(root):
799 directories[:] = sorted(
800 item
801 for item in directories
802 if not item.startswith(".") and item not in _SKIPPED_TEST_DIRECTORIES
803 )
804 for name in sorted(files):
805 path = Path(current) / name
806 relative = PurePosixPath(path.relative_to(root).as_posix())
807 if (
808 path.suffix not in _GENERIC_TEST_SUFFIXES
809 or path.is_symlink()
810 or "test" not in relative.as_posix().lower()
811 or path.stat().st_size > _MAX_GENERIC_TEST_BYTES
812 ):
813 continue
814 text = path.read_text(encoding="utf-8", errors="replace")
815 for identifier, pattern in patterns.items():
816 if pattern.search(text): 816 ↛ 815line 816 didn't jump to line 815 because the condition on line 816 was always true
817 found.setdefault(identifier, []).append(
818 NamedTest(relative.as_posix(), relative.as_posix())
819 )
820 return found
822 def named_tests(self, task: Task) -> dict[str, list[NamedTest]]:
823 corpus = self.corpus()
824 technologies = {profile.technology for profile in self.s.resolved.profiles}
825 names: dict[str, list[str]] = {}
826 for criterion in task.acceptance_criteria:
827 declared = list(criterion.verification.tests) if criterion.verification else []
828 names[criterion.criterion_id] = [criterion.criterion_id, *declared]
829 found: dict[str, list[NamedTest]] = {}
830 generic = (
831 self._generic_tests(sorted({name for values in names.values() for name in values}))
832 if technologies - {"python", "node"}
833 else {}
834 )
835 for criterion_id, identifiers in names.items():
836 tests: list[NamedTest] = []
837 for identifier in identifiers:
838 tests.extend(
839 NamedTest(item.node_id, item.path) for item in tests_naming(identifier, corpus)
840 )
841 tests.extend(generic.get(identifier, []))
842 unique = {item.node_id: item for item in tests}
843 if unique:
844 found[criterion_id] = list(unique.values())
845 return found
847 def _acceptance_named(
848 self, execution: Execution, task: Task
849 ) -> tuple[bool, dict[str, list[NamedTest]]]:
850 results = self.engine.results
851 state = results.acceptance.state(execution) or {}
852 frozen = sorted((state.get("frozen") or {}).keys())
853 if state.get("status") != "APPROVED" or not frozen: 853 ↛ 855line 853 didn't jump to line 855 because the condition on line 853 was always true
854 return False, {}
855 named: dict[str, list[NamedTest]] = {}
856 for criterion in task.acceptance_criteria:
857 pattern = word_pattern(criterion.criterion_id)
858 for path in frozen:
859 target = self.s.paths.workspace / path
860 if target.is_file() and pattern.search(
861 target.read_text(encoding="utf-8", errors="replace")
862 ):
863 named.setdefault(criterion.criterion_id, []).append(NamedTest(path, path))
864 return True, named
866 def certification_inputs(
867 self,
868 execution: Execution,
869 task: Task,
870 digest: str,
871 validations: list[ValidationResult],
872 evaluations: dict[str, ProbeEvaluation] | None = None,
873 checked: set[str] | None = None,
874 decided: bool = False,
875 ) -> CertificationInputs:
876 config = self.enabled_config()
877 own = {CERTIFICATION_ID}
878 mandatory = [
879 item for item in validations if item.mandatory and item.validator_id not in own
880 ]
881 # A pre-existing failure passes as "no worse than the baseline", not as "the tests that
882 # name the criterion passed": it gives no rung credit (#190).
883 passed = {
884 item.validator_id
885 for item in validations
886 if item.status is ResultStatus.PASSED
887 and item.kind is not ValidationKind.PREEXISTING_ERROR
888 }
889 definitions = {probe.probe_id: probe for probe in self.probes(task)}
890 if evaluations is None:
891 evaluations = _recorded_evaluations(validations, definitions)
892 refs = {
893 item.validator_id: item.evidence_refs[0] for item in validations if item.evidence_refs
894 }
895 probes = [
896 ProbeOutcome(
897 probe_id,
898 definitions[probe_id].level,
899 definitions[probe_id].criteria,
900 evaluation.passed,
901 evaluation.readiness,
902 refs.get(f"probe.{probe_id}"),
903 )
904 for probe_id, evaluation in evaluations.items()
905 if probe_id in definitions
906 ]
907 acceptance_ran = any(
908 item.validator_id == "harness.acceptance-tests" and item.status is ResultStatus.PASSED
909 for item in validations
910 )
911 approved, accepted = self._acceptance_named(execution, task)
912 waived_criteria, _ = self.waived(execution)
913 return CertificationInputs(
914 default_level=config.required_default,
915 verification_passed=bool(mandatory)
916 and all(item.status is ResultStatus.PASSED for item in mandatory),
917 digest=digest,
918 passed_validators=passed,
919 level_validators=self.level_validators(execution),
920 tests=self.named_tests(task),
921 acceptance_passed=approved and acceptance_ran,
922 acceptance_tests=accepted,
923 probes=probes,
924 deferred=self.deferred_items(execution, digest),
925 checked=checked or set(),
926 decided=decided,
927 waived=waived_criteria,
928 validation_refs=refs,
929 )
931 def record_certification(
932 self,
933 execution: Execution,
934 task: Task,
935 inputs: CertificationInputs,
936 trigger: str,
937 ) -> CertificationRecord:
938 status, criteria = certify(task.acceptance_criteria, inputs)
939 record = CertificationRecord(
940 certification_id=new_id("certification"),
941 execution_id=execution.execution_id,
942 change_set_digest=inputs.digest,
943 status=status,
944 trigger=trigger, # type: ignore[arg-type]
945 criteria=criteria,
946 )
947 self.s.state.put(
948 "certification",
949 record.certification_id,
950 record,
951 execution_id=execution.execution_id,
952 project_id=execution.project_id,
953 )
954 ref = self.engine.results.record_json(
955 execution,
956 PhaseId.DECISION if trigger != "VERIFICATION" else PhaseId.VERIFICATION,
957 record.model_dump(mode="json", by_alias=True),
958 kind="certification",
959 summary=f"Certification {status} ({trigger.lower()})",
960 evidence_kind=EvidenceKind.GATE,
961 supports=tuple(item.criterion_id for item in criteria),
962 )
963 self.s.events.append(
964 execution.execution_id,
965 "certification.recorded",
966 {
967 "certificationId": record.certification_id,
968 "status": status,
969 "trigger": trigger,
970 "changeSetDigest": inputs.digest,
971 "criteria": {item.criterion_id: item.status for item in criteria},
972 "evidenceRef": ref,
973 },
974 )
975 return record
977 def latest_certification(
978 self, execution_id: str, digest: str | None
979 ) -> CertificationRecord | None:
980 records = [
981 item
982 for item in self.s.state.list(
983 "certification", CertificationRecord, execution_id=execution_id
984 )
985 if digest is None or item.change_set_digest == digest
986 ]
987 return max(records, key=lambda item: item.created_at) if records else None
989 def _certification_validation(
990 self,
991 execution: Execution,
992 change_set: ChangeSet,
993 task: Task,
994 outputs: list[Any],
995 evaluations: dict[str, ProbeEvaluation],
996 ) -> list[ValidatorOutput]:
997 config = self.enabled_config()
998 self.ensure_deferred(execution, task, change_set.digest)
999 validations = [item.result for item in outputs]
1000 inputs = self.certification_inputs(
1001 execution, task, change_set.digest, validations, evaluations
1002 )
1003 record = self.record_certification(execution, task, inputs, "VERIFICATION")
1004 others_failed = any(
1005 item.mandatory and item.status is not ResultStatus.PASSED for item in validations
1006 )
1007 results = self.engine.results
1008 severity = FindingSeverity.HIGH if config.mode == "enforce" else FindingSeverity.LOW
1009 findings: list[Finding] = []
1010 for item in record.criteria:
1011 if not item.declared or item.status != "NOT_CERTIFIED":
1012 continue
1013 findings.append(
1014 results.record_finding(
1015 execution,
1016 validator_id=CERTIFICATION_ID,
1017 rule_id=LEVEL_NOT_REACHED_RULE,
1018 category="verification-ladder",
1019 severity=severity,
1020 message=f"Criterion {item.criterion_id} {item.reason}",
1021 recommendation=(
1022 "Add the evidence the rung needs (a test that names the criterion for "
1023 "L1, a probe for L3, deferred evidence for L4, a manual check for L5), or "
1024 "decide with APPROVE_EXCEPTION."
1025 ),
1026 )
1027 )
1028 if others_failed:
1029 # The verification already fails; the certification is recorded, the gate input
1030 # is the failing validators.
1031 return []
1032 blocking = config.mode == "enforce" and bool(findings)
1033 ref = self.latest_ref(execution)
1034 result = results.record_validation(
1035 execution,
1036 validator_id=CERTIFICATION_ID,
1037 digest=change_set.digest,
1038 status=ResultStatus.FAILED if blocking else ResultStatus.PASSED,
1039 kind=ValidationKind.VALIDATION_FAILURE if blocking else ValidationKind.SUCCESS,
1040 mandatory=config.mode == "enforce",
1041 summary=f"certification {record.status}: "
1042 + ", ".join(f"{item.criterion_id} {item.status}" for item in record.criteria[:10]),
1043 findings=tuple(findings),
1044 evidence_refs=(ref,) if ref else (change_set.diff_ref,),
1045 )
1046 return [ValidatorOutput(result, tuple(findings))]
1048 def latest_ref(self, execution: Execution) -> str | None:
1049 events = [
1050 event
1051 for event in self.s.events.list(execution.execution_id)
1052 if event.event_type == "certification.recorded"
1053 ]
1054 return str(events[-1].payload.get("evidenceRef")) if events else None
1056 def gate_reasons(self, execution: Execution, digest: str) -> tuple[str, ...]:
1057 if self.config is None:
1058 return ()
1059 record = self.latest_certification(execution.execution_id, digest)
1060 return (f"CERTIFICATION_{record.status}",) if record else ()
1062 # ----- deferred verification ----------------------------------------------------------------
1063 def deferred_items(
1064 self, execution: Execution, digest: str | None
1065 ) -> list[DeferredVerification]:
1066 now = utc_now()
1067 items: list[DeferredVerification] = []
1068 for item in self.s.state.list(
1069 "deferred_verification", DeferredVerification, execution_id=execution.execution_id
1070 ):
1071 if digest is not None and item.change_set_digest != digest: 1071 ↛ 1072line 1071 didn't jump to line 1072 because the condition on line 1071 was never true
1072 continue
1073 if item.status == "PENDING" and item.expires_at <= now:
1074 item = item.model_copy(update={"status": "EXPIRED"})
1075 items.append(item)
1076 return items
1078 def ensure_deferred(self, execution: Execution, task: Task, digest: str) -> None:
1079 """One pending item per criterion that declares ``deferred``, bound to the digest."""
1080 config = self.enabled_config()
1081 existing = {
1082 item.criterion_id
1083 for item in self.s.state.list(
1084 "deferred_verification", DeferredVerification, execution_id=execution.execution_id
1085 )
1086 if item.change_set_digest == digest
1087 }
1088 for criterion in task.acceptance_criteria:
1089 declaration = criterion.verification
1090 if (
1091 declaration is None
1092 or not declaration.deferred
1093 or criterion.criterion_id in existing
1094 ):
1095 continue
1096 now = utc_now()
1097 item = DeferredVerification(
1098 deferred_id=new_id("deferred"),
1099 item_id=f"D-{criterion.criterion_id}",
1100 project_id=execution.project_id,
1101 execution_id=execution.execution_id,
1102 task_id=task.task_id,
1103 criterion_id=criterion.criterion_id,
1104 level=declaration.level
1105 if declaration.level.rank >= VerificationLevel.L4.rank
1106 else VerificationLevel.L4,
1107 where=declaration.deferred,
1108 change_set_digest=digest,
1109 created_at=now,
1110 expires_at=now + timedelta(days=config.expiry_days),
1111 )
1112 self.s.state.put(
1113 "deferred_verification",
1114 item.deferred_id,
1115 item,
1116 execution_id=execution.execution_id,
1117 project_id=execution.project_id,
1118 )
1119 self.s.events.append(
1120 execution.execution_id,
1121 "verification.deferred.created",
1122 item.model_dump(mode="json", by_alias=True),
1123 )
1125 def bind_commit(self, execution: Execution, digest: str, commit: str) -> None:
1126 for item in self.deferred_items(execution, digest):
1127 if item.status != "PENDING" or item.commit == commit: 1127 ↛ 1128line 1127 didn't jump to line 1128 because the condition on line 1127 was never true
1128 continue
1129 bound = item.model_copy(update={"commit": commit})
1130 self.s.state.put(
1131 "deferred_verification",
1132 bound.deferred_id,
1133 bound,
1134 execution_id=execution.execution_id,
1135 project_id=execution.project_id,
1136 )
1137 self.s.events.append(
1138 execution.execution_id,
1139 "verification.deferred.bound",
1140 {"deferredId": item.deferred_id, "itemId": item.item_id, "commit": commit},
1141 )
1143 # ----- DECISION: the manual checklist -------------------------------------------------------
1144 def check_decision(
1145 self,
1146 execution: Execution,
1147 decision: DecisionKind,
1148 checked: tuple[str, ...],
1149 ) -> None:
1150 task = self.engine.run_task(execution)
1151 items = {item["itemId"] for item in self.manual_items(task)}
1152 if checked and not self.manual_checklist: 1152 ↛ 1153line 1152 didn't jump to line 1153 because the condition on line 1152 was never true
1153 raise ConfigurationError("--check needs review.manualChecklist: true in project.yaml")
1154 unknown = sorted(set(checked) - items)
1155 if unknown:
1156 raise ConfigurationError(
1157 f"unknown checklist item(s): {', '.join(unknown)}; the run's items are "
1158 + (", ".join(sorted(items)) or "none")
1159 )
1160 if decision is DecisionKind.APPROVE and items - set(checked):
1161 raise PolicyViolationError(
1162 "these checklist items need a person's check before APPROVE: "
1163 + ", ".join(sorted(items - set(checked)))
1164 + " (use --check for each, or APPROVE_EXCEPTION with a rationale)"
1165 )
1167 def after_decision(self, execution: Execution, record: HumanDecision) -> None:
1168 """Recompute the certification with the items the person ticked."""
1169 if self.config is None:
1170 return
1171 task = self.engine.run_task(execution)
1172 validations = self.engine._latest_validations(
1173 execution.execution_id, record.change_set_digest
1174 )
1175 inputs = self.certification_inputs(
1176 execution,
1177 task,
1178 record.change_set_digest,
1179 validations,
1180 checked=set(record.checked_items),
1181 decided=True,
1182 )
1183 self.record_certification(execution, task, inputs, "DECISION")
1185 def after_evidence(self, execution: Execution) -> CertificationRecord | None:
1186 """Recompute the certification once deferred evidence was attached."""
1187 if self.config is None or not execution.change_set_digest: 1187 ↛ 1188line 1187 didn't jump to line 1188 because the condition on line 1187 was never true
1188 return None
1189 task = self.engine.run_task(execution)
1190 digest = execution.change_set_digest
1191 decisions = [
1192 item
1193 for item in self.s.state.list(
1194 "decision", HumanDecision, execution_id=execution.execution_id
1195 )
1196 if item.change_set_digest == digest
1197 ]
1198 latest = max(decisions, key=lambda item: item.decided_at) if decisions else None
1199 validations = self.engine._latest_validations(execution.execution_id, digest)
1200 inputs = self.certification_inputs(
1201 execution,
1202 task,
1203 digest,
1204 validations,
1205 checked=set(latest.checked_items) if latest else set(),
1206 decided=latest is not None,
1207 )
1208 return self.record_certification(execution, task, inputs, "EVIDENCE")
1210 # ----- IMPLEMENTATION: what the agent receives ----------------------------------------------
1211 def implement_extras(self, execution: Execution, task: Task) -> dict[str, Any]:
1212 """``locations`` (the locate call) and ``attachments`` (a person's intake context): only
1213 when there is something to send, so other requests keep their form and size."""
1214 extra: dict[str, Any] = {}
1215 located = self.intake.located(execution)
1216 if located:
1217 extra["locations"] = located["locations"]
1218 attachments = self.intake_attachments(task)
1219 if attachments: 1219 ↛ 1220line 1219 didn't jump to line 1220 because the condition on line 1219 was never true
1220 extra["attachments"] = attachments
1221 return extra
1223 def located_paths(self, execution: Execution) -> list[str]:
1224 located = self.intake.located(execution)
1225 return [item["path"] for item in located["locations"]] if located else []
1227 def intake_attachments(self, task: Task) -> list[dict[str, Any]]:
1228 from governed_harness.intake import task_digest
1230 digest = task_digest(task)
1231 attachments = [
1232 item
1233 for item in self.s.state.list(
1234 "human_attachment", HumanAttachment, project_id=task.project_id
1235 )
1236 if item.task_id == task.task_id
1237 and item.execution_id is None
1238 and item.task_digest == digest
1239 ]
1240 return [
1241 {
1242 "attachmentId": item.attachment_id,
1243 "fileName": item.file_name,
1244 "mediaType": item.media_type,
1245 "digest": item.digest,
1246 "note": item.note,
1247 "path": str(self.s.artifacts.path_for(item.artifact_ref)),
1248 }
1249 for item in attachments
1250 ]
1253__all__ = [
1254 "CERTIFICATION_ID",
1255 "LEVEL_NOT_REACHED_RULE",
1256 "PROBE_FAILED_RULE",
1257 "PROBE_UNAVAILABLE_RULE",
1258 "PreflightResult",
1259 "VerificationLadder",
1260]