Coverage for src/governed_harness/ladder/certification.py: 93%
127 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Certification of a ChangeSet against the verification ladder (#55, item 1).
3For each acceptance criterion: the rung it requires (its declaration, else the project's
4``defaultLevel``), the rungs its recorded evidence reaches, and a status. Evidence per rung:
6* L0: every mandatory validator of the gate passed on the digest (VERIFICATION passed);
7* L1: an L1 validator (the unit test command) passed and a test of the workspace names the
8 criterion (by its id or by a name its declaration lists), or a frozen acceptance test that
9 names it passed;
10* L2: the same, for a test under an integration directory with an L2 validator, or an L2 probe;
11* L3: a probe linked to the criterion ran on every variant and its assertions held;
12* L4: an L4 probe passed, or a deferred verification was closed with passing evidence;
13* L5: a person ticked the criterion's manual item in a decision on this digest.
15Nothing is credited by omission: a check that did not run, was skipped, was waived or is
16unavailable adds no rung. The status of a criterion is ``CERTIFIED`` (the required rung is
17reached), ``PENDING`` (it can only be reached by deferred evidence or a manual check that is
18not recorded yet), ``WAIVED`` (a person decided in preflight to continue without it) or
19``NOT_CERTIFIED``. The run is ``CERTIFIED`` when every criterion is, ``PARTIAL`` when some
20criterion is certified or pending, ``NOT_CERTIFIED`` otherwise."""
22from __future__ import annotations
24from collections.abc import Iterable, Mapping
25from dataclasses import dataclass, field
26from typing import Any
28from governed_harness.domain.enums import VerificationLevel
29from governed_harness.domain.models import (
30 AcceptanceCriterion,
31 CertificationStatus,
32 CriterionCertification,
33 DeferredVerification,
34 LevelEvidence,
35)
37INTEGRATION_DIRECTORIES = frozenset({"integration", "integration_tests", "integration-tests", "it"})
40@dataclass(frozen=True)
41class NamedTest:
42 node_id: str
43 path: str
45 @property
46 def integration(self) -> bool:
47 return any(part in INTEGRATION_DIRECTORIES for part in self.path.split("/")[:-1])
50@dataclass(frozen=True)
51class ProbeOutcome:
52 probe_id: str
53 level: VerificationLevel
54 criteria: tuple[str, ...]
55 passed: bool
56 readiness: str
57 ref: str | None = None
60@dataclass
61class CertificationInputs:
62 default_level: VerificationLevel
63 verification_passed: bool
64 digest: str
65 passed_validators: set[str] = field(default_factory=set)
66 """Validators whose latest result on the digest passed."""
67 level_validators: Mapping[VerificationLevel, set[str]] = field(default_factory=dict)
68 """Validators that establish a rung, from the available capabilities of the profiles."""
69 tests: Mapping[str, list[NamedTest]] = field(default_factory=dict)
70 """Tests of the workspace that name each criterion."""
71 acceptance_passed: bool = False
72 acceptance_tests: Mapping[str, list[NamedTest]] = field(default_factory=dict)
73 probes: list[ProbeOutcome] = field(default_factory=list)
74 deferred: list[DeferredVerification] = field(default_factory=list)
75 checked: set[str] = field(default_factory=set)
76 """Manual items a person ticked in a decision on this digest."""
77 decided: bool = False
78 """A person already decided on this digest (a manual item not ticked then stays unticked)."""
79 waived: set[str] = field(default_factory=set)
80 validation_refs: Mapping[str, str] = field(default_factory=dict)
83def required_level(criterion: AcceptanceCriterion, default: VerificationLevel) -> VerificationLevel:
84 return criterion.verification.level if criterion.verification else default
87def _evidence(criterion: AcceptanceCriterion, inputs: CertificationInputs) -> list[LevelEvidence]:
88 return [
89 *_test_evidence(criterion.criterion_id, inputs),
90 *_probe_evidence(criterion, inputs),
91 *_deferred_evidence(criterion.criterion_id, inputs),
92 *_manual_evidence(criterion, inputs),
93 ]
96def _test_evidence(cid: str, inputs: CertificationInputs) -> list[LevelEvidence]:
97 """L0 from the verification, L1 from the tests and acceptance tests that name the
98 criterion, L2 from its integration tests."""
99 found: list[LevelEvidence] = []
100 if inputs.verification_passed:
101 found.append(
102 LevelEvidence(
103 level=VerificationLevel.L0,
104 source="verification",
105 detail=f"every mandatory validator passed on {inputs.digest}",
106 )
107 )
108 l1 = set(inputs.level_validators.get(VerificationLevel.L1, set()))
109 l2 = set(inputs.level_validators.get(VerificationLevel.L2, set()))
110 l1_passed = sorted(l1 & inputs.passed_validators)
111 l2_passed = sorted(l2 & inputs.passed_validators)
112 named = inputs.tests.get(cid, [])
113 if inputs.verification_passed and l1_passed and named:
114 found.append(
115 LevelEvidence(
116 level=VerificationLevel.L1,
117 source=", ".join(l1_passed),
118 detail="passing test(s) that name the criterion: "
119 + ", ".join(item.node_id for item in named[:5]),
120 refs=tuple(
121 inputs.validation_refs[item]
122 for item in l1_passed
123 if item in inputs.validation_refs
124 ),
125 )
126 )
127 accepted = inputs.acceptance_tests.get(cid, [])
128 if inputs.acceptance_passed and accepted: 128 ↛ 129line 128 didn't jump to line 129 because the condition on line 128 was never true
129 found.append(
130 LevelEvidence(
131 level=VerificationLevel.L1,
132 source="harness.acceptance-tests",
133 detail="frozen acceptance test(s) that name the criterion passed: "
134 + ", ".join(item.node_id for item in accepted[:5]),
135 )
136 )
137 integration = [item for item in named if item.integration]
138 if inputs.verification_passed and l2_passed and integration: 138 ↛ 139line 138 didn't jump to line 139 because the condition on line 138 was never true
139 found.append(
140 LevelEvidence(
141 level=VerificationLevel.L2,
142 source=", ".join(l2_passed),
143 detail="passing integration test(s) that name the criterion: "
144 + ", ".join(item.node_id for item in integration[:5]),
145 )
146 )
147 return found
150def _probe_evidence(
151 criterion: AcceptanceCriterion, inputs: CertificationInputs
152) -> list[LevelEvidence]:
153 """The level of every ready probe of the criterion that passed."""
154 found: list[LevelEvidence] = []
155 cid = criterion.criterion_id
156 declared_probe = criterion.verification.probe if criterion.verification else None
157 for probe in inputs.probes:
158 if probe.probe_id != declared_probe and cid not in probe.criteria:
159 continue
160 if probe.passed and probe.readiness == "READY":
161 found.append(
162 LevelEvidence(
163 level=probe.level,
164 source=f"probe.{probe.probe_id}",
165 detail="the probe ran on every variant and its assertions held",
166 refs=(probe.ref,) if probe.ref else (),
167 )
168 )
169 return found
172def _deferred_evidence(cid: str, inputs: CertificationInputs) -> list[LevelEvidence]:
173 return [
174 LevelEvidence(
175 level=item.level,
176 source=f"deferred.{item.item_id}",
177 detail=item.summary or f"deferred verification closed ({item.where})",
178 refs=(item.evidence_ref,) if item.evidence_ref else (),
179 )
180 for item in inputs.deferred
181 if item.criterion_id == cid and item.status == "PASSED"
182 ]
185def _manual_evidence(
186 criterion: AcceptanceCriterion, inputs: CertificationInputs
187) -> list[LevelEvidence]:
188 verification = criterion.verification
189 if verification and verification.manual and criterion.criterion_id in inputs.checked:
190 return [
191 LevelEvidence(
192 level=VerificationLevel.L5,
193 source="decision",
194 detail=f"a person checked: {verification.manual}",
195 )
196 ]
197 return []
200def _pending(
201 criterion: AcceptanceCriterion, required: VerificationLevel, inputs: CertificationInputs
202) -> list[str]:
203 declaration = criterion.verification
204 if declaration is None:
205 return []
206 pending: list[str] = []
207 for item in inputs.deferred:
208 if (
209 item.criterion_id == criterion.criterion_id
210 and item.status == "PENDING"
211 and item.level.rank >= required.rank
212 ):
213 pending.append(item.item_id)
214 if (
215 declaration.manual
216 and required is VerificationLevel.L5
217 and criterion.criterion_id not in inputs.checked
218 and not inputs.decided
219 ):
220 pending.append(criterion.criterion_id)
221 return pending
224def certify_criterion(
225 criterion: AcceptanceCriterion, inputs: CertificationInputs
226) -> CriterionCertification:
227 required = required_level(criterion, inputs.default_level)
228 evidence = _evidence(criterion, inputs)
229 achieved = max((item.level for item in evidence), key=lambda level: level.rank, default=None)
230 declared = criterion.verification is not None
231 pending = _pending(criterion, required, inputs)
232 if achieved is not None and achieved.rank >= required.rank:
233 status, reason = "CERTIFIED", f"reached {achieved.value} (requires {required.value})"
234 elif criterion.criterion_id in inputs.waived:
235 status = "WAIVED"
236 reason = (
237 f"requires {required.value}; a person decided to continue without it "
238 f"(reached {achieved.value if achieved else 'no rung'})"
239 )
240 elif pending:
241 status = "PENDING"
242 reason = f"requires {required.value}; waiting for {', '.join(pending)}"
243 else:
244 status = "NOT_CERTIFIED"
245 reason = (
246 f"requires {required.value}, reached {achieved.value if achieved else 'no rung'}"
247 + ("" if declared else " (default level, not declared)")
248 )
249 return CriterionCertification(
250 criterion_id=criterion.criterion_id,
251 required=required,
252 declared=declared,
253 achieved=achieved,
254 status=status, # type: ignore[arg-type]
255 reason=reason,
256 evidence=tuple(evidence),
257 pending=tuple(pending),
258 )
261def overall(criteria: Iterable[CriterionCertification]) -> CertificationStatus:
262 items = list(criteria)
263 if items and all(item.status == "CERTIFIED" for item in items):
264 return "CERTIFIED"
265 if any(item.status in {"CERTIFIED", "PENDING"} for item in items):
266 return "PARTIAL"
267 return "NOT_CERTIFIED"
270def certify(
271 criteria: Iterable[AcceptanceCriterion], inputs: CertificationInputs
272) -> tuple[CertificationStatus, tuple[CriterionCertification, ...]]:
273 results = tuple(certify_criterion(item, inputs) for item in criteria)
274 return overall(results), results
277def summary(status: str, criteria: Iterable[CriterionCertification]) -> dict[str, Any]:
278 items = list(criteria)
279 counts: dict[str, int] = {}
280 for item in items:
281 counts[item.status] = counts.get(item.status, 0) + 1
282 return {"status": status, "criteria": len(items), "byStatus": counts}
285__all__ = [
286 "INTEGRATION_DIRECTORIES",
287 "CertificationInputs",
288 "NamedTest",
289 "ProbeOutcome",
290 "certify",
291 "certify_criterion",
292 "overall",
293 "required_level",
294 "summary",
295]