Coverage for src/governed_harness/ladder/certification.py: 93%

127 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Certification of a ChangeSet against the verification ladder (#55, item 1). 

2 

3For each acceptance criterion: the rung it requires (its declaration, else the project's 

4``defaultLevel``), the rungs its recorded evidence reaches, and a status. Evidence per rung: 

5 

6* L0: every mandatory validator of the gate passed on the digest (VERIFICATION passed); 

7* L1: an L1 validator (the unit test command) passed and a test of the workspace names the 

8 criterion (by its id or by a name its declaration lists), or a frozen acceptance test that 

9 names it passed; 

10* L2: the same, for a test under an integration directory with an L2 validator, or an L2 probe; 

11* L3: a probe linked to the criterion ran on every variant and its assertions held; 

12* L4: an L4 probe passed, or a deferred verification was closed with passing evidence; 

13* L5: a person ticked the criterion's manual item in a decision on this digest. 

14 

15Nothing is credited by omission: a check that did not run, was skipped, was waived or is 

16unavailable adds no rung. The status of a criterion is ``CERTIFIED`` (the required rung is 

17reached), ``PENDING`` (it can only be reached by deferred evidence or a manual check that is 

18not recorded yet), ``WAIVED`` (a person decided in preflight to continue without it) or 

19``NOT_CERTIFIED``. The run is ``CERTIFIED`` when every criterion is, ``PARTIAL`` when some 

20criterion is certified or pending, ``NOT_CERTIFIED`` otherwise.""" 

21 

22from __future__ import annotations 

23 

24from collections.abc import Iterable, Mapping 

25from dataclasses import dataclass, field 

26from typing import Any 

27 

28from governed_harness.domain.enums import VerificationLevel 

29from governed_harness.domain.models import ( 

30 AcceptanceCriterion, 

31 CertificationStatus, 

32 CriterionCertification, 

33 DeferredVerification, 

34 LevelEvidence, 

35) 

36 

37INTEGRATION_DIRECTORIES = frozenset({"integration", "integration_tests", "integration-tests", "it"}) 

38 

39 

40@dataclass(frozen=True) 

41class NamedTest: 

42 node_id: str 

43 path: str 

44 

45 @property 

46 def integration(self) -> bool: 

47 return any(part in INTEGRATION_DIRECTORIES for part in self.path.split("/")[:-1]) 

48 

49 

50@dataclass(frozen=True) 

51class ProbeOutcome: 

52 probe_id: str 

53 level: VerificationLevel 

54 criteria: tuple[str, ...] 

55 passed: bool 

56 readiness: str 

57 ref: str | None = None 

58 

59 

60@dataclass 

61class CertificationInputs: 

62 default_level: VerificationLevel 

63 verification_passed: bool 

64 digest: str 

65 passed_validators: set[str] = field(default_factory=set) 

66 """Validators whose latest result on the digest passed.""" 

67 level_validators: Mapping[VerificationLevel, set[str]] = field(default_factory=dict) 

68 """Validators that establish a rung, from the available capabilities of the profiles.""" 

69 tests: Mapping[str, list[NamedTest]] = field(default_factory=dict) 

70 """Tests of the workspace that name each criterion.""" 

71 acceptance_passed: bool = False 

72 acceptance_tests: Mapping[str, list[NamedTest]] = field(default_factory=dict) 

73 probes: list[ProbeOutcome] = field(default_factory=list) 

74 deferred: list[DeferredVerification] = field(default_factory=list) 

75 checked: set[str] = field(default_factory=set) 

76 """Manual items a person ticked in a decision on this digest.""" 

77 decided: bool = False 

78 """A person already decided on this digest (a manual item not ticked then stays unticked).""" 

79 waived: set[str] = field(default_factory=set) 

80 validation_refs: Mapping[str, str] = field(default_factory=dict) 

81 

82 

83def required_level(criterion: AcceptanceCriterion, default: VerificationLevel) -> VerificationLevel: 

84 return criterion.verification.level if criterion.verification else default 

85 

86 

87def _evidence(criterion: AcceptanceCriterion, inputs: CertificationInputs) -> list[LevelEvidence]: 

88 return [ 

89 *_test_evidence(criterion.criterion_id, inputs), 

90 *_probe_evidence(criterion, inputs), 

91 *_deferred_evidence(criterion.criterion_id, inputs), 

92 *_manual_evidence(criterion, inputs), 

93 ] 

94 

95 

96def _test_evidence(cid: str, inputs: CertificationInputs) -> list[LevelEvidence]: 

97 """L0 from the verification, L1 from the tests and acceptance tests that name the 

98 criterion, L2 from its integration tests.""" 

99 found: list[LevelEvidence] = [] 

100 if inputs.verification_passed: 

101 found.append( 

102 LevelEvidence( 

103 level=VerificationLevel.L0, 

104 source="verification", 

105 detail=f"every mandatory validator passed on {inputs.digest}", 

106 ) 

107 ) 

108 l1 = set(inputs.level_validators.get(VerificationLevel.L1, set())) 

109 l2 = set(inputs.level_validators.get(VerificationLevel.L2, set())) 

110 l1_passed = sorted(l1 & inputs.passed_validators) 

111 l2_passed = sorted(l2 & inputs.passed_validators) 

112 named = inputs.tests.get(cid, []) 

113 if inputs.verification_passed and l1_passed and named: 

114 found.append( 

115 LevelEvidence( 

116 level=VerificationLevel.L1, 

117 source=", ".join(l1_passed), 

118 detail="passing test(s) that name the criterion: " 

119 + ", ".join(item.node_id for item in named[:5]), 

120 refs=tuple( 

121 inputs.validation_refs[item] 

122 for item in l1_passed 

123 if item in inputs.validation_refs 

124 ), 

125 ) 

126 ) 

127 accepted = inputs.acceptance_tests.get(cid, []) 

128 if inputs.acceptance_passed and accepted: 128 ↛ 129line 128 didn't jump to line 129 because the condition on line 128 was never true

129 found.append( 

130 LevelEvidence( 

131 level=VerificationLevel.L1, 

132 source="harness.acceptance-tests", 

133 detail="frozen acceptance test(s) that name the criterion passed: " 

134 + ", ".join(item.node_id for item in accepted[:5]), 

135 ) 

136 ) 

137 integration = [item for item in named if item.integration] 

138 if inputs.verification_passed and l2_passed and integration: 138 ↛ 139line 138 didn't jump to line 139 because the condition on line 138 was never true

139 found.append( 

140 LevelEvidence( 

141 level=VerificationLevel.L2, 

142 source=", ".join(l2_passed), 

143 detail="passing integration test(s) that name the criterion: " 

144 + ", ".join(item.node_id for item in integration[:5]), 

145 ) 

146 ) 

147 return found 

148 

149 

150def _probe_evidence( 

151 criterion: AcceptanceCriterion, inputs: CertificationInputs 

152) -> list[LevelEvidence]: 

153 """The level of every ready probe of the criterion that passed.""" 

154 found: list[LevelEvidence] = [] 

155 cid = criterion.criterion_id 

156 declared_probe = criterion.verification.probe if criterion.verification else None 

157 for probe in inputs.probes: 

158 if probe.probe_id != declared_probe and cid not in probe.criteria: 

159 continue 

160 if probe.passed and probe.readiness == "READY": 

161 found.append( 

162 LevelEvidence( 

163 level=probe.level, 

164 source=f"probe.{probe.probe_id}", 

165 detail="the probe ran on every variant and its assertions held", 

166 refs=(probe.ref,) if probe.ref else (), 

167 ) 

168 ) 

169 return found 

170 

171 

172def _deferred_evidence(cid: str, inputs: CertificationInputs) -> list[LevelEvidence]: 

173 return [ 

174 LevelEvidence( 

175 level=item.level, 

176 source=f"deferred.{item.item_id}", 

177 detail=item.summary or f"deferred verification closed ({item.where})", 

178 refs=(item.evidence_ref,) if item.evidence_ref else (), 

179 ) 

180 for item in inputs.deferred 

181 if item.criterion_id == cid and item.status == "PASSED" 

182 ] 

183 

184 

185def _manual_evidence( 

186 criterion: AcceptanceCriterion, inputs: CertificationInputs 

187) -> list[LevelEvidence]: 

188 verification = criterion.verification 

189 if verification and verification.manual and criterion.criterion_id in inputs.checked: 

190 return [ 

191 LevelEvidence( 

192 level=VerificationLevel.L5, 

193 source="decision", 

194 detail=f"a person checked: {verification.manual}", 

195 ) 

196 ] 

197 return [] 

198 

199 

200def _pending( 

201 criterion: AcceptanceCriterion, required: VerificationLevel, inputs: CertificationInputs 

202) -> list[str]: 

203 declaration = criterion.verification 

204 if declaration is None: 

205 return [] 

206 pending: list[str] = [] 

207 for item in inputs.deferred: 

208 if ( 

209 item.criterion_id == criterion.criterion_id 

210 and item.status == "PENDING" 

211 and item.level.rank >= required.rank 

212 ): 

213 pending.append(item.item_id) 

214 if ( 

215 declaration.manual 

216 and required is VerificationLevel.L5 

217 and criterion.criterion_id not in inputs.checked 

218 and not inputs.decided 

219 ): 

220 pending.append(criterion.criterion_id) 

221 return pending 

222 

223 

224def certify_criterion( 

225 criterion: AcceptanceCriterion, inputs: CertificationInputs 

226) -> CriterionCertification: 

227 required = required_level(criterion, inputs.default_level) 

228 evidence = _evidence(criterion, inputs) 

229 achieved = max((item.level for item in evidence), key=lambda level: level.rank, default=None) 

230 declared = criterion.verification is not None 

231 pending = _pending(criterion, required, inputs) 

232 if achieved is not None and achieved.rank >= required.rank: 

233 status, reason = "CERTIFIED", f"reached {achieved.value} (requires {required.value})" 

234 elif criterion.criterion_id in inputs.waived: 

235 status = "WAIVED" 

236 reason = ( 

237 f"requires {required.value}; a person decided to continue without it " 

238 f"(reached {achieved.value if achieved else 'no rung'})" 

239 ) 

240 elif pending: 

241 status = "PENDING" 

242 reason = f"requires {required.value}; waiting for {', '.join(pending)}" 

243 else: 

244 status = "NOT_CERTIFIED" 

245 reason = ( 

246 f"requires {required.value}, reached {achieved.value if achieved else 'no rung'}" 

247 + ("" if declared else " (default level, not declared)") 

248 ) 

249 return CriterionCertification( 

250 criterion_id=criterion.criterion_id, 

251 required=required, 

252 declared=declared, 

253 achieved=achieved, 

254 status=status, # type: ignore[arg-type] 

255 reason=reason, 

256 evidence=tuple(evidence), 

257 pending=tuple(pending), 

258 ) 

259 

260 

261def overall(criteria: Iterable[CriterionCertification]) -> CertificationStatus: 

262 items = list(criteria) 

263 if items and all(item.status == "CERTIFIED" for item in items): 

264 return "CERTIFIED" 

265 if any(item.status in {"CERTIFIED", "PENDING"} for item in items): 

266 return "PARTIAL" 

267 return "NOT_CERTIFIED" 

268 

269 

270def certify( 

271 criteria: Iterable[AcceptanceCriterion], inputs: CertificationInputs 

272) -> tuple[CertificationStatus, tuple[CriterionCertification, ...]]: 

273 results = tuple(certify_criterion(item, inputs) for item in criteria) 

274 return overall(results), results 

275 

276 

277def summary(status: str, criteria: Iterable[CriterionCertification]) -> dict[str, Any]: 

278 items = list(criteria) 

279 counts: dict[str, int] = {} 

280 for item in items: 

281 counts[item.status] = counts.get(item.status, 0) + 1 

282 return {"status": status, "criteria": len(items), "byStatus": counts} 

283 

284 

285__all__ = [ 

286 "INTEGRATION_DIRECTORIES", 

287 "CertificationInputs", 

288 "NamedTest", 

289 "ProbeOutcome", 

290 "certify", 

291 "certify_criterion", 

292 "overall", 

293 "required_level", 

294 "summary", 

295]