Coverage for src/governed_harness/orchestration/verification.py: 89%

251 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""``harness verify``: check a run's record against its event chain without aborting. 

2 

3The events are the audit authority; the ``records`` table is a projection that the CLI, the API 

4and the gate read. Before 2.0 nobody compared the two: a forged decider written into the 

5projection was shown by ``status`` while the chain still said otherwise, and a chain whose last 

6events had been deleted was reported as valid. The verifier reports, for one run: 

7 

8* the event chain (sequence, links, digests) and, when ``governance.chainAnchor`` is set, whether 

9 the chain still contains the head recorded outside ``.harness`` (truncation); 

10* every record that has an event of its own (decisions, gates, ChangeSets, validations, 

11 findings, evidence, tool and agent invocations, clarifications), rebuilt from the event and 

12 compared with the stored projection, plus the execution's pointers and the phase results; 

13* every artifact the run references: present and with the digest its record states. 

14 

15The anchor covers the chain up to the anchored head. Under ``governance.unanchoredEvents`` 

16(#149) the events after it are listed as unanchored (a warning, or a failure under ``fail``), and 

17a ``retention.artifacts.pruned`` event excuses a missing artifact only when it is within the 

18anchored part: an event appended after the anchor proves nothing the anchor vouches for. 

19 

20Nothing is repaired: the report says what differs, and the events say what happened. 

21""" 

22 

23from __future__ import annotations 

24 

25from dataclasses import dataclass, field 

26from typing import TYPE_CHECKING, Any 

27 

28from pydantic import BaseModel, ValidationError 

29 

30from governed_harness.domain.errors import NotFoundError 

31from governed_harness.domain.models import ( 

32 AgentInvocation, 

33 Artifact, 

34 ChangeSet, 

35 ClarificationRecord, 

36 ClarificationRequest, 

37 Evidence, 

38 Execution, 

39 Finding, 

40 GateEvaluation, 

41 HumanDecision, 

42 PhaseExecution, 

43 ToolInvocation, 

44 ValidationResult, 

45) 

46from governed_harness.events import AnchorStore, StoredEvent 

47 

48if TYPE_CHECKING: 

49 from governed_harness.orchestration.engine_types import EngineServices 

50 

51PROJECTED_EVENTS: dict[str, tuple[str, type[BaseModel], str]] = { 

52 "human.decision.recorded": ("decision", HumanDecision, "decision_id"), 

53 "gate.evaluated": ("gate", GateEvaluation, "gate_evaluation_id"), 

54 "changeset.created": ("change_set", ChangeSet, "change_set_id"), 

55 "validation.completed": ("validation", ValidationResult, "validation_result_id"), 

56 "finding.recorded": ("finding", Finding, "finding_id"), 

57 "evidence.recorded": ("evidence", Evidence, "evidence_id"), 

58 "tool.invocation.completed": ("tool_invocation", ToolInvocation, "invocation_id"), 

59 "agent.invocation.completed": ("agent_invocation", AgentInvocation, "invocation_id"), 

60 "intent.clarified": ("clarification", ClarificationRecord, "clarification_id"), 

61 "intent.clarification.requested": ( 

62 "clarification_request", 

63 ClarificationRequest, 

64 "request_id", 

65 ), 

66} 

67"""Event type -> (record type, model, id field) of the records projected one-to-one from events.""" 

68 

69PRUNED_EVENT = "retention.artifacts.pruned" 

70VALID_ANCHOR = frozenset({"matched", "absent", "off"}) 

71UNANCHORED = "unanchored" 

72"""The anchor status of a chain with events after its anchored head under 

73``unanchoredEvents: fail``.""" 

74SHOWN_UNANCHORED = 50 

75 

76 

77@dataclass 

78class RunVerification: 

79 execution_id: str 

80 chain: dict[str, Any] 

81 anchor: dict[str, Any] 

82 records_checked: int = 0 

83 record_problems: list[dict[str, Any]] = field(default_factory=list) 

84 artifacts_checked: int = 0 

85 artifact_problems: list[dict[str, Any]] = field(default_factory=list) 

86 warnings: list[str] = field(default_factory=list) 

87 """Since #149, under ``governance.unanchoredEvents: warn``: what verifies but the anchor does 

88 not cover. A report without warnings keeps its earlier form.""" 

89 

90 @property 

91 def valid(self) -> bool: 

92 return ( 

93 bool(self.chain["valid"]) 

94 and self.anchor["status"] in VALID_ANCHOR 

95 and not self.record_problems 

96 and not self.artifact_problems 

97 ) 

98 

99 def as_dict(self) -> dict[str, Any]: 

100 value = { 

101 "executionId": self.execution_id, 

102 "valid": self.valid, 

103 "eventChain": self.chain, 

104 "anchor": self.anchor, 

105 "records": {"checked": self.records_checked, "problems": self.record_problems}, 

106 "artifacts": {"checked": self.artifacts_checked, "problems": self.artifact_problems}, 

107 } 

108 if self.warnings: 

109 value["warnings"] = list(self.warnings) 

110 return value 

111 

112 def summary(self) -> str: 

113 reasons = [] 

114 if not self.chain["valid"]: 

115 reasons.append(f"event chain: {self.chain['error']}") 

116 if self.anchor["status"] not in VALID_ANCHOR: 

117 reasons.append(f"anchor: {self.anchor['status']}") 

118 if self.record_problems: 118 ↛ 120line 118 didn't jump to line 120 because the condition on line 118 was always true

119 reasons.append(f"{len(self.record_problems)} record(s) differ from the events") 

120 if self.artifact_problems: 120 ↛ 121line 120 didn't jump to line 121 because the condition on line 120 was never true

121 reasons.append(f"{len(self.artifact_problems)} artifact problem(s)") 

122 return "; ".join(reasons) or "valid" 

123 

124 

125class RunVerifier: 

126 def __init__(self, services: EngineServices) -> None: 

127 self.s = services 

128 

129 def anchor_store(self) -> AnchorStore: 

130 mode = self.s.resolved.project.governance_settings.chain_anchor or "off" 

131 return AnchorStore(mode, self.s.paths.workspace, self.s.resolved.project.project_id) 

132 

133 def verify(self, execution_id: str) -> RunVerification: 

134 events = self._events(execution_id) 

135 has_record = self._has_execution_record(execution_id) 

136 if not events and not has_record: 

137 raise NotFoundError(f"run not found: {execution_id}") 

138 check = self.s.events.check_chain(execution_id) 

139 report = RunVerification( 

140 execution_id=execution_id, 

141 chain=check.as_dict(), 

142 anchor=self._anchor(execution_id, events), 

143 ) 

144 self._check_unanchored(report, events) 

145 self._check_records(report, events) 

146 self._check_artifacts(report, events) 

147 return report 

148 

149 @property 

150 def unanchored_policy(self) -> str | None: 

151 """``governance.unanchoredEvents`` when an anchor is kept, else ``None`` (1.x).""" 

152 settings = self.s.resolved.project.governance_settings 

153 if not settings.chain_anchor or settings.chain_anchor == "off": 

154 return None 

155 return settings.unanchored_events 

156 

157 def _anchored_through(self, report: RunVerification) -> int | None: 

158 """The last sequence the anchor covers under ``unanchoredEvents`` (``0``: none, the 

159 anchor is absent or does not match); ``None``: no limit (the key is absent).""" 

160 if self.unanchored_policy is None: 

161 return None 

162 if report.anchor["status"] in {"matched", UNANCHORED}: 

163 return int(report.anchor["sequence"]) 

164 return 0 

165 

166 def _check_unanchored(self, report: RunVerification, events: list[StoredEvent]) -> None: 

167 """List the events after the anchored head: a warning, or a failed anchor under 

168 ``fail``.""" 

169 policy = self.unanchored_policy 

170 if policy is None or report.anchor["status"] != "matched": 

171 return 

172 sequence = int(report.anchor["sequence"]) 

173 later = [event for event in events if event.sequence > sequence] 

174 if not later: 

175 return 

176 report.anchor["unanchoredEvents"] = [ 

177 {"sequence": event.sequence, "eventType": event.event_type} 

178 for event in later[:SHOWN_UNANCHORED] 

179 ] 

180 if policy == "fail": 

181 report.anchor["status"] = UNANCHORED 

182 return 

183 report.warnings.append( 

184 f"{len(later)} event(s) after the anchored head (sequence {sequence}) are not " 

185 "covered by the anchor" 

186 ) 

187 

188 def execution_ids(self) -> list[str]: 

189 ids = dict.fromkeys(self.s.events.execution_ids()) 

190 for item in self.s.state.list_dicts( 

191 "execution", project_id=self.s.resolved.project.project_id 

192 ): 

193 ids.setdefault(str(item.get("executionId")), None) 

194 return list(ids) 

195 

196 # ----- parts ------------------------------------------------------------------ 

197 def _events(self, execution_id: str) -> list[StoredEvent]: 

198 try: 

199 return self.s.events.list(execution_id) 

200 except (ValueError, TypeError): 

201 return [] 

202 

203 def _has_execution_record(self, execution_id: str) -> bool: 

204 try: 

205 self.s.state.get_dict("execution", execution_id) 

206 except NotFoundError: 

207 return False 

208 return True 

209 

210 def _anchor(self, execution_id: str, events: list[StoredEvent]) -> dict[str, Any]: 

211 store = self.anchor_store() 

212 if store.mode == "off": 

213 return {"mode": "off", "status": "off"} 

214 anchor = store.read(execution_id) 

215 base: dict[str, Any] = {"mode": store.mode, "location": store.location} 

216 if anchor is None: 

217 return {**base, "status": "absent"} 

218 base.update(anchor.as_dict()) 

219 at = next((event for event in events if event.sequence == anchor.sequence), None) 

220 if at is None: 

221 return {**base, "status": "truncated", "eventCount": len(events)} 

222 if at.event_digest != anchor.event_digest: 222 ↛ 223line 222 didn't jump to line 223 because the condition on line 222 was never true

223 return {**base, "status": "rewritten"} 

224 later = sum(1 for event in events if event.sequence > anchor.sequence) 

225 return {**base, "status": "matched", "eventsAfterAnchor": later} 

226 

227 def _check_records(self, report: RunVerification, events: list[StoredEvent]) -> None: 

228 execution_id = report.execution_id 

229 from_events = self._projected(report, events) 

230 stored = self._stored(report, execution_id) 

231 report.records_checked = len(set(from_events) | set(stored)) 

232 for key in sorted(set(from_events) | set(stored)): 

233 record_type, record_id = key 

234 if key not in stored: 234 ↛ 235line 234 didn't jump to line 235 because the condition on line 234 was never true

235 report.record_problems.append(self._problem(record_type, record_id, "missing")) 

236 elif key not in from_events: 

237 report.record_problems.append( 

238 self._problem(record_type, record_id, "no-event", "the record has no event") 

239 ) 

240 elif stored[key] != from_events[key]: 

241 fields = sorted( 

242 name 

243 for name in set(stored[key]) | set(from_events[key]) 

244 if stored[key].get(name) != from_events[key].get(name) 

245 ) 

246 report.record_problems.append( 

247 self._problem(record_type, record_id, "differs", ", ".join(fields)) 

248 ) 

249 self._check_execution(report, events, from_events) 

250 self._check_phases(report, events) 

251 

252 def _projected( 

253 self, report: RunVerification, events: list[StoredEvent] 

254 ) -> dict[tuple[str, str], dict[str, Any]]: 

255 """The records the events project, by (record type, id); an unreadable event is a 

256 problem of the report.""" 

257 from_events: dict[tuple[str, str], dict[str, Any]] = {} 

258 for event in events: 

259 projected = PROJECTED_EVENTS.get(event.event_type) 

260 if projected is None: 

261 continue 

262 record_type, model, id_field = projected 

263 try: 

264 item = model.model_validate(event.payload) 

265 except ValidationError as error: 

266 report.record_problems.append( 

267 self._problem(record_type, None, "unreadable-event", str(error)[:300]) 

268 ) 

269 continue 

270 key = (record_type, str(getattr(item, id_field))) 

271 from_events[key] = item.model_dump(mode="json") 

272 return from_events 

273 

274 def _stored( 

275 self, report: RunVerification, execution_id: str 

276 ) -> dict[tuple[str, str], dict[str, Any]]: 

277 """The stored records of the run, by (record type, id); an unreadable record is a 

278 problem of the report.""" 

279 stored: dict[tuple[str, str], dict[str, Any]] = {} 

280 for record_type, model, id_field in { 

281 (value[0], value[1], value[2]) for value in PROJECTED_EVENTS.values() 

282 }: 

283 for raw in self.s.state.list_dicts(record_type, execution_id=execution_id): 

284 try: 

285 item = model.model_validate(raw) 

286 except ValidationError as error: 

287 report.record_problems.append( 

288 self._problem(record_type, None, "unreadable-record", str(error)[:300]) 

289 ) 

290 continue 

291 stored[(record_type, str(getattr(item, id_field)))] = item.model_dump(mode="json") 

292 return stored 

293 

294 def _check_execution( 

295 self, 

296 report: RunVerification, 

297 events: list[StoredEvent], 

298 from_events: dict[tuple[str, str], dict[str, Any]], 

299 ) -> None: 

300 try: 

301 execution = self.s.state.get("execution", report.execution_id, Execution) 

302 except NotFoundError: 

303 report.record_problems.append( 

304 self._problem("execution", report.execution_id, "missing") 

305 ) 

306 return 

307 except ValidationError as error: 

308 report.record_problems.append( 

309 self._problem("execution", report.execution_id, "unreadable-record", str(error)) 

310 ) 

311 return 

312 report.records_checked += 1 

313 created = next((event for event in events if event.event_type == "run.created"), None) 

314 mismatches: list[str] = [] 

315 if created is not None: 315 ↛ 325line 315 didn't jump to line 325 because the condition on line 315 was always true

316 expected = { 

317 "taskId": execution.task_id, 

318 "configurationDigest": execution.configuration_digest, 

319 "workflowDigest": execution.workflow_digest, 

320 "policyDigest": execution.policy_digest, 

321 } 

322 mismatches.extend( 

323 name for name, value in expected.items() if created.payload.get(name) != value 

324 ) 

325 if ( 

326 execution.human_decision_id 

327 and ("decision", execution.human_decision_id) not in from_events 

328 ): 

329 mismatches.append("humanDecisionId") 

330 if ( 330 ↛ 334line 330 didn't jump to line 334 because the condition on line 330 was never true

331 execution.gate_evaluation_id 

332 and ("gate", execution.gate_evaluation_id) not in from_events 

333 ): 

334 mismatches.append("gateEvaluationId") 

335 digests = { 

336 str(value.get("digest")) 

337 for (record_type, _), value in from_events.items() 

338 if record_type == "change_set" 

339 } 

340 if execution.change_set_digest and execution.change_set_digest not in digests: 340 ↛ 341line 340 didn't jump to line 341 because the condition on line 340 was never true

341 mismatches.append("changeSetDigest") 

342 if mismatches: 

343 report.record_problems.append( 

344 self._problem("execution", report.execution_id, "differs", ", ".join(mismatches)) 

345 ) 

346 

347 def _check_phases(self, report: RunVerification, events: list[StoredEvent]) -> None: 

348 completed = { 

349 event.phase_execution_id: event.payload 

350 for event in events 

351 if event.event_type == "phase.completed" and event.phase_execution_id 

352 } 

353 started = { 

354 event.phase_execution_id 

355 for event in events 

356 if event.event_type == "phase.started" and event.phase_execution_id 

357 } 

358 for raw in self.s.state.list_dicts("phase", execution_id=report.execution_id): 

359 try: 

360 phase = PhaseExecution.model_validate(raw) 

361 except ValidationError as error: 

362 report.record_problems.append( 

363 self._problem("phase", None, "unreadable-record", str(error)[:300]) 

364 ) 

365 continue 

366 report.records_checked += 1 

367 payload = completed.get(phase.phase_execution_id) 

368 if payload is None: 

369 if phase.phase_execution_id not in started: 

370 report.record_problems.append( 

371 self._problem("phase", phase.phase_execution_id, "no-event") 

372 ) 

373 continue 

374 if payload.get("status") != phase.status.value or payload.get("phaseId") != ( 374 ↛ 377line 374 didn't jump to line 377 because the condition on line 374 was never true

375 phase.phase_id.value 

376 ): 

377 report.record_problems.append( 

378 self._problem("phase", phase.phase_execution_id, "differs", "status, phaseId") 

379 ) 

380 

381 def _check_artifacts(self, report: RunVerification, events: list[StoredEvent]) -> None: 

382 through = self._anchored_through(report) 

383 pruned = { 

384 str(uri) 

385 for event in events 

386 if event.event_type == PRUNED_EVENT and (through is None or event.sequence <= through) 

387 for uri in event.payload.get("artifactRefs", []) 

388 } 

389 seen: set[tuple[str, str]] = set() 

390 for raw in self.s.state.list_dicts("artifact", execution_id=report.execution_id): 

391 try: 

392 artifact = Artifact.model_validate(raw) 

393 except ValidationError as error: 

394 report.artifact_problems.append( 

395 {"uri": None, "problem": "unreadable-record", "detail": str(error)[:300]} 

396 ) 

397 continue 

398 self._check_artifact(report, artifact.uri, artifact.digest, pruned, seen) 

399 for raw in self.s.state.list_dicts("evidence", execution_id=report.execution_id): 

400 try: 

401 evidence = Evidence.model_validate(raw) 

402 except ValidationError: 

403 continue # reported by the record check 

404 self._check_artifact(report, evidence.artifact_ref, evidence.digest, pruned, seen) 

405 if through is not None: 

406 self._explain_unanchored_prunes(report, events, through) 

407 

408 @staticmethod 

409 def _explain_unanchored_prunes( 

410 report: RunVerification, events: list[StoredEvent], through: int 

411 ) -> None: 

412 """Say why a missing artifact that a prune event names was not excused: the event is 

413 after the anchored head.""" 

414 later = { 

415 str(uri) 

416 for event in events 

417 if event.event_type == PRUNED_EVENT and event.sequence > through 

418 for uri in event.payload.get("artifactRefs", []) 

419 } 

420 for problem in report.artifact_problems: 

421 if problem.get("problem") == "missing" and problem.get("uri") in later: 

422 problem["detail"] = ( 

423 "named by a retention.artifacts.pruned event after the anchored head " 

424 f"(sequence {through}), which the anchor does not cover" 

425 ) 

426 

427 def _check_artifact( 

428 self, 

429 report: RunVerification, 

430 uri: str, 

431 digest: str, 

432 pruned: set[str], 

433 seen: set[tuple[str, str]], 

434 ) -> None: 

435 if (uri, digest) in seen: 

436 return 

437 seen.add((uri, digest)) 

438 if not uri.startswith("artifact://"): 438 ↛ 439line 438 didn't jump to line 439 because the condition on line 438 was never true

439 return 

440 report.artifacts_checked += 1 

441 expected = "sha256:" + uri.removeprefix("artifact://sha256/") 

442 if digest != expected: 442 ↛ 443line 442 didn't jump to line 443 because the condition on line 442 was never true

443 report.artifact_problems.append( 

444 {"uri": uri, "problem": "digest-differs", "detail": f"record states {digest}"} 

445 ) 

446 return 

447 if uri in pruned: 

448 return 

449 try: 

450 self.s.artifacts.get(uri) 

451 except FileNotFoundError: 

452 report.artifact_problems.append({"uri": uri, "problem": "missing"}) 

453 except (OSError, ValueError) as error: 

454 report.artifact_problems.append( 

455 {"uri": uri, "problem": "content-differs", "detail": str(error)} 

456 ) 

457 

458 @staticmethod 

459 def _problem( 

460 record_type: str, record_id: str | None, problem: str, detail: str | None = None 

461 ) -> dict[str, Any]: 

462 value: dict[str, Any] = { 

463 "recordType": record_type, 

464 "recordId": record_id, 

465 "problem": problem, 

466 } 

467 if detail: 

468 value["detail"] = detail 

469 return value