Coverage for src/governed_harness/orchestration/verification.py: 89%
251 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""``harness verify``: check a run's record against its event chain without aborting.
3The events are the audit authority; the ``records`` table is a projection that the CLI, the API
4and the gate read. Before 2.0 nobody compared the two: a forged decider written into the
5projection was shown by ``status`` while the chain still said otherwise, and a chain whose last
6events had been deleted was reported as valid. The verifier reports, for one run:
8* the event chain (sequence, links, digests) and, when ``governance.chainAnchor`` is set, whether
9 the chain still contains the head recorded outside ``.harness`` (truncation);
10* every record that has an event of its own (decisions, gates, ChangeSets, validations,
11 findings, evidence, tool and agent invocations, clarifications), rebuilt from the event and
12 compared with the stored projection, plus the execution's pointers and the phase results;
13* every artifact the run references: present and with the digest its record states.
15The anchor covers the chain up to the anchored head. Under ``governance.unanchoredEvents``
16(#149) the events after it are listed as unanchored (a warning, or a failure under ``fail``), and
17a ``retention.artifacts.pruned`` event excuses a missing artifact only when it is within the
18anchored part: an event appended after the anchor proves nothing the anchor vouches for.
20Nothing is repaired: the report says what differs, and the events say what happened.
21"""
23from __future__ import annotations
25from dataclasses import dataclass, field
26from typing import TYPE_CHECKING, Any
28from pydantic import BaseModel, ValidationError
30from governed_harness.domain.errors import NotFoundError
31from governed_harness.domain.models import (
32 AgentInvocation,
33 Artifact,
34 ChangeSet,
35 ClarificationRecord,
36 ClarificationRequest,
37 Evidence,
38 Execution,
39 Finding,
40 GateEvaluation,
41 HumanDecision,
42 PhaseExecution,
43 ToolInvocation,
44 ValidationResult,
45)
46from governed_harness.events import AnchorStore, StoredEvent
48if TYPE_CHECKING:
49 from governed_harness.orchestration.engine_types import EngineServices
51PROJECTED_EVENTS: dict[str, tuple[str, type[BaseModel], str]] = {
52 "human.decision.recorded": ("decision", HumanDecision, "decision_id"),
53 "gate.evaluated": ("gate", GateEvaluation, "gate_evaluation_id"),
54 "changeset.created": ("change_set", ChangeSet, "change_set_id"),
55 "validation.completed": ("validation", ValidationResult, "validation_result_id"),
56 "finding.recorded": ("finding", Finding, "finding_id"),
57 "evidence.recorded": ("evidence", Evidence, "evidence_id"),
58 "tool.invocation.completed": ("tool_invocation", ToolInvocation, "invocation_id"),
59 "agent.invocation.completed": ("agent_invocation", AgentInvocation, "invocation_id"),
60 "intent.clarified": ("clarification", ClarificationRecord, "clarification_id"),
61 "intent.clarification.requested": (
62 "clarification_request",
63 ClarificationRequest,
64 "request_id",
65 ),
66}
67"""Event type -> (record type, model, id field) of the records projected one-to-one from events."""
69PRUNED_EVENT = "retention.artifacts.pruned"
70VALID_ANCHOR = frozenset({"matched", "absent", "off"})
71UNANCHORED = "unanchored"
72"""The anchor status of a chain with events after its anchored head under
73``unanchoredEvents: fail``."""
74SHOWN_UNANCHORED = 50
77@dataclass
78class RunVerification:
79 execution_id: str
80 chain: dict[str, Any]
81 anchor: dict[str, Any]
82 records_checked: int = 0
83 record_problems: list[dict[str, Any]] = field(default_factory=list)
84 artifacts_checked: int = 0
85 artifact_problems: list[dict[str, Any]] = field(default_factory=list)
86 warnings: list[str] = field(default_factory=list)
87 """Since #149, under ``governance.unanchoredEvents: warn``: what verifies but the anchor does
88 not cover. A report without warnings keeps its earlier form."""
90 @property
91 def valid(self) -> bool:
92 return (
93 bool(self.chain["valid"])
94 and self.anchor["status"] in VALID_ANCHOR
95 and not self.record_problems
96 and not self.artifact_problems
97 )
99 def as_dict(self) -> dict[str, Any]:
100 value = {
101 "executionId": self.execution_id,
102 "valid": self.valid,
103 "eventChain": self.chain,
104 "anchor": self.anchor,
105 "records": {"checked": self.records_checked, "problems": self.record_problems},
106 "artifacts": {"checked": self.artifacts_checked, "problems": self.artifact_problems},
107 }
108 if self.warnings:
109 value["warnings"] = list(self.warnings)
110 return value
112 def summary(self) -> str:
113 reasons = []
114 if not self.chain["valid"]:
115 reasons.append(f"event chain: {self.chain['error']}")
116 if self.anchor["status"] not in VALID_ANCHOR:
117 reasons.append(f"anchor: {self.anchor['status']}")
118 if self.record_problems: 118 ↛ 120line 118 didn't jump to line 120 because the condition on line 118 was always true
119 reasons.append(f"{len(self.record_problems)} record(s) differ from the events")
120 if self.artifact_problems: 120 ↛ 121line 120 didn't jump to line 121 because the condition on line 120 was never true
121 reasons.append(f"{len(self.artifact_problems)} artifact problem(s)")
122 return "; ".join(reasons) or "valid"
125class RunVerifier:
126 def __init__(self, services: EngineServices) -> None:
127 self.s = services
129 def anchor_store(self) -> AnchorStore:
130 mode = self.s.resolved.project.governance_settings.chain_anchor or "off"
131 return AnchorStore(mode, self.s.paths.workspace, self.s.resolved.project.project_id)
133 def verify(self, execution_id: str) -> RunVerification:
134 events = self._events(execution_id)
135 has_record = self._has_execution_record(execution_id)
136 if not events and not has_record:
137 raise NotFoundError(f"run not found: {execution_id}")
138 check = self.s.events.check_chain(execution_id)
139 report = RunVerification(
140 execution_id=execution_id,
141 chain=check.as_dict(),
142 anchor=self._anchor(execution_id, events),
143 )
144 self._check_unanchored(report, events)
145 self._check_records(report, events)
146 self._check_artifacts(report, events)
147 return report
149 @property
150 def unanchored_policy(self) -> str | None:
151 """``governance.unanchoredEvents`` when an anchor is kept, else ``None`` (1.x)."""
152 settings = self.s.resolved.project.governance_settings
153 if not settings.chain_anchor or settings.chain_anchor == "off":
154 return None
155 return settings.unanchored_events
157 def _anchored_through(self, report: RunVerification) -> int | None:
158 """The last sequence the anchor covers under ``unanchoredEvents`` (``0``: none, the
159 anchor is absent or does not match); ``None``: no limit (the key is absent)."""
160 if self.unanchored_policy is None:
161 return None
162 if report.anchor["status"] in {"matched", UNANCHORED}:
163 return int(report.anchor["sequence"])
164 return 0
166 def _check_unanchored(self, report: RunVerification, events: list[StoredEvent]) -> None:
167 """List the events after the anchored head: a warning, or a failed anchor under
168 ``fail``."""
169 policy = self.unanchored_policy
170 if policy is None or report.anchor["status"] != "matched":
171 return
172 sequence = int(report.anchor["sequence"])
173 later = [event for event in events if event.sequence > sequence]
174 if not later:
175 return
176 report.anchor["unanchoredEvents"] = [
177 {"sequence": event.sequence, "eventType": event.event_type}
178 for event in later[:SHOWN_UNANCHORED]
179 ]
180 if policy == "fail":
181 report.anchor["status"] = UNANCHORED
182 return
183 report.warnings.append(
184 f"{len(later)} event(s) after the anchored head (sequence {sequence}) are not "
185 "covered by the anchor"
186 )
188 def execution_ids(self) -> list[str]:
189 ids = dict.fromkeys(self.s.events.execution_ids())
190 for item in self.s.state.list_dicts(
191 "execution", project_id=self.s.resolved.project.project_id
192 ):
193 ids.setdefault(str(item.get("executionId")), None)
194 return list(ids)
196 # ----- parts ------------------------------------------------------------------
197 def _events(self, execution_id: str) -> list[StoredEvent]:
198 try:
199 return self.s.events.list(execution_id)
200 except (ValueError, TypeError):
201 return []
203 def _has_execution_record(self, execution_id: str) -> bool:
204 try:
205 self.s.state.get_dict("execution", execution_id)
206 except NotFoundError:
207 return False
208 return True
210 def _anchor(self, execution_id: str, events: list[StoredEvent]) -> dict[str, Any]:
211 store = self.anchor_store()
212 if store.mode == "off":
213 return {"mode": "off", "status": "off"}
214 anchor = store.read(execution_id)
215 base: dict[str, Any] = {"mode": store.mode, "location": store.location}
216 if anchor is None:
217 return {**base, "status": "absent"}
218 base.update(anchor.as_dict())
219 at = next((event for event in events if event.sequence == anchor.sequence), None)
220 if at is None:
221 return {**base, "status": "truncated", "eventCount": len(events)}
222 if at.event_digest != anchor.event_digest: 222 ↛ 223line 222 didn't jump to line 223 because the condition on line 222 was never true
223 return {**base, "status": "rewritten"}
224 later = sum(1 for event in events if event.sequence > anchor.sequence)
225 return {**base, "status": "matched", "eventsAfterAnchor": later}
227 def _check_records(self, report: RunVerification, events: list[StoredEvent]) -> None:
228 execution_id = report.execution_id
229 from_events = self._projected(report, events)
230 stored = self._stored(report, execution_id)
231 report.records_checked = len(set(from_events) | set(stored))
232 for key in sorted(set(from_events) | set(stored)):
233 record_type, record_id = key
234 if key not in stored: 234 ↛ 235line 234 didn't jump to line 235 because the condition on line 234 was never true
235 report.record_problems.append(self._problem(record_type, record_id, "missing"))
236 elif key not in from_events:
237 report.record_problems.append(
238 self._problem(record_type, record_id, "no-event", "the record has no event")
239 )
240 elif stored[key] != from_events[key]:
241 fields = sorted(
242 name
243 for name in set(stored[key]) | set(from_events[key])
244 if stored[key].get(name) != from_events[key].get(name)
245 )
246 report.record_problems.append(
247 self._problem(record_type, record_id, "differs", ", ".join(fields))
248 )
249 self._check_execution(report, events, from_events)
250 self._check_phases(report, events)
252 def _projected(
253 self, report: RunVerification, events: list[StoredEvent]
254 ) -> dict[tuple[str, str], dict[str, Any]]:
255 """The records the events project, by (record type, id); an unreadable event is a
256 problem of the report."""
257 from_events: dict[tuple[str, str], dict[str, Any]] = {}
258 for event in events:
259 projected = PROJECTED_EVENTS.get(event.event_type)
260 if projected is None:
261 continue
262 record_type, model, id_field = projected
263 try:
264 item = model.model_validate(event.payload)
265 except ValidationError as error:
266 report.record_problems.append(
267 self._problem(record_type, None, "unreadable-event", str(error)[:300])
268 )
269 continue
270 key = (record_type, str(getattr(item, id_field)))
271 from_events[key] = item.model_dump(mode="json")
272 return from_events
274 def _stored(
275 self, report: RunVerification, execution_id: str
276 ) -> dict[tuple[str, str], dict[str, Any]]:
277 """The stored records of the run, by (record type, id); an unreadable record is a
278 problem of the report."""
279 stored: dict[tuple[str, str], dict[str, Any]] = {}
280 for record_type, model, id_field in {
281 (value[0], value[1], value[2]) for value in PROJECTED_EVENTS.values()
282 }:
283 for raw in self.s.state.list_dicts(record_type, execution_id=execution_id):
284 try:
285 item = model.model_validate(raw)
286 except ValidationError as error:
287 report.record_problems.append(
288 self._problem(record_type, None, "unreadable-record", str(error)[:300])
289 )
290 continue
291 stored[(record_type, str(getattr(item, id_field)))] = item.model_dump(mode="json")
292 return stored
294 def _check_execution(
295 self,
296 report: RunVerification,
297 events: list[StoredEvent],
298 from_events: dict[tuple[str, str], dict[str, Any]],
299 ) -> None:
300 try:
301 execution = self.s.state.get("execution", report.execution_id, Execution)
302 except NotFoundError:
303 report.record_problems.append(
304 self._problem("execution", report.execution_id, "missing")
305 )
306 return
307 except ValidationError as error:
308 report.record_problems.append(
309 self._problem("execution", report.execution_id, "unreadable-record", str(error))
310 )
311 return
312 report.records_checked += 1
313 created = next((event for event in events if event.event_type == "run.created"), None)
314 mismatches: list[str] = []
315 if created is not None: 315 ↛ 325line 315 didn't jump to line 325 because the condition on line 315 was always true
316 expected = {
317 "taskId": execution.task_id,
318 "configurationDigest": execution.configuration_digest,
319 "workflowDigest": execution.workflow_digest,
320 "policyDigest": execution.policy_digest,
321 }
322 mismatches.extend(
323 name for name, value in expected.items() if created.payload.get(name) != value
324 )
325 if (
326 execution.human_decision_id
327 and ("decision", execution.human_decision_id) not in from_events
328 ):
329 mismatches.append("humanDecisionId")
330 if ( 330 ↛ 334line 330 didn't jump to line 334 because the condition on line 330 was never true
331 execution.gate_evaluation_id
332 and ("gate", execution.gate_evaluation_id) not in from_events
333 ):
334 mismatches.append("gateEvaluationId")
335 digests = {
336 str(value.get("digest"))
337 for (record_type, _), value in from_events.items()
338 if record_type == "change_set"
339 }
340 if execution.change_set_digest and execution.change_set_digest not in digests: 340 ↛ 341line 340 didn't jump to line 341 because the condition on line 340 was never true
341 mismatches.append("changeSetDigest")
342 if mismatches:
343 report.record_problems.append(
344 self._problem("execution", report.execution_id, "differs", ", ".join(mismatches))
345 )
347 def _check_phases(self, report: RunVerification, events: list[StoredEvent]) -> None:
348 completed = {
349 event.phase_execution_id: event.payload
350 for event in events
351 if event.event_type == "phase.completed" and event.phase_execution_id
352 }
353 started = {
354 event.phase_execution_id
355 for event in events
356 if event.event_type == "phase.started" and event.phase_execution_id
357 }
358 for raw in self.s.state.list_dicts("phase", execution_id=report.execution_id):
359 try:
360 phase = PhaseExecution.model_validate(raw)
361 except ValidationError as error:
362 report.record_problems.append(
363 self._problem("phase", None, "unreadable-record", str(error)[:300])
364 )
365 continue
366 report.records_checked += 1
367 payload = completed.get(phase.phase_execution_id)
368 if payload is None:
369 if phase.phase_execution_id not in started:
370 report.record_problems.append(
371 self._problem("phase", phase.phase_execution_id, "no-event")
372 )
373 continue
374 if payload.get("status") != phase.status.value or payload.get("phaseId") != ( 374 ↛ 377line 374 didn't jump to line 377 because the condition on line 374 was never true
375 phase.phase_id.value
376 ):
377 report.record_problems.append(
378 self._problem("phase", phase.phase_execution_id, "differs", "status, phaseId")
379 )
381 def _check_artifacts(self, report: RunVerification, events: list[StoredEvent]) -> None:
382 through = self._anchored_through(report)
383 pruned = {
384 str(uri)
385 for event in events
386 if event.event_type == PRUNED_EVENT and (through is None or event.sequence <= through)
387 for uri in event.payload.get("artifactRefs", [])
388 }
389 seen: set[tuple[str, str]] = set()
390 for raw in self.s.state.list_dicts("artifact", execution_id=report.execution_id):
391 try:
392 artifact = Artifact.model_validate(raw)
393 except ValidationError as error:
394 report.artifact_problems.append(
395 {"uri": None, "problem": "unreadable-record", "detail": str(error)[:300]}
396 )
397 continue
398 self._check_artifact(report, artifact.uri, artifact.digest, pruned, seen)
399 for raw in self.s.state.list_dicts("evidence", execution_id=report.execution_id):
400 try:
401 evidence = Evidence.model_validate(raw)
402 except ValidationError:
403 continue # reported by the record check
404 self._check_artifact(report, evidence.artifact_ref, evidence.digest, pruned, seen)
405 if through is not None:
406 self._explain_unanchored_prunes(report, events, through)
408 @staticmethod
409 def _explain_unanchored_prunes(
410 report: RunVerification, events: list[StoredEvent], through: int
411 ) -> None:
412 """Say why a missing artifact that a prune event names was not excused: the event is
413 after the anchored head."""
414 later = {
415 str(uri)
416 for event in events
417 if event.event_type == PRUNED_EVENT and event.sequence > through
418 for uri in event.payload.get("artifactRefs", [])
419 }
420 for problem in report.artifact_problems:
421 if problem.get("problem") == "missing" and problem.get("uri") in later:
422 problem["detail"] = (
423 "named by a retention.artifacts.pruned event after the anchored head "
424 f"(sequence {through}), which the anchor does not cover"
425 )
427 def _check_artifact(
428 self,
429 report: RunVerification,
430 uri: str,
431 digest: str,
432 pruned: set[str],
433 seen: set[tuple[str, str]],
434 ) -> None:
435 if (uri, digest) in seen:
436 return
437 seen.add((uri, digest))
438 if not uri.startswith("artifact://"): 438 ↛ 439line 438 didn't jump to line 439 because the condition on line 438 was never true
439 return
440 report.artifacts_checked += 1
441 expected = "sha256:" + uri.removeprefix("artifact://sha256/")
442 if digest != expected: 442 ↛ 443line 442 didn't jump to line 443 because the condition on line 442 was never true
443 report.artifact_problems.append(
444 {"uri": uri, "problem": "digest-differs", "detail": f"record states {digest}"}
445 )
446 return
447 if uri in pruned:
448 return
449 try:
450 self.s.artifacts.get(uri)
451 except FileNotFoundError:
452 report.artifact_problems.append({"uri": uri, "problem": "missing"})
453 except (OSError, ValueError) as error:
454 report.artifact_problems.append(
455 {"uri": uri, "problem": "content-differs", "detail": str(error)}
456 )
458 @staticmethod
459 def _problem(
460 record_type: str, record_id: str | None, problem: str, detail: str | None = None
461 ) -> dict[str, Any]:
462 value: dict[str, Any] = {
463 "recordType": record_type,
464 "recordId": record_id,
465 "problem": problem,
466 }
467 if detail:
468 value["detail"] = detail
469 return value