Coverage for src/governed_harness/orchestration/provenance.py: 94%
121 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Provenance per component and the agent's self-report (``provenance``, since 2.0).
3``provenance.agentSnapshots``: before and after every agent invocation the harness records the
4state of the ChangeSet scope as a manifest of digests (only the files that differ from the
5baseline, so the record stays small in a large repository). A file is attributed to the
6invocation that changed it; a difference that appears between two invocations, or after the last
7one, was produced by no agent invocation and is recorded, per file, as an out-of-band edit
8(``provenance.out-of-band-edit`` events). When the run reaches DECISION the provenance of every
9ChangeSet file (``AGENT`` with the invocation, or ``OUT_OF_BAND``) is recorded as evidence. This
10is attribution, not a gate condition: an edit by the person who reviews is legitimate, and the
11record says that it happened.
13``provenance.selfReport``: the agent's structured self-report is stored as data of quality
14``REPORTED`` and contrasted with the ChangeSet recorded after the same invocation."""
16from __future__ import annotations
18import json
19from typing import TYPE_CHECKING, Any, Literal
21from governed_harness.agents.base import AgentExecutionResult
22from governed_harness.agents.self_report import parse_self_report
23from governed_harness.domain.enums import EvidenceKind, PhaseId
24from governed_harness.domain.ids import new_id
25from governed_harness.domain.models import (
26 AgentSelfReport,
27 ChangeSet,
28 ComponentProvenance,
29 Execution,
30 FileProvenance,
31 OutOfBandEdit,
32 PhaseExecution,
33 SelfReportContrast,
34)
35from governed_harness.evidence import sha256_json
37if TYPE_CHECKING:
38 from governed_harness.orchestration.hosts import EngineHost
40Manifest = dict[str, str | None]
41"""Path -> digest of every file of the ChangeSet scope that differs from the baseline
42(``None``: deleted)."""
45def _status(before: str | None, after: str | None) -> Literal["ADDED", "MODIFIED", "DELETED"]:
46 if before is None: 46 ↛ 47line 46 didn't jump to line 47 because the condition on line 46 was never true
47 return "ADDED"
48 if after is None: 48 ↛ 49line 48 didn't jump to line 49 because the condition on line 48 was never true
49 return "DELETED"
50 return "MODIFIED"
53class ProvenanceRecorder:
54 def __init__(self, engine: EngineHost) -> None:
55 self.engine = engine
56 self.s = engine.s
57 self._baselines: dict[str, dict[str, str]] = {}
59 def _baseline_digest(self, execution: Execution, path: str) -> str | None:
60 if execution.execution_id not in self._baselines:
61 self._baselines[execution.execution_id] = self.engine.baseline_digests(execution)
62 return self._baselines[execution.execution_id].get(path)
64 # ----- settings ----------------------------------------------------------------------
65 @property
66 def snapshots_enabled(self) -> bool:
67 return bool(self.s.resolved.project.provenance_settings.agent_snapshots)
69 @property
70 def self_report_enabled(self) -> bool:
71 return bool(self.s.resolved.project.provenance_settings.self_report)
73 # ----- snapshots around agent invocations ----------------------------------------------
74 def current_manifest(self, execution: Execution) -> Manifest:
75 diff = self.engine._compute_owned_diff(execution)
76 return {change.path: change.after_digest for change in diff.changes}
78 def _snapshots(self, execution_id: str) -> list[dict[str, Any]]:
79 raw = self.s.state.get_flag(f"agentsnapshots:{execution_id}")
80 if not raw:
81 return []
82 refs: list[str] = json.loads(raw)
83 return [json.loads(self.s.artifacts.get(ref)) for ref in refs]
85 def before_invocation(self, execution: Execution) -> None:
86 """Record what changed since the last invocation before the agent runs again."""
87 if not self.snapshots_enabled:
88 return
89 self.s.state.set_flag(
90 f"agentpre:{execution.execution_id}",
91 json.dumps(self.current_manifest(execution), sort_keys=True),
92 )
94 def after_invocation(self, execution: Execution, result: AgentExecutionResult) -> None:
95 if not self.snapshots_enabled:
96 return
97 pre_raw = self.s.state.get_flag(f"agentpre:{execution.execution_id}")
98 before: Manifest = json.loads(pre_raw) if pre_raw else {}
99 after = self.current_manifest(execution)
100 previous = self._snapshots(execution.execution_id)
101 if previous:
102 self._out_of_band(
103 execution,
104 previous[-1]["after"],
105 before,
106 previous[-1]["invocationId"],
107 PhaseId.IMPLEMENTATION,
108 )
109 changed = sorted(
110 path for path in set(before) | set(after) if before.get(path) != after.get(path)
111 )
112 record = {
113 "invocationId": result.invocation.invocation_id,
114 "provider": result.invocation.provider,
115 "before": before,
116 "after": after,
117 "changed": changed,
118 "digest": sha256_json(after),
119 }
120 ref = self.s.artifacts.put_json(
121 record,
122 metadata={"kind": "agent-snapshot", "executionId": execution.execution_id},
123 )
124 refs = [
125 *json.loads(self.s.state.get_flag(f"agentsnapshots:{execution.execution_id}") or "[]"),
126 ref.uri,
127 ]
128 self.s.state.set_flag(f"agentsnapshots:{execution.execution_id}", json.dumps(refs))
129 self.s.events.append(
130 execution.execution_id,
131 "provenance.agent.snapshot",
132 {
133 "invocationId": result.invocation.invocation_id,
134 "snapshotRef": ref.uri,
135 "changedPaths": changed,
136 "scopeDigest": record["digest"],
137 },
138 )
140 def _out_of_band(
141 self,
142 execution: Execution,
143 agent_state: Manifest,
144 current: Manifest,
145 after_invocation_id: str | None,
146 detected_in: PhaseId,
147 ) -> list[OutOfBandEdit]:
148 edits: list[OutOfBandEdit] = []
149 recorded = {
150 (item["path"], item.get("currentDigest"))
151 for item in json.loads(
152 self.s.state.get_flag(f"outofband:{execution.execution_id}") or "[]"
153 )
154 }
155 for path in sorted(set(agent_state) | set(current)):
156 if (path in agent_state) == (path in current) and agent_state.get(path) == current.get(
157 path
158 ):
159 continue
160 # A path missing from a manifest has its baseline content; None is a deletion.
161 agent_digest = agent_state.get(path, self._baseline_digest(execution, path))
162 current_digest = current.get(path, self._baseline_digest(execution, path))
163 if agent_digest == current_digest: 163 ↛ 164line 163 didn't jump to line 164 because the condition on line 163 was never true
164 continue
165 status = _status(agent_digest, current_digest)
166 edit = OutOfBandEdit(
167 path=path,
168 status=status,
169 agent_digest=agent_digest,
170 current_digest=current_digest,
171 after_invocation_id=after_invocation_id,
172 detected_in=detected_in,
173 )
174 edits.append(edit)
175 if (path, current_digest) in recorded: 175 ↛ 176line 175 didn't jump to line 176 because the condition on line 175 was never true
176 continue
177 recorded.add((path, current_digest))
178 self.s.events.append(
179 execution.execution_id,
180 "provenance.out-of-band-edit",
181 edit.model_dump(mode="json", by_alias=True),
182 )
183 self.s.state.set_flag(
184 f"outofband:{execution.execution_id}",
185 json.dumps(
186 [
187 {"path": path, "currentDigest": digest}
188 for path, digest in sorted(recorded, key=str)
189 ]
190 ),
191 )
192 return edits
194 def attribute(self, execution: Execution, change_set: ChangeSet, phase_id: PhaseId) -> None:
195 """Record who produced each file of the ChangeSet (once per ChangeSet digest)."""
196 if not self.snapshots_enabled:
197 return
198 snapshots = self._snapshots(execution.execution_id)
199 if not snapshots: 199 ↛ 200line 199 didn't jump to line 200 because the condition on line 199 was never true
200 return
201 key = f"provenance:{execution.execution_id}:{change_set.digest}"
202 if self.s.state.get_flag(key):
203 return
204 writer: dict[str, tuple[str | None, str]] = {}
205 for snapshot in snapshots:
206 for path in snapshot["changed"]:
207 writer[path] = (snapshot["after"].get(path), snapshot["invocationId"])
208 last = snapshots[-1]
209 current = {item.path: item.after_digest for item in change_set.files}
210 edits = self._out_of_band(execution, last["after"], current, last["invocationId"], phase_id)
211 edited = {item.path for item in edits}
212 files = []
213 for item in change_set.files:
214 written = writer.get(item.path)
215 agent = (
216 written is not None and written[0] == item.after_digest and item.path not in edited
217 )
218 files.append(
219 FileProvenance(
220 path=item.path,
221 status=item.status,
222 digest=item.after_digest,
223 source="AGENT" if agent else "OUT_OF_BAND",
224 invocation_id=written[1] if agent and written else None,
225 )
226 )
227 record = ComponentProvenance(
228 provenance_id=new_id("provenance"),
229 execution_id=execution.execution_id,
230 change_set_digest=change_set.digest,
231 detected_in=phase_id,
232 files=tuple(files),
233 out_of_band_edits=tuple(edits),
234 agent_snapshot_refs=tuple(
235 json.loads(
236 self.s.state.get_flag(f"agentsnapshots:{execution.execution_id}") or "[]"
237 )
238 ),
239 )
240 self.s.state.put(
241 "component_provenance",
242 record.provenance_id,
243 record,
244 execution_id=execution.execution_id,
245 project_id=execution.project_id,
246 )
247 ref = self.s.artifacts.put_json(
248 record.model_dump(mode="json", by_alias=True),
249 metadata={"kind": "component-provenance", "executionId": execution.execution_id},
250 )
251 out_of_band = sum(1 for item in files if item.source == "OUT_OF_BAND")
252 self.engine._record_evidence(
253 execution,
254 phase_id,
255 EvidenceKind.OTHER,
256 ref,
257 f"Provenance of {len(files)} ChangeSet file(s): {len(files) - out_of_band} by an agent "
258 f"invocation, {out_of_band} out of band",
259 supports=(change_set.change_set_id,),
260 )
261 self.s.state.set_flag(key, record.provenance_id)
263 # ----- self-report ---------------------------------------------------------------------
264 def record_self_report(
265 self,
266 execution: Execution,
267 phase: PhaseExecution,
268 result: AgentExecutionResult,
269 change_set: ChangeSet | None,
270 ) -> None:
271 if not self.self_report_enabled or not self.engine._external_provider(
272 execution.execution_id
273 ):
274 return
275 parsed = parse_self_report(result.self_report)
276 problems = parsed.problems if parsed else ("the agent's answer carried no self-report",)
277 contrast = None
278 if parsed is not None and change_set is not None:
279 changed = {item.path for item in change_set.files}
280 contrast = SelfReportContrast(
281 change_set_digest=change_set.digest,
282 declared_paths_not_in_change_set=tuple(
283 path for path in parsed.paths() if path not in changed
284 ),
285 unrequested_changes_in_change_set=tuple(
286 dict.fromkeys(
287 item.path
288 for item in parsed.unrequested_changes
289 if item.path and item.path in changed
290 )
291 ),
292 )
293 report = AgentSelfReport(
294 report_id=new_id("selfreport"),
295 execution_id=execution.execution_id,
296 invocation_id=result.invocation.invocation_id,
297 provider=result.invocation.provider,
298 assumptions=parsed.assumptions if parsed else (),
299 alternatives_discarded=parsed.alternatives_discarded if parsed else (),
300 low_confidence_areas=parsed.low_confidence_areas if parsed else (),
301 unrequested_changes=parsed.unrequested_changes if parsed else (),
302 disputed_acceptance_tests=parsed.disputed_acceptance_tests if parsed else (),
303 problems=problems,
304 contrast=contrast,
305 )
306 self.s.state.put(
307 "agent_self_report",
308 report.report_id,
309 report,
310 execution_id=execution.execution_id,
311 project_id=execution.project_id,
312 )
313 ref = self.s.artifacts.put_json(
314 report.model_dump(mode="json", by_alias=True),
315 metadata={"kind": "agent-self-report", "executionId": execution.execution_id},
316 )
317 self.engine._record_evidence(
318 execution,
319 phase.phase_id,
320 EvidenceKind.OTHER,
321 ref,
322 f"Agent self-report (REPORTED, not verified): {len(report.assumptions)} assumption(s), "
323 f"{len(report.low_confidence_areas)} low-confidence area(s), "
324 f"{len(report.unrequested_changes)} unrequested change(s)",
325 supports=(result.invocation.invocation_id,),
326 )
327 self.s.events.append(
328 execution.execution_id,
329 "agent.self_report.recorded",
330 {
331 "reportId": report.report_id,
332 "invocationId": report.invocation_id,
333 "quality": report.quality.value,
334 "artifactRef": ref.uri,
335 "problems": list(report.problems),
336 },
337 )