Coverage for src/governed_harness/orchestration/provenance.py: 94%

121 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Provenance per component and the agent's self-report (``provenance``, since 2.0). 

2 

3``provenance.agentSnapshots``: before and after every agent invocation the harness records the 

4state of the ChangeSet scope as a manifest of digests (only the files that differ from the 

5baseline, so the record stays small in a large repository). A file is attributed to the 

6invocation that changed it; a difference that appears between two invocations, or after the last 

7one, was produced by no agent invocation and is recorded, per file, as an out-of-band edit 

8(``provenance.out-of-band-edit`` events). When the run reaches DECISION the provenance of every 

9ChangeSet file (``AGENT`` with the invocation, or ``OUT_OF_BAND``) is recorded as evidence. This 

10is attribution, not a gate condition: an edit by the person who reviews is legitimate, and the 

11record says that it happened. 

12 

13``provenance.selfReport``: the agent's structured self-report is stored as data of quality 

14``REPORTED`` and contrasted with the ChangeSet recorded after the same invocation.""" 

15 

16from __future__ import annotations 

17 

18import json 

19from typing import TYPE_CHECKING, Any, Literal 

20 

21from governed_harness.agents.base import AgentExecutionResult 

22from governed_harness.agents.self_report import parse_self_report 

23from governed_harness.domain.enums import EvidenceKind, PhaseId 

24from governed_harness.domain.ids import new_id 

25from governed_harness.domain.models import ( 

26 AgentSelfReport, 

27 ChangeSet, 

28 ComponentProvenance, 

29 Execution, 

30 FileProvenance, 

31 OutOfBandEdit, 

32 PhaseExecution, 

33 SelfReportContrast, 

34) 

35from governed_harness.evidence import sha256_json 

36 

37if TYPE_CHECKING: 

38 from governed_harness.orchestration.hosts import EngineHost 

39 

40Manifest = dict[str, str | None] 

41"""Path -> digest of every file of the ChangeSet scope that differs from the baseline 

42(``None``: deleted).""" 

43 

44 

45def _status(before: str | None, after: str | None) -> Literal["ADDED", "MODIFIED", "DELETED"]: 

46 if before is None: 46 ↛ 47line 46 didn't jump to line 47 because the condition on line 46 was never true

47 return "ADDED" 

48 if after is None: 48 ↛ 49line 48 didn't jump to line 49 because the condition on line 48 was never true

49 return "DELETED" 

50 return "MODIFIED" 

51 

52 

53class ProvenanceRecorder: 

54 def __init__(self, engine: EngineHost) -> None: 

55 self.engine = engine 

56 self.s = engine.s 

57 self._baselines: dict[str, dict[str, str]] = {} 

58 

59 def _baseline_digest(self, execution: Execution, path: str) -> str | None: 

60 if execution.execution_id not in self._baselines: 

61 self._baselines[execution.execution_id] = self.engine.baseline_digests(execution) 

62 return self._baselines[execution.execution_id].get(path) 

63 

64 # ----- settings ---------------------------------------------------------------------- 

65 @property 

66 def snapshots_enabled(self) -> bool: 

67 return bool(self.s.resolved.project.provenance_settings.agent_snapshots) 

68 

69 @property 

70 def self_report_enabled(self) -> bool: 

71 return bool(self.s.resolved.project.provenance_settings.self_report) 

72 

73 # ----- snapshots around agent invocations ---------------------------------------------- 

74 def current_manifest(self, execution: Execution) -> Manifest: 

75 diff = self.engine._compute_owned_diff(execution) 

76 return {change.path: change.after_digest for change in diff.changes} 

77 

78 def _snapshots(self, execution_id: str) -> list[dict[str, Any]]: 

79 raw = self.s.state.get_flag(f"agentsnapshots:{execution_id}") 

80 if not raw: 

81 return [] 

82 refs: list[str] = json.loads(raw) 

83 return [json.loads(self.s.artifacts.get(ref)) for ref in refs] 

84 

85 def before_invocation(self, execution: Execution) -> None: 

86 """Record what changed since the last invocation before the agent runs again.""" 

87 if not self.snapshots_enabled: 

88 return 

89 self.s.state.set_flag( 

90 f"agentpre:{execution.execution_id}", 

91 json.dumps(self.current_manifest(execution), sort_keys=True), 

92 ) 

93 

94 def after_invocation(self, execution: Execution, result: AgentExecutionResult) -> None: 

95 if not self.snapshots_enabled: 

96 return 

97 pre_raw = self.s.state.get_flag(f"agentpre:{execution.execution_id}") 

98 before: Manifest = json.loads(pre_raw) if pre_raw else {} 

99 after = self.current_manifest(execution) 

100 previous = self._snapshots(execution.execution_id) 

101 if previous: 

102 self._out_of_band( 

103 execution, 

104 previous[-1]["after"], 

105 before, 

106 previous[-1]["invocationId"], 

107 PhaseId.IMPLEMENTATION, 

108 ) 

109 changed = sorted( 

110 path for path in set(before) | set(after) if before.get(path) != after.get(path) 

111 ) 

112 record = { 

113 "invocationId": result.invocation.invocation_id, 

114 "provider": result.invocation.provider, 

115 "before": before, 

116 "after": after, 

117 "changed": changed, 

118 "digest": sha256_json(after), 

119 } 

120 ref = self.s.artifacts.put_json( 

121 record, 

122 metadata={"kind": "agent-snapshot", "executionId": execution.execution_id}, 

123 ) 

124 refs = [ 

125 *json.loads(self.s.state.get_flag(f"agentsnapshots:{execution.execution_id}") or "[]"), 

126 ref.uri, 

127 ] 

128 self.s.state.set_flag(f"agentsnapshots:{execution.execution_id}", json.dumps(refs)) 

129 self.s.events.append( 

130 execution.execution_id, 

131 "provenance.agent.snapshot", 

132 { 

133 "invocationId": result.invocation.invocation_id, 

134 "snapshotRef": ref.uri, 

135 "changedPaths": changed, 

136 "scopeDigest": record["digest"], 

137 }, 

138 ) 

139 

140 def _out_of_band( 

141 self, 

142 execution: Execution, 

143 agent_state: Manifest, 

144 current: Manifest, 

145 after_invocation_id: str | None, 

146 detected_in: PhaseId, 

147 ) -> list[OutOfBandEdit]: 

148 edits: list[OutOfBandEdit] = [] 

149 recorded = { 

150 (item["path"], item.get("currentDigest")) 

151 for item in json.loads( 

152 self.s.state.get_flag(f"outofband:{execution.execution_id}") or "[]" 

153 ) 

154 } 

155 for path in sorted(set(agent_state) | set(current)): 

156 if (path in agent_state) == (path in current) and agent_state.get(path) == current.get( 

157 path 

158 ): 

159 continue 

160 # A path missing from a manifest has its baseline content; None is a deletion. 

161 agent_digest = agent_state.get(path, self._baseline_digest(execution, path)) 

162 current_digest = current.get(path, self._baseline_digest(execution, path)) 

163 if agent_digest == current_digest: 163 ↛ 164line 163 didn't jump to line 164 because the condition on line 163 was never true

164 continue 

165 status = _status(agent_digest, current_digest) 

166 edit = OutOfBandEdit( 

167 path=path, 

168 status=status, 

169 agent_digest=agent_digest, 

170 current_digest=current_digest, 

171 after_invocation_id=after_invocation_id, 

172 detected_in=detected_in, 

173 ) 

174 edits.append(edit) 

175 if (path, current_digest) in recorded: 175 ↛ 176line 175 didn't jump to line 176 because the condition on line 175 was never true

176 continue 

177 recorded.add((path, current_digest)) 

178 self.s.events.append( 

179 execution.execution_id, 

180 "provenance.out-of-band-edit", 

181 edit.model_dump(mode="json", by_alias=True), 

182 ) 

183 self.s.state.set_flag( 

184 f"outofband:{execution.execution_id}", 

185 json.dumps( 

186 [ 

187 {"path": path, "currentDigest": digest} 

188 for path, digest in sorted(recorded, key=str) 

189 ] 

190 ), 

191 ) 

192 return edits 

193 

194 def attribute(self, execution: Execution, change_set: ChangeSet, phase_id: PhaseId) -> None: 

195 """Record who produced each file of the ChangeSet (once per ChangeSet digest).""" 

196 if not self.snapshots_enabled: 

197 return 

198 snapshots = self._snapshots(execution.execution_id) 

199 if not snapshots: 199 ↛ 200line 199 didn't jump to line 200 because the condition on line 199 was never true

200 return 

201 key = f"provenance:{execution.execution_id}:{change_set.digest}" 

202 if self.s.state.get_flag(key): 

203 return 

204 writer: dict[str, tuple[str | None, str]] = {} 

205 for snapshot in snapshots: 

206 for path in snapshot["changed"]: 

207 writer[path] = (snapshot["after"].get(path), snapshot["invocationId"]) 

208 last = snapshots[-1] 

209 current = {item.path: item.after_digest for item in change_set.files} 

210 edits = self._out_of_band(execution, last["after"], current, last["invocationId"], phase_id) 

211 edited = {item.path for item in edits} 

212 files = [] 

213 for item in change_set.files: 

214 written = writer.get(item.path) 

215 agent = ( 

216 written is not None and written[0] == item.after_digest and item.path not in edited 

217 ) 

218 files.append( 

219 FileProvenance( 

220 path=item.path, 

221 status=item.status, 

222 digest=item.after_digest, 

223 source="AGENT" if agent else "OUT_OF_BAND", 

224 invocation_id=written[1] if agent and written else None, 

225 ) 

226 ) 

227 record = ComponentProvenance( 

228 provenance_id=new_id("provenance"), 

229 execution_id=execution.execution_id, 

230 change_set_digest=change_set.digest, 

231 detected_in=phase_id, 

232 files=tuple(files), 

233 out_of_band_edits=tuple(edits), 

234 agent_snapshot_refs=tuple( 

235 json.loads( 

236 self.s.state.get_flag(f"agentsnapshots:{execution.execution_id}") or "[]" 

237 ) 

238 ), 

239 ) 

240 self.s.state.put( 

241 "component_provenance", 

242 record.provenance_id, 

243 record, 

244 execution_id=execution.execution_id, 

245 project_id=execution.project_id, 

246 ) 

247 ref = self.s.artifacts.put_json( 

248 record.model_dump(mode="json", by_alias=True), 

249 metadata={"kind": "component-provenance", "executionId": execution.execution_id}, 

250 ) 

251 out_of_band = sum(1 for item in files if item.source == "OUT_OF_BAND") 

252 self.engine._record_evidence( 

253 execution, 

254 phase_id, 

255 EvidenceKind.OTHER, 

256 ref, 

257 f"Provenance of {len(files)} ChangeSet file(s): {len(files) - out_of_band} by an agent " 

258 f"invocation, {out_of_band} out of band", 

259 supports=(change_set.change_set_id,), 

260 ) 

261 self.s.state.set_flag(key, record.provenance_id) 

262 

263 # ----- self-report --------------------------------------------------------------------- 

264 def record_self_report( 

265 self, 

266 execution: Execution, 

267 phase: PhaseExecution, 

268 result: AgentExecutionResult, 

269 change_set: ChangeSet | None, 

270 ) -> None: 

271 if not self.self_report_enabled or not self.engine._external_provider( 

272 execution.execution_id 

273 ): 

274 return 

275 parsed = parse_self_report(result.self_report) 

276 problems = parsed.problems if parsed else ("the agent's answer carried no self-report",) 

277 contrast = None 

278 if parsed is not None and change_set is not None: 

279 changed = {item.path for item in change_set.files} 

280 contrast = SelfReportContrast( 

281 change_set_digest=change_set.digest, 

282 declared_paths_not_in_change_set=tuple( 

283 path for path in parsed.paths() if path not in changed 

284 ), 

285 unrequested_changes_in_change_set=tuple( 

286 dict.fromkeys( 

287 item.path 

288 for item in parsed.unrequested_changes 

289 if item.path and item.path in changed 

290 ) 

291 ), 

292 ) 

293 report = AgentSelfReport( 

294 report_id=new_id("selfreport"), 

295 execution_id=execution.execution_id, 

296 invocation_id=result.invocation.invocation_id, 

297 provider=result.invocation.provider, 

298 assumptions=parsed.assumptions if parsed else (), 

299 alternatives_discarded=parsed.alternatives_discarded if parsed else (), 

300 low_confidence_areas=parsed.low_confidence_areas if parsed else (), 

301 unrequested_changes=parsed.unrequested_changes if parsed else (), 

302 disputed_acceptance_tests=parsed.disputed_acceptance_tests if parsed else (), 

303 problems=problems, 

304 contrast=contrast, 

305 ) 

306 self.s.state.put( 

307 "agent_self_report", 

308 report.report_id, 

309 report, 

310 execution_id=execution.execution_id, 

311 project_id=execution.project_id, 

312 ) 

313 ref = self.s.artifacts.put_json( 

314 report.model_dump(mode="json", by_alias=True), 

315 metadata={"kind": "agent-self-report", "executionId": execution.execution_id}, 

316 ) 

317 self.engine._record_evidence( 

318 execution, 

319 phase.phase_id, 

320 EvidenceKind.OTHER, 

321 ref, 

322 f"Agent self-report (REPORTED, not verified): {len(report.assumptions)} assumption(s), " 

323 f"{len(report.low_confidence_areas)} low-confidence area(s), " 

324 f"{len(report.unrequested_changes)} unrequested change(s)", 

325 supports=(result.invocation.invocation_id,), 

326 ) 

327 self.s.events.append( 

328 execution.execution_id, 

329 "agent.self_report.recorded", 

330 { 

331 "reportId": report.report_id, 

332 "invocationId": report.invocation_id, 

333 "quality": report.quality.value, 

334 "artifactRef": ref.uri, 

335 "problems": list(report.problems), 

336 }, 

337 )