Coverage for src/governed_harness/application/agent_results.py: 87%

167 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Application helpers of the agent-results settings (#37-#44, #52): parsing of decision 

2options and the commands that act on a run (budget, quarantine, plan approval, check, 

3routing calibration).""" 

4 

5from __future__ import annotations 

6 

7from collections.abc import Sequence 

8from typing import TYPE_CHECKING, Any 

9 

10from governed_harness.configuration.models import ProjectConfiguration 

11from governed_harness.domain.actors import require_human_actor 

12from governed_harness.domain.enums import ActorType, DecisionKind, PhaseId, ResultStatus 

13from governed_harness.domain.errors import ConfigurationError, NotFoundError, PolicyViolationError 

14from governed_harness.domain.models import Actor, ChangeRequestItem 

15from governed_harness.orchestration.acceptance_dispute import DisputeDecision 

16 

17if TYPE_CHECKING: 

18 from governed_harness.orchestration.engine import EngineServices 

19 

20 

21def routing_calibration(services: EngineServices) -> dict[str, Any]: 

22 """Cost per approved task by provider family, call kind, size, model and effort, from the 

23 routing decisions and the usage the providers reported (N14). The suggested table is a 

24 suggestion: nothing is applied.""" 

25 from governed_harness.agents.routing import CalibrationRow, calibrate, suggest_table 

26 from governed_harness.domain.models import AgentInvocation, Execution, ResourceUsage 

27 

28 project_id = services.resolved.project.project_id 

29 usages = { 

30 item.invocation_id: item 

31 for item in services.state.list("resource_usage", ResourceUsage, project_id=project_id) 

32 if item.invocation_id 

33 } 

34 invocations = services.state.list("agent_invocation", AgentInvocation, project_id=project_id) 

35 rows: list[CalibrationRow] = [] 

36 for execution in services.state.list("execution", Execution, project_id=project_id): 

37 approved = execution.status is ResultStatus.PASSED 

38 # Since #85 every reviewer of the review panel records its decision; reviewers on the 

39 # same model share the run's review invocations, so they count once. 

40 reviewers: set[tuple[Any, ...]] = set() 

41 for event in services.events.list(execution.execution_id): 

42 if event.event_type != "agent.routing.decided" or not _counted( 

43 event.payload, reviewers 

44 ): 

45 continue 

46 payload = event.payload 

47 kind = str(payload.get("callKind")) 

48 model = payload.get("model") 

49 costs = [ 

50 usages[item.invocation_id].cost_usd 

51 for item in invocations 

52 if item.execution_id == execution.execution_id 

53 and (item.call_kind or "implement") == kind 

54 and (model is None or item.model == model) 

55 and item.invocation_id in usages 

56 and usages[item.invocation_id].cost_usd is not None 

57 ] 

58 rows.append( 

59 CalibrationRow( 

60 family=str(payload.get("family")), 

61 call_kind=kind, 

62 size=payload.get("size"), 

63 model=model, 

64 effort=payload.get("effort"), 

65 cost_usd=sum(item for item in costs if item is not None) if costs else None, 

66 approved=approved, 

67 run_id=execution.execution_id, 

68 ) 

69 ) 

70 groups = calibrate(rows) 

71 return { 

72 "projectId": project_id, 

73 "decisions": len(rows), 

74 "groups": groups, 

75 "suggestedTables": suggest_table(groups), 

76 "note": ( 

77 "Cost per approved task from the recorded decisions and reported usage; the " 

78 "suggested tables need at least two approved runs per group and are not applied." 

79 ), 

80 } 

81 

82 

83def _counted(payload: dict[str, Any], reviewers: set[tuple[Any, ...]]) -> bool: 

84 """Whether a routing decision is a calibration row: a reviewer's decision counts once per 

85 call kind, model and effort of the run (``reviewers`` collects them).""" 

86 if "reviewer" not in payload: 

87 return True 

88 key = (payload.get("callKind"), payload.get("model"), payload.get("effort")) 

89 if key in reviewers: 

90 return False 

91 reviewers.add(key) 

92 return True 

93 

94 

95def budget_state(services: EngineServices, execution_id: str) -> dict[str, Any]: 

96 """Usage of a run and its task against the configured limits (``budget``).""" 

97 from governed_harness.orchestration.engine import RunEngine 

98 

99 if services.resolved.project.budget is None: 99 ↛ 100line 99 didn't jump to line 100 because the condition on line 99 was never true

100 raise ConfigurationError("budget needs the budget section in project.yaml") 

101 engine = RunEngine(services) 

102 execution = engine.get_execution(execution_id) 

103 check = engine.results.budget_check(execution) 

104 return { 

105 "executionId": execution_id, 

106 "usage": { 

107 "run": engine.results.run_usage(execution).as_dict(), 

108 "task": engine.results.task_usage(execution).as_dict(), 

109 }, 

110 "limits": services.resolved.project.budget.model_dump(mode="json", by_alias=True), 

111 "raised": engine.results.raised_limits(execution), 

112 "remaining": check.remaining if check else {}, 

113 "exceeded": [item.as_dict() for item in check.exceeded] if check else [], 

114 } 

115 

116 

117def raise_budget( 

118 services: EngineServices, 

119 execution_id: str, 

120 *, 

121 scope: str, 

122 metric: str, 

123 limit: float, 

124 actor_id: str, 

125 rationale: str, 

126) -> dict[str, Any]: 

127 """A person raises a limit of the run (recorded on its chain) so the run may continue.""" 

128 from governed_harness.orchestration.budget import METRICS, SCOPES, effective_limit 

129 from governed_harness.orchestration.engine import RunEngine 

130 

131 require_human_actor(actor_id, "raise a budget limit") 

132 config = services.resolved.project.budget 

133 if config is None: 133 ↛ 134line 133 didn't jump to line 134 because the condition on line 133 was never true

134 raise ConfigurationError("budget raise needs the budget section in project.yaml") 

135 if scope not in SCOPES or metric not in METRICS: 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true

136 raise ConfigurationError( 

137 f"scope must be one of {', '.join(SCOPES)} and metric one of {', '.join(METRICS)}" 

138 ) 

139 if not rationale.strip(): 

140 raise PolicyViolationError("raising a budget limit requires a rationale") 

141 engine = RunEngine(services) 

142 execution = engine.get_execution(execution_id) 

143 raised = engine.results.raised_limits(execution) 

144 current = effective_limit(config, scope, metric, raised) 

145 if current is not None and limit <= current: 

146 raise PolicyViolationError( 

147 f"the new {scope} {metric} limit {limit:g} is not above the current {current:g}" 

148 ) 

149 raised.setdefault(scope, {})[metric] = float(limit) 

150 engine.results.set_flag_json(f"budgetraise:{execution_id}", raised) 

151 services.events.append( 

152 execution_id, 

153 "budget.raised", 

154 { 

155 "scope": scope, 

156 "metric": metric, 

157 "previousLimit": current, 

158 "limit": float(limit), 

159 "rationale": rationale.strip(), 

160 }, 

161 actor=Actor(actor_type=ActorType.HUMAN, actor_id=actor_id), 

162 ) 

163 engine.anchor_chain(execution_id) 

164 return budget_state(services, execution_id) 

165 

166 

167def acceptance_state(services: EngineServices, execution_id: str) -> dict[str, Any]: 

168 """The acceptance tests proposed or frozen for a run (``verification.acceptanceTests``).""" 

169 from governed_harness.orchestration.engine import RunEngine 

170 

171 engine = RunEngine(services) 

172 execution = engine.get_execution(execution_id) 

173 state = engine.results.acceptance.state(execution) 

174 if state is None: 174 ↛ 175line 174 didn't jump to line 175 because the condition on line 174 was never true

175 raise NotFoundError(f"run {execution_id} has no acceptance tests") 

176 dispute = engine.results.disputes.state(execution) 

177 return {"executionId": execution_id, **state, **({"dispute": dispute} if dispute else {})} 

178 

179 

180def decide_acceptance_dispute( 

181 services: EngineServices, 

182 execution_id: str, 

183 *, 

184 decision: DisputeDecision, 

185 digest: str, 

186 actor_id: str, 

187 rationale: str, 

188 continue_after: bool, 

189) -> dict[str, Any]: 

190 """Uphold the frozen acceptance tests an agent disputes, or have them written again 

191 (``verification.acceptanceTests.disputes``, #94; a person only).""" 

192 from governed_harness.orchestration.engine import RunEngine 

193 

194 require_human_actor(actor_id, "decide a dispute of acceptance tests") 

195 engine = RunEngine(services) 

196 state = engine.results.disputes.decide( 

197 engine.get_execution(execution_id), 

198 decision=decision, 

199 digest=digest, 

200 actor=Actor(actor_type=ActorType.HUMAN, actor_id=actor_id), 

201 rationale=rationale, 

202 ) 

203 result: dict[str, Any] = {"executionId": execution_id, "dispute": state} 

204 if continue_after: 204 ↛ 209line 204 didn't jump to line 209 because the condition on line 204 was always true

205 result["execution"] = engine.continue_execution(execution_id).model_dump( 

206 mode="json", by_alias=True 

207 ) 

208 else: 

209 engine.anchor_chain(execution_id) 

210 return result 

211 

212 

213def decide_acceptance( 

214 services: EngineServices, 

215 execution_id: str, 

216 *, 

217 decision: DecisionKind, 

218 digest: str, 

219 actor_id: str, 

220 rationale: str, 

221 continue_after: bool, 

222) -> dict[str, Any]: 

223 """Approve (write and freeze) or reject the proposed acceptance tests (a person only).""" 

224 from governed_harness.orchestration.engine import RunEngine 

225 

226 require_human_actor(actor_id, "decide acceptance tests") 

227 engine = RunEngine(services) 

228 state = engine.results.acceptance.decide( 

229 engine.get_execution(execution_id), 

230 decision=decision, 

231 digest=digest, 

232 actor=Actor(actor_type=ActorType.HUMAN, actor_id=actor_id), 

233 rationale=rationale, 

234 ) 

235 result: dict[str, Any] = {"executionId": execution_id, "acceptanceTests": state} 

236 if continue_after: 236 ↛ 241line 236 didn't jump to line 241 because the condition on line 236 was always true

237 result["execution"] = engine.continue_execution(execution_id).model_dump( 

238 mode="json", by_alias=True 

239 ) 

240 else: 

241 engine.anchor_chain(execution_id) 

242 return result 

243 

244 

245def plan_state(services: EngineServices, execution_id: str) -> dict[str, Any]: 

246 """The decomposition of a run (``planning.decomposition``) and the progress of its 

247 sub-tasks.""" 

248 from governed_harness.orchestration.engine import RunEngine 

249 

250 engine = RunEngine(services) 

251 execution = engine.get_execution(execution_id) 

252 state = engine.results.decomposition.state(execution) 

253 if state is None: 

254 raise NotFoundError(f"run {execution_id} has no decomposition") 

255 completed = [ 

256 event.payload 

257 for event in services.events.list(execution_id) 

258 if event.event_type == "subtask.completed" 

259 ] 

260 return { 

261 "executionId": execution_id, 

262 "status": state["status"], 

263 "digest": state["digest"], 

264 "subtasks": state["subtasks"], 

265 "currentSubtask": engine.results.decomposition.index(execution) + 1, 

266 "completed": completed, 

267 } 

268 

269 

270def decide_plan( 

271 services: EngineServices, 

272 execution_id: str, 

273 *, 

274 decision: DecisionKind, 

275 digest: str, 

276 actor_id: str, 

277 rationale: str, 

278 continue_after: bool, 

279) -> dict[str, Any]: 

280 """Approve or reject a proposed decomposition (a person only, bound to its digest).""" 

281 from governed_harness.orchestration.engine import RunEngine 

282 

283 require_human_actor(actor_id, "decide a decomposition plan") 

284 engine = RunEngine(services) 

285 execution = engine.get_execution(execution_id) 

286 state = engine.results.decomposition.decide( 

287 execution, 

288 decision=decision, 

289 digest=digest, 

290 actor=Actor(actor_type=ActorType.HUMAN, actor_id=actor_id), 

291 rationale=rationale, 

292 ) 

293 result: dict[str, Any] = {"executionId": execution_id, "plan": state} 

294 if continue_after: 294 ↛ 299line 294 didn't jump to line 299 because the condition on line 294 was always true

295 result["execution"] = engine.continue_execution(execution_id).model_dump( 

296 mode="json", by_alias=True 

297 ) 

298 else: 

299 engine.anchor_chain(execution_id) 

300 return result 

301 

302 

303def quarantine_run(services: EngineServices, execution_id: str, actor_id: str) -> dict[str, Any]: 

304 """Quarantine the changes of a run that stopped without approval and restore the baseline. 

305 

306 Refused for a run that passed, that waits in DECISION (a person decides there) or whose 

307 project does not set ``governance.stopTheLine``.""" 

308 from governed_harness.orchestration.engine import RunEngine 

309 

310 require_human_actor(actor_id, "quarantine a run's changes") 

311 if not services.resolved.project.governance_settings.stop_the_line: 311 ↛ 312line 311 didn't jump to line 312 because the condition on line 311 was never true

312 raise ConfigurationError("run quarantine needs governance.stopTheLine in project.yaml") 

313 engine = RunEngine(services) 

314 execution = engine.get_execution(execution_id) 

315 if execution.status is ResultStatus.PASSED: 315 ↛ 316line 315 didn't jump to line 316 because the condition on line 315 was never true

316 raise PolicyViolationError(f"run {execution_id} was approved and closed") 

317 if execution.current_phase in {PhaseId.DECISION, PhaseId.CLOSURE} and ( 317 ↛ 320line 317 didn't jump to line 320 because the condition on line 317 was never true

318 execution.status is not ResultStatus.FAILED 

319 ): 

320 raise PolicyViolationError( 

321 f"run {execution_id} waits for a decision; reject it or request changes instead" 

322 ) 

323 actor = Actor(actor_type=ActorType.HUMAN, actor_id=actor_id) 

324 services.events.append(execution_id, "workspace.quarantine.requested", {}, actor=actor) 

325 record = engine.results.stop_line.quarantine(execution, f"quarantined by {actor_id}") 

326 engine.anchor_chain(execution_id) 

327 return { 

328 "executionId": execution_id, 

329 "quarantined": record is not None, 

330 "record": record, 

331 } 

332 

333 

334def _off_or_value(value: Any) -> Any: 

335 """A setting as ``config validate`` shows it: ``off`` when absent, a section as JSON.""" 

336 if hasattr(value, "model_dump"): 

337 return value.model_dump(mode="json", by_alias=True) 

338 return value if value is not None else "off" 

339 

340 

341def _dump_or(section: Any, default: Any) -> Any: 

342 """A configuration section as JSON, or ``default`` when it is absent.""" 

343 return section.model_dump(mode="json", by_alias=True) if section else default 

344 

345 

346def _check_value(value: Any) -> Any: 

347 """One verification check as ``config validate`` shows it.""" 

348 if hasattr(value, "model_dump"): 

349 return value.model_dump(mode="json", by_alias=True) 

350 if isinstance(value, tuple): 350 ↛ 351line 350 didn't jump to line 351 because the condition on line 350 was never true

351 return [item.model_dump(mode="json", by_alias=True) for item in value] 

352 return value 

353 

354 

355def _checks_summary(verification: Any) -> dict[str, Any]: 

356 """The configured verification checks by alias; empty without a verification section.""" 

357 if verification is None: 357 ↛ 358line 357 didn't jump to line 358 because the condition on line 357 was never true

358 return {} 

359 return { 

360 name: _check_value(value) 

361 for name, value in ( 

362 (field.alias or key, getattr(verification, key)) 

363 for key, field in type(verification).model_fields.items() 

364 if key not in {"requirement_traceability", "output_parsers"} 

365 ) 

366 if value is not None 

367 } 

368 

369 

370def agent_results_summary(project: ProjectConfiguration) -> dict[str, Any]: 

371 """Effective agent-results settings for ``config validate``; ``off`` (or false) where a 

372 key is absent, which is the 1.0.0 behaviour.""" 

373 intake = project.intake 

374 verification = project.verification 

375 review = project.review 

376 runtime = project.runtime 

377 governance = project.governance_settings 

378 

379 off = _off_or_value 

380 return { 

381 "ambiguityReview": off(intake.ambiguity_review if intake else None), 

382 "contractRetry": off(runtime.contract_retry), 

383 "validateAnswers": bool(intake and intake.validate_answers), 

384 "agentReview": off(review.agent_review if review else None), 

385 "structuredChanges": bool(review and review.structured_changes), 

386 "gateContract": bool(runtime.gate_contract), 

387 "reproduceFirst": bool(runtime.reproduce_first), 

388 "stopTheLine": off(governance.stop_the_line), 

389 "phasePermissions": bool(governance.phase_permissions), 

390 "checks": _checks_summary(verification), 

391 "planning": _dump_or(project.planning, {"decomposition": "off"}), 

392 "context": _dump_or(project.context, {"manifest": "off"}), 

393 "budget": _dump_or(project.budget, None), 

394 "memory": _dump_or(project.memory, {"learnFromFindings": "off"}), 

395 "agentRouting": _dump_or(project.agent_routing, {"mode": "fixed"}), 

396 "agentContract": _dump_or(project.agent_contract, {"mode": "off"}), 

397 } 

398 

399 

400def parse_change_requests(values: Sequence[str]) -> tuple[ChangeRequestItem, ...]: 

401 """``description::condition`` items of a structured REQUEST_CHANGES, numbered ``CR-1``... 

402 An item without ``::`` is a ``text`` item (shown and sent, not checked by a tool).""" 

403 items: list[ChangeRequestItem] = [] 

404 for number, value in enumerate(values, start=1): 

405 description, separator, condition = value.partition("::") 

406 if not description.strip(): 406 ↛ 407line 406 didn't jump to line 407 because the condition on line 406 was never true

407 raise ConfigurationError(f"change request {number} needs a description") 

408 try: 

409 items.append( 

410 ChangeRequestItem( 

411 item_id=f"CR-{number}", 

412 description=description.strip(), 

413 condition=condition.strip() if separator else "text", 

414 ) 

415 ) 

416 except ValueError as error: 

417 raise ConfigurationError(f"invalid change request {number}: {error}") from error 

418 return tuple(items)