Coverage for src/governed_harness/application/agent_results.py: 87%
167 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Application helpers of the agent-results settings (#37-#44, #52): parsing of decision
2options and the commands that act on a run (budget, quarantine, plan approval, check,
3routing calibration)."""
5from __future__ import annotations
7from collections.abc import Sequence
8from typing import TYPE_CHECKING, Any
10from governed_harness.configuration.models import ProjectConfiguration
11from governed_harness.domain.actors import require_human_actor
12from governed_harness.domain.enums import ActorType, DecisionKind, PhaseId, ResultStatus
13from governed_harness.domain.errors import ConfigurationError, NotFoundError, PolicyViolationError
14from governed_harness.domain.models import Actor, ChangeRequestItem
15from governed_harness.orchestration.acceptance_dispute import DisputeDecision
17if TYPE_CHECKING:
18 from governed_harness.orchestration.engine import EngineServices
21def routing_calibration(services: EngineServices) -> dict[str, Any]:
22 """Cost per approved task by provider family, call kind, size, model and effort, from the
23 routing decisions and the usage the providers reported (N14). The suggested table is a
24 suggestion: nothing is applied."""
25 from governed_harness.agents.routing import CalibrationRow, calibrate, suggest_table
26 from governed_harness.domain.models import AgentInvocation, Execution, ResourceUsage
28 project_id = services.resolved.project.project_id
29 usages = {
30 item.invocation_id: item
31 for item in services.state.list("resource_usage", ResourceUsage, project_id=project_id)
32 if item.invocation_id
33 }
34 invocations = services.state.list("agent_invocation", AgentInvocation, project_id=project_id)
35 rows: list[CalibrationRow] = []
36 for execution in services.state.list("execution", Execution, project_id=project_id):
37 approved = execution.status is ResultStatus.PASSED
38 # Since #85 every reviewer of the review panel records its decision; reviewers on the
39 # same model share the run's review invocations, so they count once.
40 reviewers: set[tuple[Any, ...]] = set()
41 for event in services.events.list(execution.execution_id):
42 if event.event_type != "agent.routing.decided" or not _counted(
43 event.payload, reviewers
44 ):
45 continue
46 payload = event.payload
47 kind = str(payload.get("callKind"))
48 model = payload.get("model")
49 costs = [
50 usages[item.invocation_id].cost_usd
51 for item in invocations
52 if item.execution_id == execution.execution_id
53 and (item.call_kind or "implement") == kind
54 and (model is None or item.model == model)
55 and item.invocation_id in usages
56 and usages[item.invocation_id].cost_usd is not None
57 ]
58 rows.append(
59 CalibrationRow(
60 family=str(payload.get("family")),
61 call_kind=kind,
62 size=payload.get("size"),
63 model=model,
64 effort=payload.get("effort"),
65 cost_usd=sum(item for item in costs if item is not None) if costs else None,
66 approved=approved,
67 run_id=execution.execution_id,
68 )
69 )
70 groups = calibrate(rows)
71 return {
72 "projectId": project_id,
73 "decisions": len(rows),
74 "groups": groups,
75 "suggestedTables": suggest_table(groups),
76 "note": (
77 "Cost per approved task from the recorded decisions and reported usage; the "
78 "suggested tables need at least two approved runs per group and are not applied."
79 ),
80 }
83def _counted(payload: dict[str, Any], reviewers: set[tuple[Any, ...]]) -> bool:
84 """Whether a routing decision is a calibration row: a reviewer's decision counts once per
85 call kind, model and effort of the run (``reviewers`` collects them)."""
86 if "reviewer" not in payload:
87 return True
88 key = (payload.get("callKind"), payload.get("model"), payload.get("effort"))
89 if key in reviewers:
90 return False
91 reviewers.add(key)
92 return True
95def budget_state(services: EngineServices, execution_id: str) -> dict[str, Any]:
96 """Usage of a run and its task against the configured limits (``budget``)."""
97 from governed_harness.orchestration.engine import RunEngine
99 if services.resolved.project.budget is None: 99 ↛ 100line 99 didn't jump to line 100 because the condition on line 99 was never true
100 raise ConfigurationError("budget needs the budget section in project.yaml")
101 engine = RunEngine(services)
102 execution = engine.get_execution(execution_id)
103 check = engine.results.budget_check(execution)
104 return {
105 "executionId": execution_id,
106 "usage": {
107 "run": engine.results.run_usage(execution).as_dict(),
108 "task": engine.results.task_usage(execution).as_dict(),
109 },
110 "limits": services.resolved.project.budget.model_dump(mode="json", by_alias=True),
111 "raised": engine.results.raised_limits(execution),
112 "remaining": check.remaining if check else {},
113 "exceeded": [item.as_dict() for item in check.exceeded] if check else [],
114 }
117def raise_budget(
118 services: EngineServices,
119 execution_id: str,
120 *,
121 scope: str,
122 metric: str,
123 limit: float,
124 actor_id: str,
125 rationale: str,
126) -> dict[str, Any]:
127 """A person raises a limit of the run (recorded on its chain) so the run may continue."""
128 from governed_harness.orchestration.budget import METRICS, SCOPES, effective_limit
129 from governed_harness.orchestration.engine import RunEngine
131 require_human_actor(actor_id, "raise a budget limit")
132 config = services.resolved.project.budget
133 if config is None: 133 ↛ 134line 133 didn't jump to line 134 because the condition on line 133 was never true
134 raise ConfigurationError("budget raise needs the budget section in project.yaml")
135 if scope not in SCOPES or metric not in METRICS: 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true
136 raise ConfigurationError(
137 f"scope must be one of {', '.join(SCOPES)} and metric one of {', '.join(METRICS)}"
138 )
139 if not rationale.strip():
140 raise PolicyViolationError("raising a budget limit requires a rationale")
141 engine = RunEngine(services)
142 execution = engine.get_execution(execution_id)
143 raised = engine.results.raised_limits(execution)
144 current = effective_limit(config, scope, metric, raised)
145 if current is not None and limit <= current:
146 raise PolicyViolationError(
147 f"the new {scope} {metric} limit {limit:g} is not above the current {current:g}"
148 )
149 raised.setdefault(scope, {})[metric] = float(limit)
150 engine.results.set_flag_json(f"budgetraise:{execution_id}", raised)
151 services.events.append(
152 execution_id,
153 "budget.raised",
154 {
155 "scope": scope,
156 "metric": metric,
157 "previousLimit": current,
158 "limit": float(limit),
159 "rationale": rationale.strip(),
160 },
161 actor=Actor(actor_type=ActorType.HUMAN, actor_id=actor_id),
162 )
163 engine.anchor_chain(execution_id)
164 return budget_state(services, execution_id)
167def acceptance_state(services: EngineServices, execution_id: str) -> dict[str, Any]:
168 """The acceptance tests proposed or frozen for a run (``verification.acceptanceTests``)."""
169 from governed_harness.orchestration.engine import RunEngine
171 engine = RunEngine(services)
172 execution = engine.get_execution(execution_id)
173 state = engine.results.acceptance.state(execution)
174 if state is None: 174 ↛ 175line 174 didn't jump to line 175 because the condition on line 174 was never true
175 raise NotFoundError(f"run {execution_id} has no acceptance tests")
176 dispute = engine.results.disputes.state(execution)
177 return {"executionId": execution_id, **state, **({"dispute": dispute} if dispute else {})}
180def decide_acceptance_dispute(
181 services: EngineServices,
182 execution_id: str,
183 *,
184 decision: DisputeDecision,
185 digest: str,
186 actor_id: str,
187 rationale: str,
188 continue_after: bool,
189) -> dict[str, Any]:
190 """Uphold the frozen acceptance tests an agent disputes, or have them written again
191 (``verification.acceptanceTests.disputes``, #94; a person only)."""
192 from governed_harness.orchestration.engine import RunEngine
194 require_human_actor(actor_id, "decide a dispute of acceptance tests")
195 engine = RunEngine(services)
196 state = engine.results.disputes.decide(
197 engine.get_execution(execution_id),
198 decision=decision,
199 digest=digest,
200 actor=Actor(actor_type=ActorType.HUMAN, actor_id=actor_id),
201 rationale=rationale,
202 )
203 result: dict[str, Any] = {"executionId": execution_id, "dispute": state}
204 if continue_after: 204 ↛ 209line 204 didn't jump to line 209 because the condition on line 204 was always true
205 result["execution"] = engine.continue_execution(execution_id).model_dump(
206 mode="json", by_alias=True
207 )
208 else:
209 engine.anchor_chain(execution_id)
210 return result
213def decide_acceptance(
214 services: EngineServices,
215 execution_id: str,
216 *,
217 decision: DecisionKind,
218 digest: str,
219 actor_id: str,
220 rationale: str,
221 continue_after: bool,
222) -> dict[str, Any]:
223 """Approve (write and freeze) or reject the proposed acceptance tests (a person only)."""
224 from governed_harness.orchestration.engine import RunEngine
226 require_human_actor(actor_id, "decide acceptance tests")
227 engine = RunEngine(services)
228 state = engine.results.acceptance.decide(
229 engine.get_execution(execution_id),
230 decision=decision,
231 digest=digest,
232 actor=Actor(actor_type=ActorType.HUMAN, actor_id=actor_id),
233 rationale=rationale,
234 )
235 result: dict[str, Any] = {"executionId": execution_id, "acceptanceTests": state}
236 if continue_after: 236 ↛ 241line 236 didn't jump to line 241 because the condition on line 236 was always true
237 result["execution"] = engine.continue_execution(execution_id).model_dump(
238 mode="json", by_alias=True
239 )
240 else:
241 engine.anchor_chain(execution_id)
242 return result
245def plan_state(services: EngineServices, execution_id: str) -> dict[str, Any]:
246 """The decomposition of a run (``planning.decomposition``) and the progress of its
247 sub-tasks."""
248 from governed_harness.orchestration.engine import RunEngine
250 engine = RunEngine(services)
251 execution = engine.get_execution(execution_id)
252 state = engine.results.decomposition.state(execution)
253 if state is None:
254 raise NotFoundError(f"run {execution_id} has no decomposition")
255 completed = [
256 event.payload
257 for event in services.events.list(execution_id)
258 if event.event_type == "subtask.completed"
259 ]
260 return {
261 "executionId": execution_id,
262 "status": state["status"],
263 "digest": state["digest"],
264 "subtasks": state["subtasks"],
265 "currentSubtask": engine.results.decomposition.index(execution) + 1,
266 "completed": completed,
267 }
270def decide_plan(
271 services: EngineServices,
272 execution_id: str,
273 *,
274 decision: DecisionKind,
275 digest: str,
276 actor_id: str,
277 rationale: str,
278 continue_after: bool,
279) -> dict[str, Any]:
280 """Approve or reject a proposed decomposition (a person only, bound to its digest)."""
281 from governed_harness.orchestration.engine import RunEngine
283 require_human_actor(actor_id, "decide a decomposition plan")
284 engine = RunEngine(services)
285 execution = engine.get_execution(execution_id)
286 state = engine.results.decomposition.decide(
287 execution,
288 decision=decision,
289 digest=digest,
290 actor=Actor(actor_type=ActorType.HUMAN, actor_id=actor_id),
291 rationale=rationale,
292 )
293 result: dict[str, Any] = {"executionId": execution_id, "plan": state}
294 if continue_after: 294 ↛ 299line 294 didn't jump to line 299 because the condition on line 294 was always true
295 result["execution"] = engine.continue_execution(execution_id).model_dump(
296 mode="json", by_alias=True
297 )
298 else:
299 engine.anchor_chain(execution_id)
300 return result
303def quarantine_run(services: EngineServices, execution_id: str, actor_id: str) -> dict[str, Any]:
304 """Quarantine the changes of a run that stopped without approval and restore the baseline.
306 Refused for a run that passed, that waits in DECISION (a person decides there) or whose
307 project does not set ``governance.stopTheLine``."""
308 from governed_harness.orchestration.engine import RunEngine
310 require_human_actor(actor_id, "quarantine a run's changes")
311 if not services.resolved.project.governance_settings.stop_the_line: 311 ↛ 312line 311 didn't jump to line 312 because the condition on line 311 was never true
312 raise ConfigurationError("run quarantine needs governance.stopTheLine in project.yaml")
313 engine = RunEngine(services)
314 execution = engine.get_execution(execution_id)
315 if execution.status is ResultStatus.PASSED: 315 ↛ 316line 315 didn't jump to line 316 because the condition on line 315 was never true
316 raise PolicyViolationError(f"run {execution_id} was approved and closed")
317 if execution.current_phase in {PhaseId.DECISION, PhaseId.CLOSURE} and ( 317 ↛ 320line 317 didn't jump to line 320 because the condition on line 317 was never true
318 execution.status is not ResultStatus.FAILED
319 ):
320 raise PolicyViolationError(
321 f"run {execution_id} waits for a decision; reject it or request changes instead"
322 )
323 actor = Actor(actor_type=ActorType.HUMAN, actor_id=actor_id)
324 services.events.append(execution_id, "workspace.quarantine.requested", {}, actor=actor)
325 record = engine.results.stop_line.quarantine(execution, f"quarantined by {actor_id}")
326 engine.anchor_chain(execution_id)
327 return {
328 "executionId": execution_id,
329 "quarantined": record is not None,
330 "record": record,
331 }
334def _off_or_value(value: Any) -> Any:
335 """A setting as ``config validate`` shows it: ``off`` when absent, a section as JSON."""
336 if hasattr(value, "model_dump"):
337 return value.model_dump(mode="json", by_alias=True)
338 return value if value is not None else "off"
341def _dump_or(section: Any, default: Any) -> Any:
342 """A configuration section as JSON, or ``default`` when it is absent."""
343 return section.model_dump(mode="json", by_alias=True) if section else default
346def _check_value(value: Any) -> Any:
347 """One verification check as ``config validate`` shows it."""
348 if hasattr(value, "model_dump"):
349 return value.model_dump(mode="json", by_alias=True)
350 if isinstance(value, tuple): 350 ↛ 351line 350 didn't jump to line 351 because the condition on line 350 was never true
351 return [item.model_dump(mode="json", by_alias=True) for item in value]
352 return value
355def _checks_summary(verification: Any) -> dict[str, Any]:
356 """The configured verification checks by alias; empty without a verification section."""
357 if verification is None: 357 ↛ 358line 357 didn't jump to line 358 because the condition on line 357 was never true
358 return {}
359 return {
360 name: _check_value(value)
361 for name, value in (
362 (field.alias or key, getattr(verification, key))
363 for key, field in type(verification).model_fields.items()
364 if key not in {"requirement_traceability", "output_parsers"}
365 )
366 if value is not None
367 }
370def agent_results_summary(project: ProjectConfiguration) -> dict[str, Any]:
371 """Effective agent-results settings for ``config validate``; ``off`` (or false) where a
372 key is absent, which is the 1.0.0 behaviour."""
373 intake = project.intake
374 verification = project.verification
375 review = project.review
376 runtime = project.runtime
377 governance = project.governance_settings
379 off = _off_or_value
380 return {
381 "ambiguityReview": off(intake.ambiguity_review if intake else None),
382 "contractRetry": off(runtime.contract_retry),
383 "validateAnswers": bool(intake and intake.validate_answers),
384 "agentReview": off(review.agent_review if review else None),
385 "structuredChanges": bool(review and review.structured_changes),
386 "gateContract": bool(runtime.gate_contract),
387 "reproduceFirst": bool(runtime.reproduce_first),
388 "stopTheLine": off(governance.stop_the_line),
389 "phasePermissions": bool(governance.phase_permissions),
390 "checks": _checks_summary(verification),
391 "planning": _dump_or(project.planning, {"decomposition": "off"}),
392 "context": _dump_or(project.context, {"manifest": "off"}),
393 "budget": _dump_or(project.budget, None),
394 "memory": _dump_or(project.memory, {"learnFromFindings": "off"}),
395 "agentRouting": _dump_or(project.agent_routing, {"mode": "fixed"}),
396 "agentContract": _dump_or(project.agent_contract, {"mode": "off"}),
397 }
400def parse_change_requests(values: Sequence[str]) -> tuple[ChangeRequestItem, ...]:
401 """``description::condition`` items of a structured REQUEST_CHANGES, numbered ``CR-1``...
402 An item without ``::`` is a ``text`` item (shown and sent, not checked by a tool)."""
403 items: list[ChangeRequestItem] = []
404 for number, value in enumerate(values, start=1):
405 description, separator, condition = value.partition("::")
406 if not description.strip(): 406 ↛ 407line 406 didn't jump to line 407 because the condition on line 406 was never true
407 raise ConfigurationError(f"change request {number} needs a description")
408 try:
409 items.append(
410 ChangeRequestItem(
411 item_id=f"CR-{number}",
412 description=description.strip(),
413 condition=condition.strip() if separator else "text",
414 )
415 )
416 except ValueError as error:
417 raise ConfigurationError(f"invalid change request {number}: {error}") from error
418 return tuple(items)