Coverage for src/governed_harness/orchestration/budget.py: 99%
96 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Governed budget of agent calls (#42): accounting from the provider's records and limits.
3Usage is what the providers reported (``ResourceUsage``) and the wall time the harness measured
4for each agent invocation; nothing is estimated. Limits apply per agent call, per task (every
5run of the task id) and per run, for cost (USD, only when a provider reports it), tokens
6(input, output and reasoning) and wall seconds. Crossing ``warnAt`` of a limit records a
7warning once; crossing a limit fails closed: no further agent call is made until a person
8raises the limit (``harness budget raise``), which is recorded on the run's event chain."""
10from __future__ import annotations
12from collections.abc import Iterable
13from dataclasses import dataclass, field
14from typing import Any, Literal
16from governed_harness.configuration.agent_results import BudgetConfig, BudgetLimits
17from governed_harness.domain.models import AgentInvocation, ResourceUsage
19Scope = Literal["call", "task", "run"]
20Metric = Literal["costUsd", "tokens", "wallSeconds"]
21METRICS: tuple[Metric, ...] = ("costUsd", "tokens", "wallSeconds")
22SCOPES: tuple[Scope, ...] = ("call", "task", "run")
25@dataclass(frozen=True)
26class Usage:
27 cost_usd: float | None = None
28 tokens: int = 0
29 wall_seconds: float = 0.0
30 calls: int = 0
32 def value(self, metric: Metric) -> float | None:
33 if metric == "costUsd":
34 return self.cost_usd
35 if metric == "tokens":
36 return float(self.tokens)
37 return self.wall_seconds
39 def as_dict(self) -> dict[str, Any]:
40 return {
41 "costUsd": None if self.cost_usd is None else round(self.cost_usd, 6),
42 "tokens": self.tokens,
43 "wallSeconds": round(self.wall_seconds, 3),
44 "calls": self.calls,
45 }
48def usage_of(invocations: Iterable[AgentInvocation], usages: Iterable[ResourceUsage]) -> Usage:
49 """The usage of a set of agent invocations: reported tokens and cost, measured wall time."""
50 by_invocation = {item.invocation_id: item for item in usages if item.invocation_id}
51 cost: float | None = None
52 tokens = 0
53 wall = 0.0
54 calls = 0
55 for invocation in invocations:
56 calls += 1
57 wall += max(0.0, (invocation.finished_at - invocation.started_at).total_seconds())
58 reported = by_invocation.get(invocation.invocation_id)
59 if reported is None:
60 continue
61 tokens += (
62 (reported.input_tokens or 0)
63 + (reported.output_tokens or 0)
64 + (reported.reasoning_tokens or 0)
65 )
66 if reported.cost_usd is not None: 66 ↛ 55line 66 didn't jump to line 55 because the condition on line 66 was always true
67 cost = (cost or 0.0) + reported.cost_usd
68 return Usage(cost_usd=cost, tokens=tokens, wall_seconds=wall, calls=calls)
71@dataclass(frozen=True)
72class Crossing:
73 scope: Scope
74 metric: Metric
75 used: float
76 limit: float
78 @property
79 def key(self) -> str:
80 return f"{self.scope}:{self.metric}:{self.limit:g}"
82 def as_dict(self) -> dict[str, Any]:
83 return {"scope": self.scope, "metric": self.metric, "used": self.used, "limit": self.limit}
85 def describe(self) -> str:
86 unit = {"costUsd": "USD", "tokens": "tokens", "wallSeconds": "s"}[self.metric]
87 return f"{self.scope} {self.metric} {self.used:g} of {self.limit:g} {unit}"
90@dataclass(frozen=True)
91class BudgetCheck:
92 exceeded: tuple[Crossing, ...] = ()
93 warnings: tuple[Crossing, ...] = ()
94 remaining: dict[str, dict[str, float]] = field(default_factory=dict)
97def _limits(config: BudgetConfig, scope: Scope) -> BudgetLimits | None:
98 return {"call": config.per_call, "task": config.per_task, "run": config.per_run}[scope]
101def effective_limit(
102 config: BudgetConfig, scope: Scope, metric: Metric, raised: dict[str, dict[str, float]]
103) -> float | None:
104 """The configured limit, or the one a person raised it to (never lower)."""
105 limits = _limits(config, scope)
106 configured: float | None = None
107 if limits is not None:
108 value = {
109 "costUsd": limits.cost_usd,
110 "tokens": limits.tokens,
111 "wallSeconds": limits.wall_seconds,
112 }[metric]
113 configured = float(value) if value is not None else None
114 override = raised.get(scope, {}).get(metric)
115 if override is None:
116 return configured
117 return override if configured is None else max(configured, override)
120def evaluate(
121 config: BudgetConfig,
122 usage: dict[Scope, Usage],
123 raised: dict[str, dict[str, float]] | None = None,
124) -> BudgetCheck:
125 """Compare the usage of each scope with its limits. A scope without a usage entry is not
126 checked (the call scope is checked only after a call)."""
127 raised = raised or {}
128 exceeded: list[Crossing] = []
129 warnings: list[Crossing] = []
130 remaining: dict[str, dict[str, float]] = {}
131 for scope in SCOPES:
132 current = usage.get(scope)
133 for metric in METRICS:
134 _check_metric(config, scope, metric, current, raised, exceeded, warnings, remaining)
135 return BudgetCheck(tuple(exceeded), tuple(warnings), remaining)
138def _check_metric(
139 config: BudgetConfig,
140 scope: Scope,
141 metric: Metric,
142 current: Usage | None,
143 raised: dict[str, dict[str, float]],
144 exceeded: list[Crossing],
145 warnings: list[Crossing],
146 remaining: dict[str, dict[str, float]],
147) -> None:
148 """Compare one metric of one scope with its limit: what remains, and a crossing of the
149 limit or of its warning ratio."""
150 limit = effective_limit(config, scope, metric, raised)
151 if limit is None:
152 return
153 used = current.value(metric) if current is not None else None
154 if scope != "call":
155 remaining.setdefault(scope, {})[metric] = max(0.0, limit - (used or 0.0))
156 else:
157 remaining.setdefault(scope, {})[metric] = limit
158 if used is None:
159 return
160 crossing = Crossing(scope, metric, round(used, 6), limit)
161 if used >= limit:
162 exceeded.append(crossing)
163 elif used >= limit * config.warning_ratio:
164 warnings.append(crossing)