Coverage for src/governed_harness/orchestration/budget.py: 99%

96 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Governed budget of agent calls (#42): accounting from the provider's records and limits. 

2 

3Usage is what the providers reported (``ResourceUsage``) and the wall time the harness measured 

4for each agent invocation; nothing is estimated. Limits apply per agent call, per task (every 

5run of the task id) and per run, for cost (USD, only when a provider reports it), tokens 

6(input, output and reasoning) and wall seconds. Crossing ``warnAt`` of a limit records a 

7warning once; crossing a limit fails closed: no further agent call is made until a person 

8raises the limit (``harness budget raise``), which is recorded on the run's event chain.""" 

9 

10from __future__ import annotations 

11 

12from collections.abc import Iterable 

13from dataclasses import dataclass, field 

14from typing import Any, Literal 

15 

16from governed_harness.configuration.agent_results import BudgetConfig, BudgetLimits 

17from governed_harness.domain.models import AgentInvocation, ResourceUsage 

18 

19Scope = Literal["call", "task", "run"] 

20Metric = Literal["costUsd", "tokens", "wallSeconds"] 

21METRICS: tuple[Metric, ...] = ("costUsd", "tokens", "wallSeconds") 

22SCOPES: tuple[Scope, ...] = ("call", "task", "run") 

23 

24 

25@dataclass(frozen=True) 

26class Usage: 

27 cost_usd: float | None = None 

28 tokens: int = 0 

29 wall_seconds: float = 0.0 

30 calls: int = 0 

31 

32 def value(self, metric: Metric) -> float | None: 

33 if metric == "costUsd": 

34 return self.cost_usd 

35 if metric == "tokens": 

36 return float(self.tokens) 

37 return self.wall_seconds 

38 

39 def as_dict(self) -> dict[str, Any]: 

40 return { 

41 "costUsd": None if self.cost_usd is None else round(self.cost_usd, 6), 

42 "tokens": self.tokens, 

43 "wallSeconds": round(self.wall_seconds, 3), 

44 "calls": self.calls, 

45 } 

46 

47 

48def usage_of(invocations: Iterable[AgentInvocation], usages: Iterable[ResourceUsage]) -> Usage: 

49 """The usage of a set of agent invocations: reported tokens and cost, measured wall time.""" 

50 by_invocation = {item.invocation_id: item for item in usages if item.invocation_id} 

51 cost: float | None = None 

52 tokens = 0 

53 wall = 0.0 

54 calls = 0 

55 for invocation in invocations: 

56 calls += 1 

57 wall += max(0.0, (invocation.finished_at - invocation.started_at).total_seconds()) 

58 reported = by_invocation.get(invocation.invocation_id) 

59 if reported is None: 

60 continue 

61 tokens += ( 

62 (reported.input_tokens or 0) 

63 + (reported.output_tokens or 0) 

64 + (reported.reasoning_tokens or 0) 

65 ) 

66 if reported.cost_usd is not None: 66 ↛ 55line 66 didn't jump to line 55 because the condition on line 66 was always true

67 cost = (cost or 0.0) + reported.cost_usd 

68 return Usage(cost_usd=cost, tokens=tokens, wall_seconds=wall, calls=calls) 

69 

70 

71@dataclass(frozen=True) 

72class Crossing: 

73 scope: Scope 

74 metric: Metric 

75 used: float 

76 limit: float 

77 

78 @property 

79 def key(self) -> str: 

80 return f"{self.scope}:{self.metric}:{self.limit:g}" 

81 

82 def as_dict(self) -> dict[str, Any]: 

83 return {"scope": self.scope, "metric": self.metric, "used": self.used, "limit": self.limit} 

84 

85 def describe(self) -> str: 

86 unit = {"costUsd": "USD", "tokens": "tokens", "wallSeconds": "s"}[self.metric] 

87 return f"{self.scope} {self.metric} {self.used:g} of {self.limit:g} {unit}" 

88 

89 

90@dataclass(frozen=True) 

91class BudgetCheck: 

92 exceeded: tuple[Crossing, ...] = () 

93 warnings: tuple[Crossing, ...] = () 

94 remaining: dict[str, dict[str, float]] = field(default_factory=dict) 

95 

96 

97def _limits(config: BudgetConfig, scope: Scope) -> BudgetLimits | None: 

98 return {"call": config.per_call, "task": config.per_task, "run": config.per_run}[scope] 

99 

100 

101def effective_limit( 

102 config: BudgetConfig, scope: Scope, metric: Metric, raised: dict[str, dict[str, float]] 

103) -> float | None: 

104 """The configured limit, or the one a person raised it to (never lower).""" 

105 limits = _limits(config, scope) 

106 configured: float | None = None 

107 if limits is not None: 

108 value = { 

109 "costUsd": limits.cost_usd, 

110 "tokens": limits.tokens, 

111 "wallSeconds": limits.wall_seconds, 

112 }[metric] 

113 configured = float(value) if value is not None else None 

114 override = raised.get(scope, {}).get(metric) 

115 if override is None: 

116 return configured 

117 return override if configured is None else max(configured, override) 

118 

119 

120def evaluate( 

121 config: BudgetConfig, 

122 usage: dict[Scope, Usage], 

123 raised: dict[str, dict[str, float]] | None = None, 

124) -> BudgetCheck: 

125 """Compare the usage of each scope with its limits. A scope without a usage entry is not 

126 checked (the call scope is checked only after a call).""" 

127 raised = raised or {} 

128 exceeded: list[Crossing] = [] 

129 warnings: list[Crossing] = [] 

130 remaining: dict[str, dict[str, float]] = {} 

131 for scope in SCOPES: 

132 current = usage.get(scope) 

133 for metric in METRICS: 

134 _check_metric(config, scope, metric, current, raised, exceeded, warnings, remaining) 

135 return BudgetCheck(tuple(exceeded), tuple(warnings), remaining) 

136 

137 

138def _check_metric( 

139 config: BudgetConfig, 

140 scope: Scope, 

141 metric: Metric, 

142 current: Usage | None, 

143 raised: dict[str, dict[str, float]], 

144 exceeded: list[Crossing], 

145 warnings: list[Crossing], 

146 remaining: dict[str, dict[str, float]], 

147) -> None: 

148 """Compare one metric of one scope with its limit: what remains, and a crossing of the 

149 limit or of its warning ratio.""" 

150 limit = effective_limit(config, scope, metric, raised) 

151 if limit is None: 

152 return 

153 used = current.value(metric) if current is not None else None 

154 if scope != "call": 

155 remaining.setdefault(scope, {})[metric] = max(0.0, limit - (used or 0.0)) 

156 else: 

157 remaining.setdefault(scope, {})[metric] = limit 

158 if used is None: 

159 return 

160 crossing = Crossing(scope, metric, round(used, 6), limit) 

161 if used >= limit: 

162 exceeded.append(crossing) 

163 elif used >= limit * config.warning_ratio: 

164 warnings.append(crossing)