Coverage for src/governed_harness/telemetry/metrics.py: 99%
80 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1from __future__ import annotations
3from dataclasses import dataclass
4from datetime import datetime
5from typing import Any
7from governed_harness.domain.enums import MetricQuality, ResultStatus
8from governed_harness.domain.models import (
9 AgentInvocation,
10 ChangeSet,
11 Finding,
12 HumanDecision,
13 PhaseExecution,
14 ResourceUsage,
15 ToolInvocation,
16 ValidationResult,
17)
18from governed_harness.events.sqlite_store import StoredEvent
19from governed_harness.storage.sqlite import SQLiteStateStore
21_EVENT_STORE = "event store"
23HUMAN_INTERACTION_EVENTS: dict[str, str] = {
24 "human.decision.recorded": "decision",
25 "intent.clarified": "clarification",
26 "planning.decomposition.decided": "plan",
27 "acceptance.tests.decided": "acceptance",
28 "budget.raised": "budget",
29 "workspace.quarantine.requested": "quarantine",
30 "verification.preflight.decided": "preflight",
31 "contract.confirmed": "contract",
32 "evidence.attached": "evidence",
33 "verification.deferred.closed": "evidence",
34 "decision.preauthorized": "preauthorization",
35 "plan.approval.decided": "plan",
36}
37"""Events a person causes on a run, by the kind of interruption they count as (#55; the
38pre-authorised approval and the plan approval since #58)."""
40_SAME_INTERACTION = ("withAnswers", "withPreAuthorization")
41"""Payload markers of an event recorded in the same act as another counted one."""
44def human_interactions(events: list[StoredEvent]) -> dict[str, int]:
45 """The human interactions of a run by kind: events of the types above recorded with a
46 human actor. A contract confirmed in the answers to a clarification (``withAnswers``) or with
47 a pre-authorised approval (``withPreAuthorization``) is part of that one interaction and is
48 not counted again; a decision the harness records from a pre-authorisation has the harness
49 as its actor and is not counted either."""
50 counts: dict[str, int] = {}
51 for event in events:
52 kind = HUMAN_INTERACTION_EVENTS.get(event.event_type)
53 if any(event.payload.get(marker) is True for marker in _SAME_INTERACTION):
54 continue
55 if kind is not None and event.actor.get("actorType") == "HUMAN":
56 counts[kind] = counts.get(kind, 0) + 1
57 return counts
60@dataclass(frozen=True)
61class MetricValue:
62 name: str
63 value: int | float | None
64 unit: str
65 quality: MetricQuality
66 definition: str
67 source: str
68 limitations: tuple[str, ...] = ()
70 def as_dict(self) -> dict[str, Any]:
71 return {
72 "name": self.name,
73 "value": self.value,
74 "unit": self.unit,
75 "quality": self.quality,
76 "definition": self.definition,
77 "source": self.source,
78 "limitations": list(self.limitations),
79 }
82class MetricsProjector:
83 def __init__(self, state: SQLiteStateStore) -> None:
84 self.state = state
86 def project(self, execution_id: str, events: list[StoredEvent]) -> dict[str, MetricValue]:
87 phases = self.state.list("phase", PhaseExecution, execution_id=execution_id)
88 agents = self.state.list("agent_invocation", AgentInvocation, execution_id=execution_id)
89 tools = self.state.list("tool_invocation", ToolInvocation, execution_id=execution_id)
90 validations = self.state.list("validation", ValidationResult, execution_id=execution_id)
91 changes = self.state.list("change_set", ChangeSet, execution_id=execution_id)
92 decisions = self.state.list("decision", HumanDecision, execution_id=execution_id)
93 usages = self.state.list("resource_usage", ResourceUsage, execution_id=execution_id)
94 timestamps = [datetime.fromisoformat(event.occurred_at) for event in events]
95 total_ms = (
96 int((max(timestamps) - min(timestamps)).total_seconds() * 1000)
97 if len(timestamps) >= 2
98 else 0
99 )
100 phase_ms = sum(
101 int((phase.finished_at - phase.started_at).total_seconds() * 1000)
102 for phase in phases
103 if phase.started_at and phase.finished_at
104 )
105 agent_ms = sum(
106 int((item.finished_at - item.started_at).total_seconds() * 1000) for item in agents
107 )
108 tool_ms = sum(
109 int((item.finished_at - item.started_at).total_seconds() * 1000) for item in tools
110 )
111 human_wait_ms: int | None = None
112 gate_events = [event for event in events if event.event_type == "gate.evaluated"]
113 decision_events = [
114 event for event in events if event.event_type == "human.decision.recorded"
115 ]
116 if gate_events and decision_events:
117 decision_time = datetime.fromisoformat(decision_events[-1].occurred_at)
118 eligible_gates = [
119 event
120 for event in gate_events
121 if datetime.fromisoformat(event.occurred_at) <= decision_time
122 ]
123 if eligible_gates: 123 ↛ 130line 123 didn't jump to line 130 because the condition on line 123 was always true
124 human_wait_ms = int(
125 (
126 decision_time - datetime.fromisoformat(eligible_gates[-1].occurred_at)
127 ).total_seconds()
128 * 1000
129 )
130 corrections = [event for event in events if event.event_type == "correction.authorized"]
131 correction_cycles = len(corrections)
132 verification_cycles = sum(
133 1 for event in corrections if event.payload.get("trigger") == "VERIFICATION_FAILED"
134 )
135 transient_retries = sum(
136 1 for event in events if event.event_type == "agent.invocation.retried"
137 )
138 unsupported_claims = sum(
139 1
140 for item in self.state.list("finding", Finding, execution_id=execution_id)
141 if item.rule_id == "agent.unsupported-claim"
142 )
143 implementation_attempts = sum(
144 1
145 for event in events
146 if event.event_type == "phase.started"
147 and event.payload.get("phaseId") == "IMPLEMENTATION"
148 )
149 review_cycles = sum(
150 1
151 for event in events
152 if event.event_type == "phase.started"
153 and event.payload.get("phaseId") == "INDEPENDENT_REVIEW"
154 )
155 replans = sum(1 for event in events if event.event_type == "plan.replaced")
156 failed_validations = sum(
157 1 for item in validations if item.status is not ResultStatus.PASSED
158 )
159 files_changed = len({file.path for change in changes for file in change.files})
160 input_tokens = self._sum_optional(usages, "input_tokens")
161 output_tokens = self._sum_optional(usages, "output_tokens")
162 reasoning_tokens = self._sum_optional(usages, "reasoning_tokens")
163 cost = self._sum_optional(usages, "cost_usd")
164 common = {
165 "duration.total_ms": MetricValue(
166 "duration.total_ms",
167 total_ms,
168 "ms",
169 MetricQuality.OBSERVED,
170 "Wall-clock time between the first and last persisted execution event.",
171 _EVENT_STORE,
172 ),
173 "duration.phase_ms": MetricValue(
174 "duration.phase_ms",
175 phase_ms,
176 "ms",
177 MetricQuality.DERIVED,
178 "Sum of completed phase wall-clock durations; parallel phases would be double-counted.",
179 "phase records",
180 ("Not equal to total wall time when phases overlap.",),
181 ),
182 "duration.agent_ms": MetricValue(
183 "duration.agent_ms",
184 agent_ms,
185 "ms",
186 MetricQuality.DERIVED,
187 "Sum of persisted agent invocation durations.",
188 "agent invocation records",
189 ),
190 "duration.tool_ms": MetricValue(
191 "duration.tool_ms",
192 tool_ms,
193 "ms",
194 MetricQuality.DERIVED,
195 "Sum of persisted tool invocation durations.",
196 "tool invocation records",
197 ),
198 "duration.human_wait_ms": MetricValue(
199 "duration.human_wait_ms",
200 human_wait_ms,
201 "ms",
202 MetricQuality.DERIVED if human_wait_ms is not None else MetricQuality.NOT_AVAILABLE,
203 "Elapsed time from the latest gate evaluation to the latest human decision.",
204 "gate and decision events",
205 () if human_wait_ms is not None else ("No completed human-decision interval.",),
206 ),
207 "agent.invocations": MetricValue(
208 "agent.invocations",
209 len(agents),
210 "count",
211 MetricQuality.OBSERVED,
212 "Number of persisted agent invocation records.",
213 "agent invocation records",
214 ),
215 "tool.invocations": MetricValue(
216 "tool.invocations",
217 len(tools),
218 "count",
219 MetricQuality.OBSERVED,
220 "Number of persisted tool invocation records.",
221 "tool invocation records",
222 ),
223 "implementation.attempts": MetricValue(
224 "implementation.attempts",
225 implementation_attempts,
226 "count",
227 MetricQuality.DERIVED,
228 "Count of IMPLEMENTATION phase-start events.",
229 _EVENT_STORE,
230 ),
231 "correction.cycles": MetricValue(
232 "correction.cycles",
233 correction_cycles,
234 "count",
235 MetricQuality.OBSERVED,
236 "Count of authorized transitions back to IMPLEMENTATION: REQUEST_CHANGES "
237 "decisions and automatic corrections after a failed VERIFICATION.",
238 _EVENT_STORE,
239 ),
240 "correction.verification_cycles": MetricValue(
241 "correction.verification_cycles",
242 verification_cycles,
243 "count",
244 MetricQuality.OBSERVED,
245 "Automatic corrections after a failed VERIFICATION (runtime.verificationCorrections).",
246 _EVENT_STORE,
247 ),
248 "agent.transient_retries": MetricValue(
249 "agent.transient_retries",
250 transient_retries,
251 "count",
252 MetricQuality.OBSERVED,
253 "Command-provider calls repeated after a transient failure (runtime.providerRetries).",
254 _EVENT_STORE,
255 ),
256 "agent.unsupported_claims": MetricValue(
257 "agent.unsupported_claims",
258 unsupported_claims,
259 "count",
260 MetricQuality.DERIVED,
261 "Findings agent.unsupported-claim: the agent reported success and verification failed.",
262 "finding records",
263 ),
264 "review.cycles": MetricValue(
265 "review.cycles",
266 review_cycles,
267 "count",
268 MetricQuality.DERIVED,
269 "Count of INDEPENDENT_REVIEW phase-start events.",
270 _EVENT_STORE,
271 ),
272 "replanning.count": MetricValue(
273 "replanning.count",
274 replans,
275 "count",
276 MetricQuality.OBSERVED,
277 "Count of plan replacement events after the first accepted plan.",
278 _EVENT_STORE,
279 ),
280 "validation.non_passed": MetricValue(
281 "validation.non_passed",
282 failed_validations,
283 "count",
284 MetricQuality.DERIVED,
285 "Validation results whose normalized status is not PASSED.",
286 "validation records",
287 ),
288 "changesets.count": MetricValue(
289 "changesets.count",
290 len(changes),
291 "count",
292 MetricQuality.OBSERVED,
293 "Number of distinct persisted ChangeSet records.",
294 "ChangeSet records",
295 ),
296 "changeset.files": MetricValue(
297 "changeset.files",
298 files_changed,
299 "count",
300 MetricQuality.DERIVED,
301 "Unique paths appearing in persisted ChangeSets.",
302 "ChangeSet records",
303 ),
304 "human.decisions": MetricValue(
305 "human.decisions",
306 len(decisions),
307 "count",
308 MetricQuality.OBSERVED,
309 "Number of persisted human decisions.",
310 "decision records",
311 ),
312 "human.interactions": MetricValue(
313 "human.interactions",
314 sum(human_interactions(events).values()),
315 "count",
316 MetricQuality.OBSERVED,
317 "Events a person caused on the run: decisions, clarification answers, plan, "
318 "acceptance and preflight decisions, budget raises, quarantines, contract "
319 "confirmations and attached evidence.",
320 "run events with a human actor",
321 ),
322 "tokens.input": self._token_metric("tokens.input", input_tokens, "input_tokens"),
323 "tokens.output": self._token_metric("tokens.output", output_tokens, "output_tokens"),
324 "tokens.reasoning": self._token_metric(
325 "tokens.reasoning", reasoning_tokens, "reasoning_tokens"
326 ),
327 "cost.usd": MetricValue(
328 "cost.usd",
329 cost,
330 "USD",
331 MetricQuality.REPORTED if cost is not None else MetricQuality.NOT_AVAILABLE,
332 "Sum of provider-reported costs; never inferred from tokens.",
333 "resource usage records",
334 () if cost is not None else ("The provider did not report cost.",),
335 ),
336 }
337 return common
339 @staticmethod
340 def _sum_optional(records: list[ResourceUsage], field: str) -> int | float | None:
341 values = [
342 getattr(record, field) for record in records if getattr(record, field) is not None
343 ]
344 return sum(values) if values else None
346 @staticmethod
347 def _token_metric(name: str, value: int | float | None, field: str) -> MetricValue:
348 return MetricValue(
349 name,
350 value,
351 "tokens",
352 MetricQuality.REPORTED if value is not None else MetricQuality.NOT_AVAILABLE,
353 f"Sum of provider-reported {field}; no estimation is performed.",
354 "resource usage records",
355 () if value is not None else ("The provider did not expose this token category.",),
356 )