Coverage for src/governed_harness/telemetry/metrics.py: 99%

80 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1from __future__ import annotations 

2 

3from dataclasses import dataclass 

4from datetime import datetime 

5from typing import Any 

6 

7from governed_harness.domain.enums import MetricQuality, ResultStatus 

8from governed_harness.domain.models import ( 

9 AgentInvocation, 

10 ChangeSet, 

11 Finding, 

12 HumanDecision, 

13 PhaseExecution, 

14 ResourceUsage, 

15 ToolInvocation, 

16 ValidationResult, 

17) 

18from governed_harness.events.sqlite_store import StoredEvent 

19from governed_harness.storage.sqlite import SQLiteStateStore 

20 

21_EVENT_STORE = "event store" 

22 

23HUMAN_INTERACTION_EVENTS: dict[str, str] = { 

24 "human.decision.recorded": "decision", 

25 "intent.clarified": "clarification", 

26 "planning.decomposition.decided": "plan", 

27 "acceptance.tests.decided": "acceptance", 

28 "budget.raised": "budget", 

29 "workspace.quarantine.requested": "quarantine", 

30 "verification.preflight.decided": "preflight", 

31 "contract.confirmed": "contract", 

32 "evidence.attached": "evidence", 

33 "verification.deferred.closed": "evidence", 

34 "decision.preauthorized": "preauthorization", 

35 "plan.approval.decided": "plan", 

36} 

37"""Events a person causes on a run, by the kind of interruption they count as (#55; the 

38pre-authorised approval and the plan approval since #58).""" 

39 

40_SAME_INTERACTION = ("withAnswers", "withPreAuthorization") 

41"""Payload markers of an event recorded in the same act as another counted one.""" 

42 

43 

44def human_interactions(events: list[StoredEvent]) -> dict[str, int]: 

45 """The human interactions of a run by kind: events of the types above recorded with a 

46 human actor. A contract confirmed in the answers to a clarification (``withAnswers``) or with 

47 a pre-authorised approval (``withPreAuthorization``) is part of that one interaction and is 

48 not counted again; a decision the harness records from a pre-authorisation has the harness 

49 as its actor and is not counted either.""" 

50 counts: dict[str, int] = {} 

51 for event in events: 

52 kind = HUMAN_INTERACTION_EVENTS.get(event.event_type) 

53 if any(event.payload.get(marker) is True for marker in _SAME_INTERACTION): 

54 continue 

55 if kind is not None and event.actor.get("actorType") == "HUMAN": 

56 counts[kind] = counts.get(kind, 0) + 1 

57 return counts 

58 

59 

60@dataclass(frozen=True) 

61class MetricValue: 

62 name: str 

63 value: int | float | None 

64 unit: str 

65 quality: MetricQuality 

66 definition: str 

67 source: str 

68 limitations: tuple[str, ...] = () 

69 

70 def as_dict(self) -> dict[str, Any]: 

71 return { 

72 "name": self.name, 

73 "value": self.value, 

74 "unit": self.unit, 

75 "quality": self.quality, 

76 "definition": self.definition, 

77 "source": self.source, 

78 "limitations": list(self.limitations), 

79 } 

80 

81 

82class MetricsProjector: 

83 def __init__(self, state: SQLiteStateStore) -> None: 

84 self.state = state 

85 

86 def project(self, execution_id: str, events: list[StoredEvent]) -> dict[str, MetricValue]: 

87 phases = self.state.list("phase", PhaseExecution, execution_id=execution_id) 

88 agents = self.state.list("agent_invocation", AgentInvocation, execution_id=execution_id) 

89 tools = self.state.list("tool_invocation", ToolInvocation, execution_id=execution_id) 

90 validations = self.state.list("validation", ValidationResult, execution_id=execution_id) 

91 changes = self.state.list("change_set", ChangeSet, execution_id=execution_id) 

92 decisions = self.state.list("decision", HumanDecision, execution_id=execution_id) 

93 usages = self.state.list("resource_usage", ResourceUsage, execution_id=execution_id) 

94 timestamps = [datetime.fromisoformat(event.occurred_at) for event in events] 

95 total_ms = ( 

96 int((max(timestamps) - min(timestamps)).total_seconds() * 1000) 

97 if len(timestamps) >= 2 

98 else 0 

99 ) 

100 phase_ms = sum( 

101 int((phase.finished_at - phase.started_at).total_seconds() * 1000) 

102 for phase in phases 

103 if phase.started_at and phase.finished_at 

104 ) 

105 agent_ms = sum( 

106 int((item.finished_at - item.started_at).total_seconds() * 1000) for item in agents 

107 ) 

108 tool_ms = sum( 

109 int((item.finished_at - item.started_at).total_seconds() * 1000) for item in tools 

110 ) 

111 human_wait_ms: int | None = None 

112 gate_events = [event for event in events if event.event_type == "gate.evaluated"] 

113 decision_events = [ 

114 event for event in events if event.event_type == "human.decision.recorded" 

115 ] 

116 if gate_events and decision_events: 

117 decision_time = datetime.fromisoformat(decision_events[-1].occurred_at) 

118 eligible_gates = [ 

119 event 

120 for event in gate_events 

121 if datetime.fromisoformat(event.occurred_at) <= decision_time 

122 ] 

123 if eligible_gates: 123 ↛ 130line 123 didn't jump to line 130 because the condition on line 123 was always true

124 human_wait_ms = int( 

125 ( 

126 decision_time - datetime.fromisoformat(eligible_gates[-1].occurred_at) 

127 ).total_seconds() 

128 * 1000 

129 ) 

130 corrections = [event for event in events if event.event_type == "correction.authorized"] 

131 correction_cycles = len(corrections) 

132 verification_cycles = sum( 

133 1 for event in corrections if event.payload.get("trigger") == "VERIFICATION_FAILED" 

134 ) 

135 transient_retries = sum( 

136 1 for event in events if event.event_type == "agent.invocation.retried" 

137 ) 

138 unsupported_claims = sum( 

139 1 

140 for item in self.state.list("finding", Finding, execution_id=execution_id) 

141 if item.rule_id == "agent.unsupported-claim" 

142 ) 

143 implementation_attempts = sum( 

144 1 

145 for event in events 

146 if event.event_type == "phase.started" 

147 and event.payload.get("phaseId") == "IMPLEMENTATION" 

148 ) 

149 review_cycles = sum( 

150 1 

151 for event in events 

152 if event.event_type == "phase.started" 

153 and event.payload.get("phaseId") == "INDEPENDENT_REVIEW" 

154 ) 

155 replans = sum(1 for event in events if event.event_type == "plan.replaced") 

156 failed_validations = sum( 

157 1 for item in validations if item.status is not ResultStatus.PASSED 

158 ) 

159 files_changed = len({file.path for change in changes for file in change.files}) 

160 input_tokens = self._sum_optional(usages, "input_tokens") 

161 output_tokens = self._sum_optional(usages, "output_tokens") 

162 reasoning_tokens = self._sum_optional(usages, "reasoning_tokens") 

163 cost = self._sum_optional(usages, "cost_usd") 

164 common = { 

165 "duration.total_ms": MetricValue( 

166 "duration.total_ms", 

167 total_ms, 

168 "ms", 

169 MetricQuality.OBSERVED, 

170 "Wall-clock time between the first and last persisted execution event.", 

171 _EVENT_STORE, 

172 ), 

173 "duration.phase_ms": MetricValue( 

174 "duration.phase_ms", 

175 phase_ms, 

176 "ms", 

177 MetricQuality.DERIVED, 

178 "Sum of completed phase wall-clock durations; parallel phases would be double-counted.", 

179 "phase records", 

180 ("Not equal to total wall time when phases overlap.",), 

181 ), 

182 "duration.agent_ms": MetricValue( 

183 "duration.agent_ms", 

184 agent_ms, 

185 "ms", 

186 MetricQuality.DERIVED, 

187 "Sum of persisted agent invocation durations.", 

188 "agent invocation records", 

189 ), 

190 "duration.tool_ms": MetricValue( 

191 "duration.tool_ms", 

192 tool_ms, 

193 "ms", 

194 MetricQuality.DERIVED, 

195 "Sum of persisted tool invocation durations.", 

196 "tool invocation records", 

197 ), 

198 "duration.human_wait_ms": MetricValue( 

199 "duration.human_wait_ms", 

200 human_wait_ms, 

201 "ms", 

202 MetricQuality.DERIVED if human_wait_ms is not None else MetricQuality.NOT_AVAILABLE, 

203 "Elapsed time from the latest gate evaluation to the latest human decision.", 

204 "gate and decision events", 

205 () if human_wait_ms is not None else ("No completed human-decision interval.",), 

206 ), 

207 "agent.invocations": MetricValue( 

208 "agent.invocations", 

209 len(agents), 

210 "count", 

211 MetricQuality.OBSERVED, 

212 "Number of persisted agent invocation records.", 

213 "agent invocation records", 

214 ), 

215 "tool.invocations": MetricValue( 

216 "tool.invocations", 

217 len(tools), 

218 "count", 

219 MetricQuality.OBSERVED, 

220 "Number of persisted tool invocation records.", 

221 "tool invocation records", 

222 ), 

223 "implementation.attempts": MetricValue( 

224 "implementation.attempts", 

225 implementation_attempts, 

226 "count", 

227 MetricQuality.DERIVED, 

228 "Count of IMPLEMENTATION phase-start events.", 

229 _EVENT_STORE, 

230 ), 

231 "correction.cycles": MetricValue( 

232 "correction.cycles", 

233 correction_cycles, 

234 "count", 

235 MetricQuality.OBSERVED, 

236 "Count of authorized transitions back to IMPLEMENTATION: REQUEST_CHANGES " 

237 "decisions and automatic corrections after a failed VERIFICATION.", 

238 _EVENT_STORE, 

239 ), 

240 "correction.verification_cycles": MetricValue( 

241 "correction.verification_cycles", 

242 verification_cycles, 

243 "count", 

244 MetricQuality.OBSERVED, 

245 "Automatic corrections after a failed VERIFICATION (runtime.verificationCorrections).", 

246 _EVENT_STORE, 

247 ), 

248 "agent.transient_retries": MetricValue( 

249 "agent.transient_retries", 

250 transient_retries, 

251 "count", 

252 MetricQuality.OBSERVED, 

253 "Command-provider calls repeated after a transient failure (runtime.providerRetries).", 

254 _EVENT_STORE, 

255 ), 

256 "agent.unsupported_claims": MetricValue( 

257 "agent.unsupported_claims", 

258 unsupported_claims, 

259 "count", 

260 MetricQuality.DERIVED, 

261 "Findings agent.unsupported-claim: the agent reported success and verification failed.", 

262 "finding records", 

263 ), 

264 "review.cycles": MetricValue( 

265 "review.cycles", 

266 review_cycles, 

267 "count", 

268 MetricQuality.DERIVED, 

269 "Count of INDEPENDENT_REVIEW phase-start events.", 

270 _EVENT_STORE, 

271 ), 

272 "replanning.count": MetricValue( 

273 "replanning.count", 

274 replans, 

275 "count", 

276 MetricQuality.OBSERVED, 

277 "Count of plan replacement events after the first accepted plan.", 

278 _EVENT_STORE, 

279 ), 

280 "validation.non_passed": MetricValue( 

281 "validation.non_passed", 

282 failed_validations, 

283 "count", 

284 MetricQuality.DERIVED, 

285 "Validation results whose normalized status is not PASSED.", 

286 "validation records", 

287 ), 

288 "changesets.count": MetricValue( 

289 "changesets.count", 

290 len(changes), 

291 "count", 

292 MetricQuality.OBSERVED, 

293 "Number of distinct persisted ChangeSet records.", 

294 "ChangeSet records", 

295 ), 

296 "changeset.files": MetricValue( 

297 "changeset.files", 

298 files_changed, 

299 "count", 

300 MetricQuality.DERIVED, 

301 "Unique paths appearing in persisted ChangeSets.", 

302 "ChangeSet records", 

303 ), 

304 "human.decisions": MetricValue( 

305 "human.decisions", 

306 len(decisions), 

307 "count", 

308 MetricQuality.OBSERVED, 

309 "Number of persisted human decisions.", 

310 "decision records", 

311 ), 

312 "human.interactions": MetricValue( 

313 "human.interactions", 

314 sum(human_interactions(events).values()), 

315 "count", 

316 MetricQuality.OBSERVED, 

317 "Events a person caused on the run: decisions, clarification answers, plan, " 

318 "acceptance and preflight decisions, budget raises, quarantines, contract " 

319 "confirmations and attached evidence.", 

320 "run events with a human actor", 

321 ), 

322 "tokens.input": self._token_metric("tokens.input", input_tokens, "input_tokens"), 

323 "tokens.output": self._token_metric("tokens.output", output_tokens, "output_tokens"), 

324 "tokens.reasoning": self._token_metric( 

325 "tokens.reasoning", reasoning_tokens, "reasoning_tokens" 

326 ), 

327 "cost.usd": MetricValue( 

328 "cost.usd", 

329 cost, 

330 "USD", 

331 MetricQuality.REPORTED if cost is not None else MetricQuality.NOT_AVAILABLE, 

332 "Sum of provider-reported costs; never inferred from tokens.", 

333 "resource usage records", 

334 () if cost is not None else ("The provider did not report cost.",), 

335 ), 

336 } 

337 return common 

338 

339 @staticmethod 

340 def _sum_optional(records: list[ResourceUsage], field: str) -> int | float | None: 

341 values = [ 

342 getattr(record, field) for record in records if getattr(record, field) is not None 

343 ] 

344 return sum(values) if values else None 

345 

346 @staticmethod 

347 def _token_metric(name: str, value: int | float | None, field: str) -> MetricValue: 

348 return MetricValue( 

349 name, 

350 value, 

351 "tokens", 

352 MetricQuality.REPORTED if value is not None else MetricQuality.NOT_AVAILABLE, 

353 f"Sum of provider-reported {field}; no estimation is performed.", 

354 "resource usage records", 

355 () if value is not None else ("The provider did not expose this token category.",), 

356 )