Coverage for src/governed_harness/metrics/report.py: 85%

510 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""``harness metrics``: the runs of one repository, or of every repository of the run registry, 

2aggregated into one report (#58, items 8 to 10). Deterministic: the report is computed from the 

3records and the event chains only, with zero model calls. 

4 

5What it never contains: a person's name or any per-person indicator. Interactions, approvals 

6and human wait are counted per run, task and project (the thesis requirement RD-16 the 

7earlier ``scripts/metrics_report.py`` already kept); lines are attributed to an agent invocation 

8or to a person (out of band), never to whom.""" 

9 

10from __future__ import annotations 

11 

12import json 

13import re 

14import statistics 

15from collections import defaultdict 

16from collections.abc import Iterable, Mapping 

17from dataclasses import dataclass, field 

18from datetime import UTC, date, datetime, timedelta 

19from typing import Any 

20 

21from governed_harness import __version__ 

22from governed_harness.agents.routing import TaskSignals, classify_size 

23from governed_harness.configuration.friction import ( 

24 DEFAULT_FRICTION_TARGETS, 

25 FrictionTarget, 

26 ModelPrice, 

27) 

28from governed_harness.domain.enums import DecisionKind, FindingSeverity, ResultStatus 

29from governed_harness.domain.models import ( 

30 AgentInvocation, 

31 ChangeSet, 

32 ComponentProvenance, 

33 Execution, 

34 Finding, 

35 GateEvaluation, 

36 HumanDecision, 

37 PhaseExecution, 

38 ResourceUsage, 

39 Task, 

40) 

41from governed_harness.events.sqlite_store import SQLiteEventStore, StoredEvent 

42from governed_harness.metrics.prices import call_cost 

43from governed_harness.metrics.summary import period_status 

44from governed_harness.storage.sqlite import SQLiteStateStore 

45from governed_harness.telemetry.metrics import HUMAN_INTERACTION_EVENTS, human_interactions 

46 

47SCHEMA_VERSION = "1.0" 

48_ISSUE_TEXT = re.compile(r"\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+#(\d+)\b", re.IGNORECASE) 

49_APPROVALS = frozenset({DecisionKind.APPROVE, DecisionKind.APPROVE_EXCEPTION}) 

50_RELATIVE_UNITS = {"h": "hours", "d": "days", "w": "weeks"} 

51_CORRECTION = "correction.authorized" 

52# Events that record a limit being reached, with the key they get in quality.limitsReached. 

53_LIMIT_EVENTS = { 

54 "budget.exceeded": "budget", 

55 "phase.attempts.exhausted": "phaseAttempts", 

56 "correction.exhausted": "corrections", 

57} 

58_BLOCKING_SEVERITIES = frozenset({FindingSeverity.HIGH, FindingSeverity.CRITICAL}) 

59_FAILED_GATES = frozenset( 

60 {ResultStatus.FAILED, ResultStatus.ERROR, ResultStatus.TIMED_OUT, ResultStatus.INCONCLUSIVE} 

61) 

62_SEVERITIES = tuple(item.value for item in FindingSeverity) 

63_TOP_FILES = 20 

64_ESCALATED_LANE = "fast→full" 

65_PHASES = ( 

66 "INTENT", 

67 "DISCOVERY", 

68 "SPECIFICATION", 

69 "PLANNING", 

70 "IMPLEMENTATION", 

71 "VERIFICATION", 

72 "INDEPENDENT_REVIEW", 

73 "DECISION", 

74 "CLOSURE", 

75) 

76 

77 

78@dataclass 

79class ProjectSource: 

80 """The records of one project: its state and event stores and its settings.""" 

81 

82 project_id: str 

83 workspace: str 

84 state: SQLiteStateStore 

85 events: SQLiteEventStore 

86 targets: Mapping[Any, FrictionTarget] | None = None 

87 forge: dict[str, str] | None = None 

88 """``kind``, ``host`` and ``repository`` of the origin remote, for issue links.""" 

89 

90 

91@dataclass(frozen=True) 

92class Filters: 

93 since: datetime | None = None 

94 task: str | None = None 

95 model: str | None = None 

96 agent: str | None = None 

97 all_repos: bool = False 

98 

99 def as_dict(self) -> dict[str, Any]: 

100 return { 

101 "since": self.since.isoformat() if self.since else None, 

102 "task": self.task, 

103 "model": self.model, 

104 "agent": self.agent, 

105 "allRepos": self.all_repos, 

106 } 

107 

108 

109def parse_since(value: str | None, now: datetime | None = None) -> datetime | None: 

110 """``YYYY-MM-DD``, an ISO 8601 instant, or a relative ``7d``/``2w``/``12h``.""" 

111 if not value: 

112 return None 

113 text = value.strip() 

114 # A plain parse instead of a regular expression: linear on any input (``--since`` comes 

115 # from the command line). ``isdecimal`` accepts exactly what ``\d`` accepted. 

116 unit = _RELATIVE_UNITS.get(text[-1:]) 

117 if unit is not None and text[:-1].isdecimal(): 

118 return (now or datetime.now(UTC)) - timedelta(**{unit: int(text[:-1])}) 

119 try: 

120 parsed = datetime.fromisoformat(text) 

121 except ValueError: 

122 try: 

123 parsed = datetime.combine(date.fromisoformat(text), datetime.min.time()) 

124 except ValueError as error: 

125 raise ValueError( 

126 f"--since takes YYYY-MM-DD, an ISO 8601 instant or 12h, 7d, 2w: {value!r}" 

127 ) from error 

128 return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) 

129 

130 

131@dataclass 

132class _Call: 

133 invocation: AgentInvocation 

134 usage: dict[str, Any] | None 

135 reported: float | None 

136 estimated: float | None 

137 cost_source: str 

138 

139 @property 

140 def tokens(self) -> dict[str, int]: 

141 usage = self.usage or {} 

142 return { 

143 name: int(usage.get(field) or 0) 

144 for name, field in ( 

145 ("input", "input_tokens"), 

146 ("output", "output_tokens"), 

147 ("reasoning", "reasoning_tokens"), 

148 ("cache", "cache_tokens"), 

149 ("cacheRead", "cache_read_tokens"), 

150 ) 

151 } 

152 

153 @property 

154 def duration_ms(self) -> int: 

155 delta = self.invocation.finished_at - self.invocation.started_at 

156 return max(0, int(delta.total_seconds() * 1000)) 

157 

158 

159@dataclass 

160class _Run: 

161 project_id: str 

162 execution: Execution 

163 task: Task | None 

164 events: list[StoredEvent] 

165 calls: list[_Call] 

166 phases: list[PhaseExecution] 

167 decisions: list[HumanDecision] 

168 change_set: ChangeSet | None 

169 provenance: ComponentProvenance | None 

170 findings: list[Finding] 

171 lane: dict[str, Any] | None 

172 size: str 

173 issues: list[dict[str, Any]] = field(default_factory=list) 

174 pull_request: dict[str, Any] | None = None 

175 latest_gate: ResultStatus | None = None 

176 

177 @property 

178 def run_id(self) -> str: 

179 return self.execution.execution_id 

180 

181 def times(self) -> tuple[datetime, datetime]: 

182 stamps = [datetime.fromisoformat(event.occurred_at) for event in self.events] 

183 if not stamps: 183 ↛ 184line 183 didn't jump to line 184 because the condition on line 183 was never true

184 return self.execution.created_at, self.execution.updated_at 

185 return min(stamps), max(stamps) 

186 

187 

188def _flag_json(state: SQLiteStateStore, key: str) -> Any: 

189 raw = state.get_flag(key) 

190 try: 

191 return json.loads(raw) if raw else None 

192 except ValueError: 

193 return None 

194 

195 

196def _size(state: SQLiteStateStore, execution: Execution, task: Task | None) -> str: 

197 """The lane's size when the run recorded one, else the router's classification of the 

198 task's requirements and owned paths (lines are not read: the workspace may be elsewhere).""" 

199 lane = _flag_json(state, f"lane:{execution.execution_id}") 

200 if isinstance(lane, dict) and lane.get("size") in {"S", "M", "L"}: 200 ↛ 201line 200 didn't jump to line 201 because the condition on line 200 was never true

201 return str(lane["size"]) 

202 if task is None: 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true

203 return "S" 

204 owned = task.metadata.get("ownedPaths") or [patch.path for patch in task.implementation.patches] 

205 signals = TaskSignals( 

206 requirements=len(task.requirements) or len(task.acceptance_criteria), 

207 files=len(list(owned)), 

208 loc=0, 

209 ) 

210 return classify_size(signals, None)[0] 

211 

212 

213def _issue_link(forge: Mapping[str, str] | None, number: str) -> str | None: 

214 if not forge or not forge.get("host") or not forge.get("repository"): 214 ↛ 216line 214 didn't jump to line 216 because the condition on line 214 was always true

215 return None 

216 base = f"https://{forge['host']}/{forge['repository']}" 

217 kind = forge.get("kind") 

218 if kind == "gitlab": 

219 return f"{base}/-/issues/{number}" 

220 if kind == "azure-devops": 

221 parts = forge["repository"].split("/") 

222 return ( 

223 f"https://{forge['host']}/{parts[0]}/{parts[1]}/_workitems/edit/{number}" 

224 if len(parts) >= 2 

225 else None 

226 ) 

227 return f"{base}/issues/{number}" 

228 

229 

230def _pull_link(forge: Mapping[str, str] | None, number: Any) -> str | None: 

231 if not forge or not isinstance(number, int): 

232 return None 

233 base = f"https://{forge.get('host')}/{forge.get('repository')}" 

234 kind = forge.get("kind") 

235 if kind == "gitlab": 

236 return f"{base}/-/merge_requests/{number}" 

237 if kind == "bitbucket": 

238 return f"{base}/pull-requests/{number}" 

239 if kind == "azure-devops": 

240 return None 

241 return f"{base}/pull/{number}" 

242 

243 

244def _as_list(value: Any) -> list[Any]: 

245 if isinstance(value, list): 

246 return value 

247 return [] if value is None else [value] 

248 

249 

250def task_issues(task: Task) -> list[str]: 

251 """Issue numbers a task names: ``metadata.issues`` / ``metadata.issue`` and closing 

252 keywords (``Closes #12``) in its title and intent.""" 

253 found: list[str] = [] 

254 for key in ("issues", "issue"): 

255 for item in _as_list(task.metadata.get(key)): 

256 text = str(item).strip().lstrip("#") 

257 if text.isdigit(): 257 ↛ 255line 257 didn't jump to line 255 because the condition on line 257 was always true

258 found.append(text) 

259 for text in (task.title, task.intent): 

260 found.extend(_ISSUE_TEXT.findall(text)) 

261 return list(dict.fromkeys(found)) 

262 

263 

264def _selected(execution: Execution, filters: Filters) -> bool: 

265 if filters.task and execution.task_id != filters.task: 

266 return False 

267 return not (filters.since and execution.created_at < filters.since) 

268 

269 

270def _load_task(state: SQLiteStateStore, execution: Execution) -> Task | None: 

271 try: 

272 return state.get("task", execution.task_id, Task) 

273 except Exception: # noqa: BLE001 - a missing task still counts the run 

274 return None 

275 

276 

277def _calls(state: SQLiteStateStore, run_id: str, filters: Filters) -> list[_Call]: 

278 """The agent calls of a run that pass the ``model`` and ``agent`` filters.""" 

279 invocations = state.list("agent_invocation", AgentInvocation, execution_id=run_id) 

280 usages = { 

281 item.invocation_id: item.model_dump(mode="json") 

282 for item in state.list("resource_usage", ResourceUsage, execution_id=run_id) 

283 if item.invocation_id 

284 } 

285 return [ 

286 _Call(invocation, usages.get(invocation.invocation_id), None, None, "none") 

287 for invocation in invocations 

288 if not (filters.model and invocation.model != filters.model) 

289 and not (filters.agent and invocation.provider != filters.agent) 

290 ] 

291 

292 

293def _latest_gate(state: SQLiteStateStore, execution: Execution) -> ResultStatus | None: 

294 """The status of the gate evaluation the run points at, if any.""" 

295 if not execution.gate_evaluation_id: 295 ↛ 296line 295 didn't jump to line 296 because the condition on line 295 was never true

296 return None 

297 try: 

298 return state.get("gate", execution.gate_evaluation_id, GateEvaluation).status 

299 except Exception: # noqa: BLE001 - a missing gate record only leaves the status unknown 

300 return None 

301 

302 

303def _change_set(state: SQLiteStateStore, execution: Execution) -> ChangeSet | None: 

304 """The run's current change set, else its latest one.""" 

305 changes = state.list("change_set", ChangeSet, execution_id=execution.execution_id) 

306 current = [item for item in changes if item.digest == execution.change_set_digest] 

307 ordered = current or sorted(changes, key=lambda item: item.created_at) 

308 return ordered[-1] if ordered else None 

309 

310 

311def _provenance( 

312 state: SQLiteStateStore, run_id: str, change_set: ChangeSet | None 

313) -> ComponentProvenance | None: 

314 if change_set is None: 314 ↛ 315line 314 didn't jump to line 315 because the condition on line 314 was never true

315 return None 

316 provenances = [ 

317 item 

318 for item in state.list("component_provenance", ComponentProvenance, execution_id=run_id) 

319 if item.change_set_digest == change_set.digest 

320 ] 

321 return provenances[-1] if provenances else None 

322 

323 

324def _pull_request( 

325 events: list[StoredEvent], forge: Mapping[str, str] | None 

326) -> dict[str, Any] | None: 

327 """The last pull request the run's delivery opened, with a link to it.""" 

328 pull: dict[str, Any] | None = None 

329 for event in events: 

330 if event.event_type != "delivery.pull-request.created": 330 ↛ 332line 330 didn't jump to line 332 because the condition on line 330 was always true

331 continue 

332 payload = event.payload 

333 number = payload.get("number") 

334 event_forge = ( 

335 { 

336 "kind": str(payload.get("forge") or ""), 

337 "host": str(payload.get("host") or ""), 

338 "repository": str(payload.get("repository") or ""), 

339 } 

340 if payload.get("host") 

341 else forge 

342 ) 

343 pull = { 

344 "number": number, 

345 "forge": payload.get("forge"), 

346 "repository": payload.get("repository"), 

347 "url": payload.get("url") or _pull_link(event_forge, number), 

348 } 

349 return pull 

350 

351 

352def _run(source: ProjectSource, execution: Execution, filters: Filters) -> _Run | None: 

353 """The records of one run, or ``None`` when the ``model``/``agent`` filters leave it no 

354 call.""" 

355 state = source.state 

356 task = _load_task(state, execution) 

357 run_id = execution.execution_id 

358 calls = _calls(state, run_id, filters) 

359 if (filters.model or filters.agent) and not calls: 

360 return None 

361 change_set = _change_set(state, execution) 

362 provenance = _provenance(state, run_id, change_set) 

363 events = source.events.list(run_id) 

364 lane = _flag_json(state, f"lane:{run_id}") 

365 run = _Run( 

366 project_id=source.project_id, 

367 execution=execution, 

368 task=task, 

369 events=events, 

370 calls=calls, 

371 phases=state.list("phase", PhaseExecution, execution_id=run_id), 

372 decisions=sorted( 

373 state.list("decision", HumanDecision, execution_id=run_id), 

374 key=lambda item: item.decided_at, 

375 ), 

376 change_set=change_set, 

377 provenance=provenance, 

378 findings=state.list("finding", Finding, execution_id=run_id), 

379 lane=lane if isinstance(lane, dict) else None, 

380 size=_size(state, execution, task), 

381 pull_request=_pull_request(events, source.forge), 

382 latest_gate=_latest_gate(state, execution), 

383 ) 

384 if task is not None: 384 ↛ 389line 384 didn't jump to line 389 because the condition on line 384 was always true

385 run.issues = [ 

386 {"issue": f"#{number}", "url": _issue_link(source.forge, number)} 

387 for number in task_issues(task) 

388 ] 

389 return run 

390 

391 

392def collect(sources: Iterable[ProjectSource], filters: Filters) -> list[_Run]: 

393 runs: list[_Run] = [] 

394 for source in sources: 

395 executions = source.state.list("execution", Execution, project_id=source.project_id) 

396 for execution in executions: 

397 if not _selected(execution, filters): 

398 continue 

399 run = _run(source, execution, filters) 

400 if run is not None: 

401 runs.append(run) 

402 return sorted(runs, key=lambda item: item.execution.created_at) 

403 

404 

405# ----- aggregation ------------------------------------------------------------------------------ 

406def _token_row(key: str) -> dict[str, Any]: 

407 return { 

408 "key": key, 

409 "calls": 0, 

410 "input": 0, 

411 "output": 0, 

412 "reasoning": 0, 

413 "cache": 0, 

414 "cacheRead": 0, 

415 "reportedCostUsd": 0.0, 

416 "estimatedCostUsd": 0.0, 

417 "unpricedCalls": 0, 

418 "durationMs": 0, 

419 } 

420 

421 

422def _add_call(row: dict[str, Any], call: _Call) -> None: 

423 row["calls"] += 1 

424 for name, value in call.tokens.items(): 

425 row[name] += value 

426 row["reportedCostUsd"] += call.reported or 0.0 

427 row["estimatedCostUsd"] += call.estimated or 0.0 

428 row["unpricedCalls"] += 1 if call.cost_source == "unpriced" else 0 

429 row["durationMs"] += call.duration_ms 

430 

431 

432def _finish_rows(rows: Mapping[str, dict[str, Any]]) -> list[dict[str, Any]]: 

433 finished = [] 

434 for row in rows.values(): 

435 row["reportedCostUsd"] = round(row["reportedCostUsd"], 6) 

436 row["estimatedCostUsd"] = round(row["estimatedCostUsd"], 6) 

437 row["costUsd"] = round(row["reportedCostUsd"] + row["estimatedCostUsd"], 6) 

438 row["tokens"] = row["input"] + row["output"] 

439 finished.append(row) 

440 return sorted(finished, key=lambda item: (-item["tokens"], item["key"])) 

441 

442 

443def _human_wait_ms(events: list[StoredEvent]) -> int: 

444 """Time the run waited for a person: from each phase that ended BLOCKED to the next event 

445 a person caused or the next phase that started, whichever came first.""" 

446 total = 0 

447 waiting_since: datetime | None = None 

448 for event in events: 

449 moment = datetime.fromisoformat(event.occurred_at) 

450 if ( 

451 event.event_type == "phase.completed" 

452 and event.payload.get("status") == ResultStatus.BLOCKED.value 

453 ): 

454 waiting_since = waiting_since or moment 

455 continue 

456 if waiting_since is None: 

457 continue 

458 human = ( 

459 event.event_type in HUMAN_INTERACTION_EVENTS and event.actor.get("actorType") == "HUMAN" 

460 ) 

461 if human or event.event_type == "phase.started": 461 ↛ 448line 461 didn't jump to line 448 because the condition on line 461 was always true

462 total += max(0, int((moment - waiting_since).total_seconds() * 1000)) 

463 waiting_since = None 

464 return total 

465 

466 

467def _percentile(values: list[int], fraction: float) -> int | None: 

468 if not values: 

469 return None 

470 ordered = sorted(values) 

471 index = min(len(ordered) - 1, max(0, round(fraction * (len(ordered) - 1)))) 

472 return ordered[index] 

473 

474 

475def _price_calls(runs: list[_Run], prices: Mapping[str, ModelPrice]) -> None: 

476 for run in runs: 

477 for call in run.calls: 

478 priced = call_cost(call.usage, prices, call.invocation.provider, call.invocation.model) 

479 call.reported, call.estimated, call.cost_source = ( 

480 priced.reported, 

481 priced.estimated, 

482 priced.source, 

483 ) 

484 

485 

486def _lane_label(lane: dict[str, Any] | None) -> Any: 

487 """``fast→full`` for a lane that escalated, else the lane the run recorded.""" 

488 if not lane: 488 ↛ 490line 488 didn't jump to line 490 because the condition on line 488 was always true

489 return None 

490 return _ESCALATED_LANE if lane.get("escalated") else lane.get("lane") 

491 

492 

493def _cost_label(sources: set[str]) -> str: 

494 """How a model's costs are known: all reported, some estimated, unpriced or none.""" 

495 if sources == {"reported"}: 495 ↛ 496line 495 didn't jump to line 496 because the condition on line 495 was never true

496 return "reported" 

497 for label in ("estimated", "unpriced"): 

498 if label in sources: 

499 return label 

500 return "none" 

501 

502 

503def _phase_ms(phase: PhaseExecution) -> int | None: 

504 if not (phase.started_at and phase.finished_at): 504 ↛ 505line 504 didn't jump to line 505 because the condition on line 504 was never true

505 return None 

506 return int((phase.finished_at - phase.started_at).total_seconds() * 1000) 

507 

508 

509@dataclass 

510class _TokenTables: 

511 """Token, cost and duration rows of the agent calls, per dimension and in total.""" 

512 

513 by: dict[str, dict[str, dict[str, Any]]] = field( 

514 default_factory=lambda: { 

515 name: {} for name in ("agent", "model", "task", "phase", "callKind", "project") 

516 } 

517 ) 

518 totals: dict[str, Any] = field(default_factory=lambda: _token_row("total")) 

519 models: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict) 

520 durations: list[int] = field(default_factory=list) 

521 

522 def add(self, run: _Run) -> None: 

523 for call in run.calls: 

524 invocation = call.invocation 

525 model = invocation.model or "unknown" 

526 keys = { 

527 "agent": invocation.provider, 

528 "model": model, 

529 "task": run.execution.task_id, 

530 "phase": invocation.phase_id.value, 

531 "callKind": invocation.call_kind or "implement", 

532 "project": run.project_id, 

533 } 

534 for name, value in keys.items(): 

535 _add_call(self.by[name].setdefault(value, _token_row(value)), call) 

536 _add_call(self.totals, call) 

537 self.durations.append(call.duration_ms) 

538 model_entry = self.models.setdefault( 

539 (invocation.provider, model), 

540 {"provider": invocation.provider, "model": model, "calls": 0, "costSources": set()}, 

541 ) 

542 model_entry["calls"] += 1 

543 model_entry["costSources"].add(call.cost_source) 

544 

545 def model_rows(self) -> list[dict[str, Any]]: 

546 return [ 

547 { 

548 "provider": entry["provider"], 

549 "model": entry["model"], 

550 "calls": entry["calls"], 

551 "cost": _cost_label(entry["costSources"]), 

552 } 

553 for entry in sorted( 

554 self.models.values(), key=lambda item: (-item["calls"], item["model"]) 

555 ) 

556 ] 

557 

558 

559@dataclass(frozen=True) 

560class _RunTotals: 

561 """What one run counts: its span, the people's part and the agents' tokens and cost.""" 

562 

563 started: datetime 

564 finished: datetime 

565 interactions: int 

566 approvals: int 

567 wait_ms: int 

568 corrections: int 

569 closed: bool 

570 tokens: int 

571 cost: float 

572 

573 @classmethod 

574 def of(cls, run: _Run) -> _RunTotals: 

575 started, finished = run.times() 

576 return cls( 

577 started=started, 

578 finished=finished, 

579 interactions=sum(human_interactions(run.events).values()), 

580 approvals=sum(1 for item in run.decisions if item.decision in _APPROVALS), 

581 wait_ms=_human_wait_ms(run.events), 

582 corrections=sum(1 for event in run.events if event.event_type == _CORRECTION), 

583 closed=run.execution.status is ResultStatus.PASSED, 

584 tokens=sum(call.tokens["input"] + call.tokens["output"] for call in run.calls), 

585 cost=sum((call.reported or 0.0) + (call.estimated or 0.0) for call in run.calls), 

586 ) 

587 

588 

589def _empty_lines() -> dict[str, Any]: 

590 return { 

591 "added": 0, 

592 "removed": 0, 

593 "agent": {"added": 0, "removed": 0, "files": 0}, 

594 "person": {"added": 0, "removed": 0, "files": 0}, 

595 "unattributed": {"added": 0, "removed": 0, "files": 0}, 

596 } 

597 

598 

599def _empty_quality(runs: int) -> dict[str, int]: 

600 return { 

601 "runs": runs, 

602 "decided": 0, 

603 "approvedFirstTime": 0, 

604 "corrections": 0, 

605 "requestedChanges": 0, 

606 "blockedRuns": 0, 

607 "waitingDecision": 0, 

608 "rejectedRuns": 0, 

609 "closedRuns": 0, 

610 "preAuthorizedApprovals": 0, 

611 } 

612 

613 

614_AUTHORS = {"AGENT": "agent", "OUT_OF_BAND": "person"} 

615 

616 

617@dataclass 

618class _Aggregate: 

619 """The per-run sections of the report, filled one run at a time.""" 

620 

621 quality: dict[str, int] 

622 lines: dict[str, Any] = field(default_factory=_empty_lines) 

623 issues: list[dict[str, Any]] = field(default_factory=list) 

624 features: list[dict[str, Any]] = field(default_factory=list) 

625 tasks: dict[str, dict[str, Any]] = field(default_factory=dict) 

626 phase_times: dict[str, list[int]] = field(default_factory=lambda: defaultdict(list)) 

627 findings_by_rule: dict[str, dict[str, Any]] = field(default_factory=dict) 

628 findings_by_severity: dict[str, int] = field( 

629 default_factory=lambda: dict.fromkeys(_SEVERITIES, 0) 

630 ) 

631 findings_by_file: dict[str, dict[str, Any]] = field(default_factory=dict) 

632 blocked_detail: list[dict[str, Any]] = field(default_factory=list) 

633 limits: dict[str, int] = field(default_factory=lambda: dict.fromkeys(_LIMIT_EVENTS.values(), 0)) 

634 gate_results: dict[str, int] = field(default_factory=dict) 

635 open_blocking: int = 0 

636 open_failed_gates: int = 0 

637 run_rows: list[dict[str, Any]] = field(default_factory=list) 

638 daily: dict[str, dict[str, Any]] = field(default_factory=dict) 

639 weekly: dict[str, dict[str, Any]] = field(default_factory=dict) 

640 

641 def add(self, run: _Run) -> None: 

642 totals = _RunTotals.of(run) 

643 title = run.task.title if run.task else run.execution.task_id 

644 self._add_quality(run, totals) 

645 self._add_findings(run, totals.closed) 

646 self._add_limits_and_gates(run, totals.closed) 

647 for phase in run.phases: 

648 elapsed = _phase_ms(phase) 

649 if elapsed is not None: 649 ↛ 647line 649 didn't jump to line 647 because the condition on line 649 was always true

650 self.phase_times[phase.phase_id.value].append(elapsed) 

651 added, removed = self._add_lines(run) 

652 if totals.closed: 652 ↛ 654line 652 didn't jump to line 654 because the condition on line 652 was always true

653 self._add_delivered(run, title, totals.finished.isoformat()) 

654 self._add_task(run, title, totals) 

655 self._add_run_row(run, totals, added, removed) 

656 self._add_trends(run, totals) 

657 

658 def _add_quality(self, run: _Run, totals: _RunTotals) -> None: 

659 quality = self.quality 

660 execution = run.execution 

661 quality["corrections"] += totals.corrections 

662 quality["requestedChanges"] += sum( 

663 1 for item in run.decisions if item.decision is DecisionKind.REQUEST_CHANGES 

664 ) 

665 if run.decisions: 665 ↛ 669line 665 didn't jump to line 669 because the condition on line 665 was always true

666 quality["decided"] += 1 

667 if _approved_first_time(run): 667 ↛ 669line 667 didn't jump to line 669 because the condition on line 667 was always true

668 quality["approvedFirstTime"] += 1 

669 quality["preAuthorizedApprovals"] += sum( 

670 1 for item in run.decisions if item.pre_authorization_id 

671 ) 

672 if execution.status is ResultStatus.BLOCKED: 672 ↛ 673line 672 didn't jump to line 673 because the condition on line 672 was never true

673 waiting = execution.current_phase.value == "DECISION" 

674 quality["waitingDecision" if waiting else "blockedRuns"] += 1 

675 if any(item.decision is DecisionKind.REJECT for item in run.decisions): 675 ↛ 676line 675 didn't jump to line 676 because the condition on line 675 was never true

676 quality["rejectedRuns"] += 1 

677 if totals.closed: 677 ↛ exitline 677 didn't return from function '_add_quality' because the condition on line 677 was always true

678 quality["closedRuns"] += 1 

679 

680 def _add_findings(self, run: _Run, closed: bool) -> None: 

681 for finding in run.findings: 

682 row = self.findings_by_rule.setdefault( 

683 finding.rule_id, {"ruleId": finding.rule_id, "count": 0, "bySeverity": {}} 

684 ) 

685 row["count"] += 1 

686 severity = finding.severity.value 

687 row["bySeverity"][severity] = row["bySeverity"].get(severity, 0) + 1 

688 self.findings_by_severity[severity] += 1 

689 if finding.severity in _BLOCKING_SEVERITIES and not closed: 689 ↛ 690line 689 didn't jump to line 690 because the condition on line 689 was never true

690 self.open_blocking += 1 

691 path = finding.location.path if finding.location else None 

692 if path: 

693 entry = self.findings_by_file.setdefault( 

694 path, {"path": path, "count": 0, "bySeverity": {}} 

695 ) 

696 entry["count"] += 1 

697 entry["bySeverity"][severity] = entry["bySeverity"].get(severity, 0) + 1 

698 

699 def _add_limits_and_gates(self, run: _Run, closed: bool) -> None: 

700 """Limits reached (events), the latest gate result of the run and, for a run that 

701 stopped, the reason it recorded.""" 

702 for event in run.events: 

703 key = _LIMIT_EVENTS.get(event.event_type) 

704 if key: 704 ↛ 705line 704 didn't jump to line 705 because the condition on line 704 was never true

705 self.limits[key] += 1 

706 if run.latest_gate is not None: 706 ↛ 711line 706 didn't jump to line 711 because the condition on line 706 was always true

707 name = run.latest_gate.value 

708 self.gate_results[name] = self.gate_results.get(name, 0) + 1 

709 if not closed and run.latest_gate in _FAILED_GATES: 709 ↛ 710line 709 didn't jump to line 710 because the condition on line 709 was never true

710 self.open_failed_gates += 1 

711 execution = run.execution 

712 if execution.status is ResultStatus.BLOCKED and execution.current_phase.value != "DECISION": 712 ↛ 713line 712 didn't jump to line 713 because the condition on line 712 was never true

713 self.blocked_detail.append( 

714 { 

715 "projectId": run.project_id, 

716 "runId": run.run_id, 

717 "taskId": execution.task_id, 

718 "phase": execution.current_phase.value, 

719 "reason": execution.terminal_reason, 

720 } 

721 ) 

722 

723 def _add_lines(self, run: _Run) -> tuple[int, int]: 

724 """Adds the run's changed lines, attributed to an agent, a person or neither.""" 

725 added = removed = 0 

726 if run.change_set is not None: 726 ↛ 737line 726 didn't jump to line 737 because the condition on line 726 was always true

727 sources = ( 

728 {item.path: item.source for item in run.provenance.files} if run.provenance else {} 

729 ) 

730 for changed in run.change_set.files: 

731 added += changed.additions 

732 removed += changed.deletions 

733 author = self.lines[_AUTHORS.get(sources.get(changed.path) or "", "unattributed")] 

734 author["added"] += changed.additions 

735 author["removed"] += changed.deletions 

736 author["files"] += 1 

737 self.lines["added"] += added 

738 self.lines["removed"] += removed 

739 return added, removed 

740 

741 def _add_delivered(self, run: _Run, title: str, closed_at: str) -> None: 

742 task_id = run.execution.task_id 

743 self.features.append( 

744 { 

745 "projectId": run.project_id, 

746 "taskId": task_id, 

747 "title": title, 

748 "runId": run.run_id, 

749 "closedAt": closed_at, 

750 "issues": run.issues, 

751 "pullRequest": run.pull_request, 

752 } 

753 ) 

754 for issue in run.issues: 

755 self.issues.append( 

756 { 

757 **issue, 

758 "projectId": run.project_id, 

759 "taskId": task_id, 

760 "runId": run.run_id, 

761 "closedAt": closed_at, 

762 } 

763 ) 

764 

765 def _add_task(self, run: _Run, title: str, totals: _RunTotals) -> None: 

766 task_id = run.execution.task_id 

767 entry = self.tasks.setdefault( 

768 f"{run.project_id}:{task_id}", 

769 { 

770 "projectId": run.project_id, 

771 "taskId": task_id, 

772 "title": title, 

773 "size": run.size, 

774 "lanes": [], 

775 "runs": 0, 

776 "interactions": 0, 

777 "approvals": 0, 

778 "corrections": 0, 

779 "humanWaitMs": 0, 

780 "agentMs": 0, 

781 "tokens": 0, 

782 "costUsd": 0.0, 

783 "start": totals.started, 

784 "end": totals.finished, 

785 "phasesMs": {}, 

786 }, 

787 ) 

788 entry["runs"] += 1 

789 entry["interactions"] += totals.interactions 

790 entry["approvals"] += totals.approvals 

791 entry["corrections"] += totals.corrections 

792 entry["humanWaitMs"] += totals.wait_ms 

793 entry["agentMs"] += sum(call.duration_ms for call in run.calls) 

794 entry["tokens"] += totals.tokens 

795 entry["costUsd"] += totals.cost 

796 entry["start"] = min(entry["start"], totals.started) 

797 entry["end"] = max(entry["end"], totals.finished) 

798 if run.lane and run.lane.get("lane"): 798 ↛ 799line 798 didn't jump to line 799 because the condition on line 798 was never true

799 entry["lanes"].append(str(_lane_label(run.lane))) 

800 for phase in run.phases: 

801 elapsed = _phase_ms(phase) 

802 if elapsed is not None: 802 ↛ 800line 802 didn't jump to line 800 because the condition on line 802 was always true

803 phase_key = phase.phase_id.value 

804 entry["phasesMs"][phase_key] = entry["phasesMs"].get(phase_key, 0) + elapsed 

805 

806 def _add_run_row(self, run: _Run, totals: _RunTotals, added: int, removed: int) -> None: 

807 execution = run.execution 

808 self.run_rows.append( 

809 { 

810 "projectId": run.project_id, 

811 "runId": run.run_id, 

812 "taskId": execution.task_id, 

813 "status": execution.status.value, 

814 "phase": execution.current_phase.value, 

815 "size": run.size, 

816 "lane": _lane_label(run.lane), 

817 "createdAt": execution.created_at.isoformat(), 

818 "wallMs": int((totals.finished - totals.started).total_seconds() * 1000), 

819 "humanWaitMs": totals.wait_ms, 

820 "agentCalls": len(run.calls), 

821 "inputTokens": sum(call.tokens["input"] for call in run.calls), 

822 "outputTokens": sum(call.tokens["output"] for call in run.calls), 

823 "cacheTokens": sum(call.tokens["cache"] for call in run.calls), 

824 "reportedCostUsd": round(sum(call.reported or 0.0 for call in run.calls), 6), 

825 "estimatedCostUsd": round(sum(call.estimated or 0.0 for call in run.calls), 6), 

826 "interactions": totals.interactions, 

827 "approvals": totals.approvals, 

828 "corrections": totals.corrections, 

829 "linesAdded": added, 

830 "linesRemoved": removed, 

831 } 

832 ) 

833 

834 def _add_trends(self, run: _Run, totals: _RunTotals) -> None: 

835 day = run.execution.created_at.astimezone(UTC).date() 

836 iso = day.isocalendar() 

837 for bucket_map, period in ( 

838 (self.daily, day.isoformat()), 

839 (self.weekly, f"{iso.year}-W{iso.week:02d}"), 

840 ): 

841 bucket = bucket_map.setdefault( 

842 period, 

843 { 

844 "period": period, 

845 "runs": 0, 

846 "tokens": 0, 

847 "costUsd": 0.0, 

848 "interactions": 0, 

849 "approved": 0, 

850 "closed": 0, 

851 }, 

852 ) 

853 bucket["runs"] += 1 

854 bucket["tokens"] += totals.tokens 

855 bucket["costUsd"] = round(bucket["costUsd"] + totals.cost, 6) 

856 bucket["interactions"] += totals.interactions 

857 bucket["approved"] += 1 if totals.approvals else 0 

858 bucket["closed"] += 1 if totals.closed else 0 

859 

860 

861def _approved_first_time(run: _Run) -> bool: 

862 """The first decision approved and no correction was authorized before it.""" 

863 first = run.decisions[0] 

864 if first.decision not in _APPROVALS: 864 ↛ 865line 864 didn't jump to line 865 because the condition on line 864 was never true

865 return False 

866 return not any( 

867 event.event_type == _CORRECTION 

868 and datetime.fromisoformat(event.occurred_at) <= first.decided_at 

869 for event in run.events 

870 ) 

871 

872 

873def _over_target(target: FrictionTarget | None, interactions: int, minutes: float) -> list[str]: 

874 over: list[str] = [] 

875 if target is None: 875 ↛ 876line 875 didn't jump to line 876 because the condition on line 875 was never true

876 return over 

877 if target.interactions is not None and interactions > target.interactions: 877 ↛ 878line 877 didn't jump to line 878 because the condition on line 877 was never true

878 over.append("interactions") 

879 if target.minutes is not None and minutes > target.minutes: 879 ↛ 880line 879 didn't jump to line 880 because the condition on line 879 was never true

880 over.append("minutes") 

881 return over 

882 

883 

884def _friction( 

885 tasks: dict[str, dict[str, Any]], targets: Mapping[Any, FrictionTarget] 

886) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: 

887 """One friction row per task against its size's target, and the means per size.""" 

888 friction_rows = [] 

889 by_size: dict[str, dict[str, Any]] = {} 

890 for entry in tasks.values(): 

891 wall_minutes = round((entry["end"] - entry["start"]).total_seconds() / 60, 2) 

892 target = targets.get(entry["size"]) 

893 over = _over_target(target, entry["interactions"], wall_minutes) 

894 friction_rows.append( 

895 { 

896 "projectId": entry["projectId"], 

897 "taskId": entry["taskId"], 

898 "title": entry["title"], 

899 "size": entry["size"], 

900 "lanes": entry["lanes"], 

901 "runs": entry["runs"], 

902 "interactions": entry["interactions"], 

903 "approvals": entry["approvals"], 

904 "corrections": entry["corrections"], 

905 "wallMinutes": wall_minutes, 

906 "humanWaitMinutes": round(entry["humanWaitMs"] / 60000, 2), 

907 "agentMinutes": round(entry["agentMs"] / 60000, 2), 

908 "tokens": entry["tokens"], 

909 "costUsd": round(entry["costUsd"], 6), 

910 "phasesMs": entry["phasesMs"], 

911 "target": target.model_dump(mode="json", by_alias=True) if target else None, 

912 "overTarget": over, 

913 } 

914 ) 

915 size_row = by_size.setdefault( 

916 entry["size"], {"size": entry["size"], "tasks": 0, "overTarget": 0, "_i": [], "_m": []} 

917 ) 

918 size_row["tasks"] += 1 

919 size_row["overTarget"] += 1 if over else 0 

920 size_row["_i"].append(entry["interactions"]) 

921 size_row["_m"].append(wall_minutes) 

922 for size_row in by_size.values(): 

923 size_row["meanInteractions"] = round(statistics.fmean(size_row.pop("_i")), 2) 

924 size_row["meanWallMinutes"] = round(statistics.fmean(size_row.pop("_m")), 2) 

925 return friction_rows, by_size 

926 

927 

928def build_report( 

929 runs: list[_Run], 

930 *, 

931 filters: Filters, 

932 prices: Mapping[str, ModelPrice], 

933 targets: Mapping[Any, FrictionTarget] | None, 

934 projects: list[dict[str, Any]], 

935 now: datetime | None = None, 

936) -> dict[str, Any]: 

937 """The report of ``runs``: deterministic for the same records, filters, prices and 

938 ``now``.""" 

939 generated = now or datetime.now(UTC) 

940 effective_targets = { 

941 size: FrictionTarget.model_validate(values) 

942 for size, values in DEFAULT_FRICTION_TARGETS.items() 

943 } 

944 targets_source = "default" 

945 if targets: 945 ↛ 946line 945 didn't jump to line 946 because the condition on line 945 was never true

946 effective_targets.update(targets) 

947 targets_source = "configured" 

948 _price_calls(runs, prices) 

949 tables = _TokenTables() 

950 aggregate = _Aggregate(quality=_empty_quality(len(runs))) 

951 for run in runs: 

952 tables.add(run) 

953 for run in runs: 

954 aggregate.add(run) 

955 friction_rows, by_size = _friction(aggregate.tasks, effective_targets) 

956 totals = _finish_rows({"total": tables.totals})[0] 

957 quality = aggregate.quality 

958 decided = quality["decided"] 

959 durations = tables.durations 

960 phase_times = aggregate.phase_times 

961 run_rows = aggregate.run_rows 

962 report: dict[str, Any] = { 

963 "schemaVersion": SCHEMA_VERSION, 

964 "generatedAt": generated.isoformat(), 

965 "harnessVersion": __version__, 

966 "filters": filters.as_dict(), 

967 "projects": projects, 

968 "notes": [ 

969 "Computed from the run records and event chains only; no model was called.", 

970 ( 

971 "Costs: reportedCostUsd is what providers reported; estimatedCostUsd is computed " 

972 "from the price table for calls that reported tokens without a cost (estimated)." 

973 ), 

974 ( 

975 "Tokens: input includes cache tokens when the provider reports both; reasoning " 

976 "tokens are shown apart and not priced separately." 

977 ), 

978 ( 

979 "No per-person indicator: interactions, approvals and waits are counted per run, " 

980 "task and project." 

981 ), 

982 ], 

983 "totals": { 

984 "runs": len(runs), 

985 "tasks": len(aggregate.tasks), 

986 "agentCalls": totals["calls"], 

987 "tokens": { 

988 "input": totals["input"], 

989 "output": totals["output"], 

990 "reasoning": totals["reasoning"], 

991 "cache": totals["cache"], 

992 "cacheRead": totals["cacheRead"], 

993 "total": totals["tokens"], 

994 }, 

995 "cost": { 

996 "reportedUsd": totals["reportedCostUsd"], 

997 "estimatedUsd": totals["estimatedCostUsd"], 

998 "totalUsd": totals["costUsd"], 

999 "unpricedCalls": totals["unpricedCalls"], 

1000 }, 

1001 "linesAdded": aggregate.lines["added"], 

1002 "linesRemoved": aggregate.lines["removed"], 

1003 "humanInteractions": sum(row["interactions"] for row in run_rows), 

1004 "approvals": sum(row["approvals"] for row in run_rows), 

1005 "humanWaitMs": sum(row["humanWaitMs"] for row in run_rows), 

1006 "issuesResolved": len(aggregate.issues), 

1007 "featuresDelivered": len(aggregate.features), 

1008 "costPerFeatureUsd": round(totals["costUsd"] / len(aggregate.features), 6) 

1009 if aggregate.features 

1010 else None, 

1011 }, 

1012 "tokens": { 

1013 "byAgent": _finish_rows(tables.by["agent"]), 

1014 "byModel": _finish_rows(tables.by["model"]), 

1015 "byTask": _finish_rows(tables.by["task"]), 

1016 "byPhase": _finish_rows(tables.by["phase"]), 

1017 "byCallKind": _finish_rows(tables.by["callKind"]), 

1018 "byProject": _finish_rows(tables.by["project"]), 

1019 }, 

1020 "models": tables.model_rows(), 

1021 "lines": aggregate.lines, 

1022 "delivery": {"issuesResolved": aggregate.issues, "featuresDelivered": aggregate.features}, 

1023 "time": { 

1024 "perPhase": [ 

1025 { 

1026 "phase": phase, 

1027 "attempts": len(phase_times[phase]), 

1028 "totalMs": sum(phase_times[phase]), 

1029 "meanMs": int(statistics.fmean(phase_times[phase])), 

1030 } 

1031 for phase in _PHASES 

1032 if phase_times.get(phase) 

1033 ], 

1034 "agentCalls": { 

1035 "count": len(durations), 

1036 "totalMs": sum(durations), 

1037 "meanMs": int(statistics.fmean(durations)) if durations else None, 

1038 "p50Ms": _percentile(durations, 0.5), 

1039 "p90Ms": _percentile(durations, 0.9), 

1040 }, 

1041 "humanWaitMs": sum(row["humanWaitMs"] for row in run_rows), 

1042 }, 

1043 "quality": { 

1044 **quality, 

1045 "approvedFirstTimeRate": round(quality["approvedFirstTime"] / decided, 4) 

1046 if decided 

1047 else None, 

1048 "findingsByRule": sorted( 

1049 aggregate.findings_by_rule.values(), 

1050 key=lambda item: (-item["count"], item["ruleId"]), 

1051 ), 

1052 "findingsBySeverity": aggregate.findings_by_severity, 

1053 "findingsByFile": sorted( 

1054 aggregate.findings_by_file.values(), 

1055 key=lambda item: (-item["count"], item["path"]), 

1056 )[:_TOP_FILES], 

1057 "openBlockingFindings": aggregate.open_blocking, 

1058 "gateResults": { 

1059 "latestByRun": dict(sorted(aggregate.gate_results.items())), 

1060 "openFailed": aggregate.open_failed_gates, 

1061 }, 

1062 "limitsReached": aggregate.limits, 

1063 "blockedRunsDetail": aggregate.blocked_detail, 

1064 }, 

1065 "friction": { 

1066 "targets": { 

1067 size: target.model_dump(mode="json", by_alias=True) 

1068 for size, target in sorted(effective_targets.items()) 

1069 }, 

1070 "targetsSource": targets_source, 

1071 "perTask": sorted(friction_rows, key=lambda item: (item["projectId"], item["taskId"])), 

1072 "bySize": [by_size[size] for size in ("S", "M", "L") if size in by_size], 

1073 }, 

1074 "trends": { 

1075 "daily": [aggregate.daily[key] for key in sorted(aggregate.daily)], 

1076 "weekly": [aggregate.weekly[key] for key in sorted(aggregate.weekly)], 

1077 }, 

1078 "runs": run_rows, 

1079 } 

1080 report["summary"] = period_status( 

1081 { 

1082 "blockedRuns": quality["blockedRuns"], 

1083 "rejectedRuns": quality["rejectedRuns"], 

1084 "openBlockingFindings": aggregate.open_blocking, 

1085 "openFailedGates": aggregate.open_failed_gates, 

1086 "waitingDecision": quality["waitingDecision"], 

1087 }, 

1088 len(runs), 

1089 ) 

1090 return report 

1091 

1092 

1093__all__ = [ 

1094 "SCHEMA_VERSION", 

1095 "Filters", 

1096 "ProjectSource", 

1097 "build_report", 

1098 "collect", 

1099 "parse_since", 

1100 "task_issues", 

1101]