Coverage for src/governed_harness/metrics/report.py: 85%
510 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""``harness metrics``: the runs of one repository, or of every repository of the run registry,
2aggregated into one report (#58, items 8 to 10). Deterministic: the report is computed from the
3records and the event chains only, with zero model calls.
5What it never contains: a person's name or any per-person indicator. Interactions, approvals
6and human wait are counted per run, task and project (the thesis requirement RD-16 the
7earlier ``scripts/metrics_report.py`` already kept); lines are attributed to an agent invocation
8or to a person (out of band), never to whom."""
10from __future__ import annotations
12import json
13import re
14import statistics
15from collections import defaultdict
16from collections.abc import Iterable, Mapping
17from dataclasses import dataclass, field
18from datetime import UTC, date, datetime, timedelta
19from typing import Any
21from governed_harness import __version__
22from governed_harness.agents.routing import TaskSignals, classify_size
23from governed_harness.configuration.friction import (
24 DEFAULT_FRICTION_TARGETS,
25 FrictionTarget,
26 ModelPrice,
27)
28from governed_harness.domain.enums import DecisionKind, FindingSeverity, ResultStatus
29from governed_harness.domain.models import (
30 AgentInvocation,
31 ChangeSet,
32 ComponentProvenance,
33 Execution,
34 Finding,
35 GateEvaluation,
36 HumanDecision,
37 PhaseExecution,
38 ResourceUsage,
39 Task,
40)
41from governed_harness.events.sqlite_store import SQLiteEventStore, StoredEvent
42from governed_harness.metrics.prices import call_cost
43from governed_harness.metrics.summary import period_status
44from governed_harness.storage.sqlite import SQLiteStateStore
45from governed_harness.telemetry.metrics import HUMAN_INTERACTION_EVENTS, human_interactions
47SCHEMA_VERSION = "1.0"
48_ISSUE_TEXT = re.compile(r"\b(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+#(\d+)\b", re.IGNORECASE)
49_APPROVALS = frozenset({DecisionKind.APPROVE, DecisionKind.APPROVE_EXCEPTION})
50_RELATIVE_UNITS = {"h": "hours", "d": "days", "w": "weeks"}
51_CORRECTION = "correction.authorized"
52# Events that record a limit being reached, with the key they get in quality.limitsReached.
53_LIMIT_EVENTS = {
54 "budget.exceeded": "budget",
55 "phase.attempts.exhausted": "phaseAttempts",
56 "correction.exhausted": "corrections",
57}
58_BLOCKING_SEVERITIES = frozenset({FindingSeverity.HIGH, FindingSeverity.CRITICAL})
59_FAILED_GATES = frozenset(
60 {ResultStatus.FAILED, ResultStatus.ERROR, ResultStatus.TIMED_OUT, ResultStatus.INCONCLUSIVE}
61)
62_SEVERITIES = tuple(item.value for item in FindingSeverity)
63_TOP_FILES = 20
64_ESCALATED_LANE = "fast→full"
65_PHASES = (
66 "INTENT",
67 "DISCOVERY",
68 "SPECIFICATION",
69 "PLANNING",
70 "IMPLEMENTATION",
71 "VERIFICATION",
72 "INDEPENDENT_REVIEW",
73 "DECISION",
74 "CLOSURE",
75)
78@dataclass
79class ProjectSource:
80 """The records of one project: its state and event stores and its settings."""
82 project_id: str
83 workspace: str
84 state: SQLiteStateStore
85 events: SQLiteEventStore
86 targets: Mapping[Any, FrictionTarget] | None = None
87 forge: dict[str, str] | None = None
88 """``kind``, ``host`` and ``repository`` of the origin remote, for issue links."""
91@dataclass(frozen=True)
92class Filters:
93 since: datetime | None = None
94 task: str | None = None
95 model: str | None = None
96 agent: str | None = None
97 all_repos: bool = False
99 def as_dict(self) -> dict[str, Any]:
100 return {
101 "since": self.since.isoformat() if self.since else None,
102 "task": self.task,
103 "model": self.model,
104 "agent": self.agent,
105 "allRepos": self.all_repos,
106 }
109def parse_since(value: str | None, now: datetime | None = None) -> datetime | None:
110 """``YYYY-MM-DD``, an ISO 8601 instant, or a relative ``7d``/``2w``/``12h``."""
111 if not value:
112 return None
113 text = value.strip()
114 # A plain parse instead of a regular expression: linear on any input (``--since`` comes
115 # from the command line). ``isdecimal`` accepts exactly what ``\d`` accepted.
116 unit = _RELATIVE_UNITS.get(text[-1:])
117 if unit is not None and text[:-1].isdecimal():
118 return (now or datetime.now(UTC)) - timedelta(**{unit: int(text[:-1])})
119 try:
120 parsed = datetime.fromisoformat(text)
121 except ValueError:
122 try:
123 parsed = datetime.combine(date.fromisoformat(text), datetime.min.time())
124 except ValueError as error:
125 raise ValueError(
126 f"--since takes YYYY-MM-DD, an ISO 8601 instant or 12h, 7d, 2w: {value!r}"
127 ) from error
128 return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
131@dataclass
132class _Call:
133 invocation: AgentInvocation
134 usage: dict[str, Any] | None
135 reported: float | None
136 estimated: float | None
137 cost_source: str
139 @property
140 def tokens(self) -> dict[str, int]:
141 usage = self.usage or {}
142 return {
143 name: int(usage.get(field) or 0)
144 for name, field in (
145 ("input", "input_tokens"),
146 ("output", "output_tokens"),
147 ("reasoning", "reasoning_tokens"),
148 ("cache", "cache_tokens"),
149 ("cacheRead", "cache_read_tokens"),
150 )
151 }
153 @property
154 def duration_ms(self) -> int:
155 delta = self.invocation.finished_at - self.invocation.started_at
156 return max(0, int(delta.total_seconds() * 1000))
159@dataclass
160class _Run:
161 project_id: str
162 execution: Execution
163 task: Task | None
164 events: list[StoredEvent]
165 calls: list[_Call]
166 phases: list[PhaseExecution]
167 decisions: list[HumanDecision]
168 change_set: ChangeSet | None
169 provenance: ComponentProvenance | None
170 findings: list[Finding]
171 lane: dict[str, Any] | None
172 size: str
173 issues: list[dict[str, Any]] = field(default_factory=list)
174 pull_request: dict[str, Any] | None = None
175 latest_gate: ResultStatus | None = None
177 @property
178 def run_id(self) -> str:
179 return self.execution.execution_id
181 def times(self) -> tuple[datetime, datetime]:
182 stamps = [datetime.fromisoformat(event.occurred_at) for event in self.events]
183 if not stamps: 183 ↛ 184line 183 didn't jump to line 184 because the condition on line 183 was never true
184 return self.execution.created_at, self.execution.updated_at
185 return min(stamps), max(stamps)
188def _flag_json(state: SQLiteStateStore, key: str) -> Any:
189 raw = state.get_flag(key)
190 try:
191 return json.loads(raw) if raw else None
192 except ValueError:
193 return None
196def _size(state: SQLiteStateStore, execution: Execution, task: Task | None) -> str:
197 """The lane's size when the run recorded one, else the router's classification of the
198 task's requirements and owned paths (lines are not read: the workspace may be elsewhere)."""
199 lane = _flag_json(state, f"lane:{execution.execution_id}")
200 if isinstance(lane, dict) and lane.get("size") in {"S", "M", "L"}: 200 ↛ 201line 200 didn't jump to line 201 because the condition on line 200 was never true
201 return str(lane["size"])
202 if task is None: 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true
203 return "S"
204 owned = task.metadata.get("ownedPaths") or [patch.path for patch in task.implementation.patches]
205 signals = TaskSignals(
206 requirements=len(task.requirements) or len(task.acceptance_criteria),
207 files=len(list(owned)),
208 loc=0,
209 )
210 return classify_size(signals, None)[0]
213def _issue_link(forge: Mapping[str, str] | None, number: str) -> str | None:
214 if not forge or not forge.get("host") or not forge.get("repository"): 214 ↛ 216line 214 didn't jump to line 216 because the condition on line 214 was always true
215 return None
216 base = f"https://{forge['host']}/{forge['repository']}"
217 kind = forge.get("kind")
218 if kind == "gitlab":
219 return f"{base}/-/issues/{number}"
220 if kind == "azure-devops":
221 parts = forge["repository"].split("/")
222 return (
223 f"https://{forge['host']}/{parts[0]}/{parts[1]}/_workitems/edit/{number}"
224 if len(parts) >= 2
225 else None
226 )
227 return f"{base}/issues/{number}"
230def _pull_link(forge: Mapping[str, str] | None, number: Any) -> str | None:
231 if not forge or not isinstance(number, int):
232 return None
233 base = f"https://{forge.get('host')}/{forge.get('repository')}"
234 kind = forge.get("kind")
235 if kind == "gitlab":
236 return f"{base}/-/merge_requests/{number}"
237 if kind == "bitbucket":
238 return f"{base}/pull-requests/{number}"
239 if kind == "azure-devops":
240 return None
241 return f"{base}/pull/{number}"
244def _as_list(value: Any) -> list[Any]:
245 if isinstance(value, list):
246 return value
247 return [] if value is None else [value]
250def task_issues(task: Task) -> list[str]:
251 """Issue numbers a task names: ``metadata.issues`` / ``metadata.issue`` and closing
252 keywords (``Closes #12``) in its title and intent."""
253 found: list[str] = []
254 for key in ("issues", "issue"):
255 for item in _as_list(task.metadata.get(key)):
256 text = str(item).strip().lstrip("#")
257 if text.isdigit(): 257 ↛ 255line 257 didn't jump to line 255 because the condition on line 257 was always true
258 found.append(text)
259 for text in (task.title, task.intent):
260 found.extend(_ISSUE_TEXT.findall(text))
261 return list(dict.fromkeys(found))
264def _selected(execution: Execution, filters: Filters) -> bool:
265 if filters.task and execution.task_id != filters.task:
266 return False
267 return not (filters.since and execution.created_at < filters.since)
270def _load_task(state: SQLiteStateStore, execution: Execution) -> Task | None:
271 try:
272 return state.get("task", execution.task_id, Task)
273 except Exception: # noqa: BLE001 - a missing task still counts the run
274 return None
277def _calls(state: SQLiteStateStore, run_id: str, filters: Filters) -> list[_Call]:
278 """The agent calls of a run that pass the ``model`` and ``agent`` filters."""
279 invocations = state.list("agent_invocation", AgentInvocation, execution_id=run_id)
280 usages = {
281 item.invocation_id: item.model_dump(mode="json")
282 for item in state.list("resource_usage", ResourceUsage, execution_id=run_id)
283 if item.invocation_id
284 }
285 return [
286 _Call(invocation, usages.get(invocation.invocation_id), None, None, "none")
287 for invocation in invocations
288 if not (filters.model and invocation.model != filters.model)
289 and not (filters.agent and invocation.provider != filters.agent)
290 ]
293def _latest_gate(state: SQLiteStateStore, execution: Execution) -> ResultStatus | None:
294 """The status of the gate evaluation the run points at, if any."""
295 if not execution.gate_evaluation_id: 295 ↛ 296line 295 didn't jump to line 296 because the condition on line 295 was never true
296 return None
297 try:
298 return state.get("gate", execution.gate_evaluation_id, GateEvaluation).status
299 except Exception: # noqa: BLE001 - a missing gate record only leaves the status unknown
300 return None
303def _change_set(state: SQLiteStateStore, execution: Execution) -> ChangeSet | None:
304 """The run's current change set, else its latest one."""
305 changes = state.list("change_set", ChangeSet, execution_id=execution.execution_id)
306 current = [item for item in changes if item.digest == execution.change_set_digest]
307 ordered = current or sorted(changes, key=lambda item: item.created_at)
308 return ordered[-1] if ordered else None
311def _provenance(
312 state: SQLiteStateStore, run_id: str, change_set: ChangeSet | None
313) -> ComponentProvenance | None:
314 if change_set is None: 314 ↛ 315line 314 didn't jump to line 315 because the condition on line 314 was never true
315 return None
316 provenances = [
317 item
318 for item in state.list("component_provenance", ComponentProvenance, execution_id=run_id)
319 if item.change_set_digest == change_set.digest
320 ]
321 return provenances[-1] if provenances else None
324def _pull_request(
325 events: list[StoredEvent], forge: Mapping[str, str] | None
326) -> dict[str, Any] | None:
327 """The last pull request the run's delivery opened, with a link to it."""
328 pull: dict[str, Any] | None = None
329 for event in events:
330 if event.event_type != "delivery.pull-request.created": 330 ↛ 332line 330 didn't jump to line 332 because the condition on line 330 was always true
331 continue
332 payload = event.payload
333 number = payload.get("number")
334 event_forge = (
335 {
336 "kind": str(payload.get("forge") or ""),
337 "host": str(payload.get("host") or ""),
338 "repository": str(payload.get("repository") or ""),
339 }
340 if payload.get("host")
341 else forge
342 )
343 pull = {
344 "number": number,
345 "forge": payload.get("forge"),
346 "repository": payload.get("repository"),
347 "url": payload.get("url") or _pull_link(event_forge, number),
348 }
349 return pull
352def _run(source: ProjectSource, execution: Execution, filters: Filters) -> _Run | None:
353 """The records of one run, or ``None`` when the ``model``/``agent`` filters leave it no
354 call."""
355 state = source.state
356 task = _load_task(state, execution)
357 run_id = execution.execution_id
358 calls = _calls(state, run_id, filters)
359 if (filters.model or filters.agent) and not calls:
360 return None
361 change_set = _change_set(state, execution)
362 provenance = _provenance(state, run_id, change_set)
363 events = source.events.list(run_id)
364 lane = _flag_json(state, f"lane:{run_id}")
365 run = _Run(
366 project_id=source.project_id,
367 execution=execution,
368 task=task,
369 events=events,
370 calls=calls,
371 phases=state.list("phase", PhaseExecution, execution_id=run_id),
372 decisions=sorted(
373 state.list("decision", HumanDecision, execution_id=run_id),
374 key=lambda item: item.decided_at,
375 ),
376 change_set=change_set,
377 provenance=provenance,
378 findings=state.list("finding", Finding, execution_id=run_id),
379 lane=lane if isinstance(lane, dict) else None,
380 size=_size(state, execution, task),
381 pull_request=_pull_request(events, source.forge),
382 latest_gate=_latest_gate(state, execution),
383 )
384 if task is not None: 384 ↛ 389line 384 didn't jump to line 389 because the condition on line 384 was always true
385 run.issues = [
386 {"issue": f"#{number}", "url": _issue_link(source.forge, number)}
387 for number in task_issues(task)
388 ]
389 return run
392def collect(sources: Iterable[ProjectSource], filters: Filters) -> list[_Run]:
393 runs: list[_Run] = []
394 for source in sources:
395 executions = source.state.list("execution", Execution, project_id=source.project_id)
396 for execution in executions:
397 if not _selected(execution, filters):
398 continue
399 run = _run(source, execution, filters)
400 if run is not None:
401 runs.append(run)
402 return sorted(runs, key=lambda item: item.execution.created_at)
405# ----- aggregation ------------------------------------------------------------------------------
406def _token_row(key: str) -> dict[str, Any]:
407 return {
408 "key": key,
409 "calls": 0,
410 "input": 0,
411 "output": 0,
412 "reasoning": 0,
413 "cache": 0,
414 "cacheRead": 0,
415 "reportedCostUsd": 0.0,
416 "estimatedCostUsd": 0.0,
417 "unpricedCalls": 0,
418 "durationMs": 0,
419 }
422def _add_call(row: dict[str, Any], call: _Call) -> None:
423 row["calls"] += 1
424 for name, value in call.tokens.items():
425 row[name] += value
426 row["reportedCostUsd"] += call.reported or 0.0
427 row["estimatedCostUsd"] += call.estimated or 0.0
428 row["unpricedCalls"] += 1 if call.cost_source == "unpriced" else 0
429 row["durationMs"] += call.duration_ms
432def _finish_rows(rows: Mapping[str, dict[str, Any]]) -> list[dict[str, Any]]:
433 finished = []
434 for row in rows.values():
435 row["reportedCostUsd"] = round(row["reportedCostUsd"], 6)
436 row["estimatedCostUsd"] = round(row["estimatedCostUsd"], 6)
437 row["costUsd"] = round(row["reportedCostUsd"] + row["estimatedCostUsd"], 6)
438 row["tokens"] = row["input"] + row["output"]
439 finished.append(row)
440 return sorted(finished, key=lambda item: (-item["tokens"], item["key"]))
443def _human_wait_ms(events: list[StoredEvent]) -> int:
444 """Time the run waited for a person: from each phase that ended BLOCKED to the next event
445 a person caused or the next phase that started, whichever came first."""
446 total = 0
447 waiting_since: datetime | None = None
448 for event in events:
449 moment = datetime.fromisoformat(event.occurred_at)
450 if (
451 event.event_type == "phase.completed"
452 and event.payload.get("status") == ResultStatus.BLOCKED.value
453 ):
454 waiting_since = waiting_since or moment
455 continue
456 if waiting_since is None:
457 continue
458 human = (
459 event.event_type in HUMAN_INTERACTION_EVENTS and event.actor.get("actorType") == "HUMAN"
460 )
461 if human or event.event_type == "phase.started": 461 ↛ 448line 461 didn't jump to line 448 because the condition on line 461 was always true
462 total += max(0, int((moment - waiting_since).total_seconds() * 1000))
463 waiting_since = None
464 return total
467def _percentile(values: list[int], fraction: float) -> int | None:
468 if not values:
469 return None
470 ordered = sorted(values)
471 index = min(len(ordered) - 1, max(0, round(fraction * (len(ordered) - 1))))
472 return ordered[index]
475def _price_calls(runs: list[_Run], prices: Mapping[str, ModelPrice]) -> None:
476 for run in runs:
477 for call in run.calls:
478 priced = call_cost(call.usage, prices, call.invocation.provider, call.invocation.model)
479 call.reported, call.estimated, call.cost_source = (
480 priced.reported,
481 priced.estimated,
482 priced.source,
483 )
486def _lane_label(lane: dict[str, Any] | None) -> Any:
487 """``fast→full`` for a lane that escalated, else the lane the run recorded."""
488 if not lane: 488 ↛ 490line 488 didn't jump to line 490 because the condition on line 488 was always true
489 return None
490 return _ESCALATED_LANE if lane.get("escalated") else lane.get("lane")
493def _cost_label(sources: set[str]) -> str:
494 """How a model's costs are known: all reported, some estimated, unpriced or none."""
495 if sources == {"reported"}: 495 ↛ 496line 495 didn't jump to line 496 because the condition on line 495 was never true
496 return "reported"
497 for label in ("estimated", "unpriced"):
498 if label in sources:
499 return label
500 return "none"
503def _phase_ms(phase: PhaseExecution) -> int | None:
504 if not (phase.started_at and phase.finished_at): 504 ↛ 505line 504 didn't jump to line 505 because the condition on line 504 was never true
505 return None
506 return int((phase.finished_at - phase.started_at).total_seconds() * 1000)
509@dataclass
510class _TokenTables:
511 """Token, cost and duration rows of the agent calls, per dimension and in total."""
513 by: dict[str, dict[str, dict[str, Any]]] = field(
514 default_factory=lambda: {
515 name: {} for name in ("agent", "model", "task", "phase", "callKind", "project")
516 }
517 )
518 totals: dict[str, Any] = field(default_factory=lambda: _token_row("total"))
519 models: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict)
520 durations: list[int] = field(default_factory=list)
522 def add(self, run: _Run) -> None:
523 for call in run.calls:
524 invocation = call.invocation
525 model = invocation.model or "unknown"
526 keys = {
527 "agent": invocation.provider,
528 "model": model,
529 "task": run.execution.task_id,
530 "phase": invocation.phase_id.value,
531 "callKind": invocation.call_kind or "implement",
532 "project": run.project_id,
533 }
534 for name, value in keys.items():
535 _add_call(self.by[name].setdefault(value, _token_row(value)), call)
536 _add_call(self.totals, call)
537 self.durations.append(call.duration_ms)
538 model_entry = self.models.setdefault(
539 (invocation.provider, model),
540 {"provider": invocation.provider, "model": model, "calls": 0, "costSources": set()},
541 )
542 model_entry["calls"] += 1
543 model_entry["costSources"].add(call.cost_source)
545 def model_rows(self) -> list[dict[str, Any]]:
546 return [
547 {
548 "provider": entry["provider"],
549 "model": entry["model"],
550 "calls": entry["calls"],
551 "cost": _cost_label(entry["costSources"]),
552 }
553 for entry in sorted(
554 self.models.values(), key=lambda item: (-item["calls"], item["model"])
555 )
556 ]
559@dataclass(frozen=True)
560class _RunTotals:
561 """What one run counts: its span, the people's part and the agents' tokens and cost."""
563 started: datetime
564 finished: datetime
565 interactions: int
566 approvals: int
567 wait_ms: int
568 corrections: int
569 closed: bool
570 tokens: int
571 cost: float
573 @classmethod
574 def of(cls, run: _Run) -> _RunTotals:
575 started, finished = run.times()
576 return cls(
577 started=started,
578 finished=finished,
579 interactions=sum(human_interactions(run.events).values()),
580 approvals=sum(1 for item in run.decisions if item.decision in _APPROVALS),
581 wait_ms=_human_wait_ms(run.events),
582 corrections=sum(1 for event in run.events if event.event_type == _CORRECTION),
583 closed=run.execution.status is ResultStatus.PASSED,
584 tokens=sum(call.tokens["input"] + call.tokens["output"] for call in run.calls),
585 cost=sum((call.reported or 0.0) + (call.estimated or 0.0) for call in run.calls),
586 )
589def _empty_lines() -> dict[str, Any]:
590 return {
591 "added": 0,
592 "removed": 0,
593 "agent": {"added": 0, "removed": 0, "files": 0},
594 "person": {"added": 0, "removed": 0, "files": 0},
595 "unattributed": {"added": 0, "removed": 0, "files": 0},
596 }
599def _empty_quality(runs: int) -> dict[str, int]:
600 return {
601 "runs": runs,
602 "decided": 0,
603 "approvedFirstTime": 0,
604 "corrections": 0,
605 "requestedChanges": 0,
606 "blockedRuns": 0,
607 "waitingDecision": 0,
608 "rejectedRuns": 0,
609 "closedRuns": 0,
610 "preAuthorizedApprovals": 0,
611 }
614_AUTHORS = {"AGENT": "agent", "OUT_OF_BAND": "person"}
617@dataclass
618class _Aggregate:
619 """The per-run sections of the report, filled one run at a time."""
621 quality: dict[str, int]
622 lines: dict[str, Any] = field(default_factory=_empty_lines)
623 issues: list[dict[str, Any]] = field(default_factory=list)
624 features: list[dict[str, Any]] = field(default_factory=list)
625 tasks: dict[str, dict[str, Any]] = field(default_factory=dict)
626 phase_times: dict[str, list[int]] = field(default_factory=lambda: defaultdict(list))
627 findings_by_rule: dict[str, dict[str, Any]] = field(default_factory=dict)
628 findings_by_severity: dict[str, int] = field(
629 default_factory=lambda: dict.fromkeys(_SEVERITIES, 0)
630 )
631 findings_by_file: dict[str, dict[str, Any]] = field(default_factory=dict)
632 blocked_detail: list[dict[str, Any]] = field(default_factory=list)
633 limits: dict[str, int] = field(default_factory=lambda: dict.fromkeys(_LIMIT_EVENTS.values(), 0))
634 gate_results: dict[str, int] = field(default_factory=dict)
635 open_blocking: int = 0
636 open_failed_gates: int = 0
637 run_rows: list[dict[str, Any]] = field(default_factory=list)
638 daily: dict[str, dict[str, Any]] = field(default_factory=dict)
639 weekly: dict[str, dict[str, Any]] = field(default_factory=dict)
641 def add(self, run: _Run) -> None:
642 totals = _RunTotals.of(run)
643 title = run.task.title if run.task else run.execution.task_id
644 self._add_quality(run, totals)
645 self._add_findings(run, totals.closed)
646 self._add_limits_and_gates(run, totals.closed)
647 for phase in run.phases:
648 elapsed = _phase_ms(phase)
649 if elapsed is not None: 649 ↛ 647line 649 didn't jump to line 647 because the condition on line 649 was always true
650 self.phase_times[phase.phase_id.value].append(elapsed)
651 added, removed = self._add_lines(run)
652 if totals.closed: 652 ↛ 654line 652 didn't jump to line 654 because the condition on line 652 was always true
653 self._add_delivered(run, title, totals.finished.isoformat())
654 self._add_task(run, title, totals)
655 self._add_run_row(run, totals, added, removed)
656 self._add_trends(run, totals)
658 def _add_quality(self, run: _Run, totals: _RunTotals) -> None:
659 quality = self.quality
660 execution = run.execution
661 quality["corrections"] += totals.corrections
662 quality["requestedChanges"] += sum(
663 1 for item in run.decisions if item.decision is DecisionKind.REQUEST_CHANGES
664 )
665 if run.decisions: 665 ↛ 669line 665 didn't jump to line 669 because the condition on line 665 was always true
666 quality["decided"] += 1
667 if _approved_first_time(run): 667 ↛ 669line 667 didn't jump to line 669 because the condition on line 667 was always true
668 quality["approvedFirstTime"] += 1
669 quality["preAuthorizedApprovals"] += sum(
670 1 for item in run.decisions if item.pre_authorization_id
671 )
672 if execution.status is ResultStatus.BLOCKED: 672 ↛ 673line 672 didn't jump to line 673 because the condition on line 672 was never true
673 waiting = execution.current_phase.value == "DECISION"
674 quality["waitingDecision" if waiting else "blockedRuns"] += 1
675 if any(item.decision is DecisionKind.REJECT for item in run.decisions): 675 ↛ 676line 675 didn't jump to line 676 because the condition on line 675 was never true
676 quality["rejectedRuns"] += 1
677 if totals.closed: 677 ↛ exitline 677 didn't return from function '_add_quality' because the condition on line 677 was always true
678 quality["closedRuns"] += 1
680 def _add_findings(self, run: _Run, closed: bool) -> None:
681 for finding in run.findings:
682 row = self.findings_by_rule.setdefault(
683 finding.rule_id, {"ruleId": finding.rule_id, "count": 0, "bySeverity": {}}
684 )
685 row["count"] += 1
686 severity = finding.severity.value
687 row["bySeverity"][severity] = row["bySeverity"].get(severity, 0) + 1
688 self.findings_by_severity[severity] += 1
689 if finding.severity in _BLOCKING_SEVERITIES and not closed: 689 ↛ 690line 689 didn't jump to line 690 because the condition on line 689 was never true
690 self.open_blocking += 1
691 path = finding.location.path if finding.location else None
692 if path:
693 entry = self.findings_by_file.setdefault(
694 path, {"path": path, "count": 0, "bySeverity": {}}
695 )
696 entry["count"] += 1
697 entry["bySeverity"][severity] = entry["bySeverity"].get(severity, 0) + 1
699 def _add_limits_and_gates(self, run: _Run, closed: bool) -> None:
700 """Limits reached (events), the latest gate result of the run and, for a run that
701 stopped, the reason it recorded."""
702 for event in run.events:
703 key = _LIMIT_EVENTS.get(event.event_type)
704 if key: 704 ↛ 705line 704 didn't jump to line 705 because the condition on line 704 was never true
705 self.limits[key] += 1
706 if run.latest_gate is not None: 706 ↛ 711line 706 didn't jump to line 711 because the condition on line 706 was always true
707 name = run.latest_gate.value
708 self.gate_results[name] = self.gate_results.get(name, 0) + 1
709 if not closed and run.latest_gate in _FAILED_GATES: 709 ↛ 710line 709 didn't jump to line 710 because the condition on line 709 was never true
710 self.open_failed_gates += 1
711 execution = run.execution
712 if execution.status is ResultStatus.BLOCKED and execution.current_phase.value != "DECISION": 712 ↛ 713line 712 didn't jump to line 713 because the condition on line 712 was never true
713 self.blocked_detail.append(
714 {
715 "projectId": run.project_id,
716 "runId": run.run_id,
717 "taskId": execution.task_id,
718 "phase": execution.current_phase.value,
719 "reason": execution.terminal_reason,
720 }
721 )
723 def _add_lines(self, run: _Run) -> tuple[int, int]:
724 """Adds the run's changed lines, attributed to an agent, a person or neither."""
725 added = removed = 0
726 if run.change_set is not None: 726 ↛ 737line 726 didn't jump to line 737 because the condition on line 726 was always true
727 sources = (
728 {item.path: item.source for item in run.provenance.files} if run.provenance else {}
729 )
730 for changed in run.change_set.files:
731 added += changed.additions
732 removed += changed.deletions
733 author = self.lines[_AUTHORS.get(sources.get(changed.path) or "", "unattributed")]
734 author["added"] += changed.additions
735 author["removed"] += changed.deletions
736 author["files"] += 1
737 self.lines["added"] += added
738 self.lines["removed"] += removed
739 return added, removed
741 def _add_delivered(self, run: _Run, title: str, closed_at: str) -> None:
742 task_id = run.execution.task_id
743 self.features.append(
744 {
745 "projectId": run.project_id,
746 "taskId": task_id,
747 "title": title,
748 "runId": run.run_id,
749 "closedAt": closed_at,
750 "issues": run.issues,
751 "pullRequest": run.pull_request,
752 }
753 )
754 for issue in run.issues:
755 self.issues.append(
756 {
757 **issue,
758 "projectId": run.project_id,
759 "taskId": task_id,
760 "runId": run.run_id,
761 "closedAt": closed_at,
762 }
763 )
765 def _add_task(self, run: _Run, title: str, totals: _RunTotals) -> None:
766 task_id = run.execution.task_id
767 entry = self.tasks.setdefault(
768 f"{run.project_id}:{task_id}",
769 {
770 "projectId": run.project_id,
771 "taskId": task_id,
772 "title": title,
773 "size": run.size,
774 "lanes": [],
775 "runs": 0,
776 "interactions": 0,
777 "approvals": 0,
778 "corrections": 0,
779 "humanWaitMs": 0,
780 "agentMs": 0,
781 "tokens": 0,
782 "costUsd": 0.0,
783 "start": totals.started,
784 "end": totals.finished,
785 "phasesMs": {},
786 },
787 )
788 entry["runs"] += 1
789 entry["interactions"] += totals.interactions
790 entry["approvals"] += totals.approvals
791 entry["corrections"] += totals.corrections
792 entry["humanWaitMs"] += totals.wait_ms
793 entry["agentMs"] += sum(call.duration_ms for call in run.calls)
794 entry["tokens"] += totals.tokens
795 entry["costUsd"] += totals.cost
796 entry["start"] = min(entry["start"], totals.started)
797 entry["end"] = max(entry["end"], totals.finished)
798 if run.lane and run.lane.get("lane"): 798 ↛ 799line 798 didn't jump to line 799 because the condition on line 798 was never true
799 entry["lanes"].append(str(_lane_label(run.lane)))
800 for phase in run.phases:
801 elapsed = _phase_ms(phase)
802 if elapsed is not None: 802 ↛ 800line 802 didn't jump to line 800 because the condition on line 802 was always true
803 phase_key = phase.phase_id.value
804 entry["phasesMs"][phase_key] = entry["phasesMs"].get(phase_key, 0) + elapsed
806 def _add_run_row(self, run: _Run, totals: _RunTotals, added: int, removed: int) -> None:
807 execution = run.execution
808 self.run_rows.append(
809 {
810 "projectId": run.project_id,
811 "runId": run.run_id,
812 "taskId": execution.task_id,
813 "status": execution.status.value,
814 "phase": execution.current_phase.value,
815 "size": run.size,
816 "lane": _lane_label(run.lane),
817 "createdAt": execution.created_at.isoformat(),
818 "wallMs": int((totals.finished - totals.started).total_seconds() * 1000),
819 "humanWaitMs": totals.wait_ms,
820 "agentCalls": len(run.calls),
821 "inputTokens": sum(call.tokens["input"] for call in run.calls),
822 "outputTokens": sum(call.tokens["output"] for call in run.calls),
823 "cacheTokens": sum(call.tokens["cache"] for call in run.calls),
824 "reportedCostUsd": round(sum(call.reported or 0.0 for call in run.calls), 6),
825 "estimatedCostUsd": round(sum(call.estimated or 0.0 for call in run.calls), 6),
826 "interactions": totals.interactions,
827 "approvals": totals.approvals,
828 "corrections": totals.corrections,
829 "linesAdded": added,
830 "linesRemoved": removed,
831 }
832 )
834 def _add_trends(self, run: _Run, totals: _RunTotals) -> None:
835 day = run.execution.created_at.astimezone(UTC).date()
836 iso = day.isocalendar()
837 for bucket_map, period in (
838 (self.daily, day.isoformat()),
839 (self.weekly, f"{iso.year}-W{iso.week:02d}"),
840 ):
841 bucket = bucket_map.setdefault(
842 period,
843 {
844 "period": period,
845 "runs": 0,
846 "tokens": 0,
847 "costUsd": 0.0,
848 "interactions": 0,
849 "approved": 0,
850 "closed": 0,
851 },
852 )
853 bucket["runs"] += 1
854 bucket["tokens"] += totals.tokens
855 bucket["costUsd"] = round(bucket["costUsd"] + totals.cost, 6)
856 bucket["interactions"] += totals.interactions
857 bucket["approved"] += 1 if totals.approvals else 0
858 bucket["closed"] += 1 if totals.closed else 0
861def _approved_first_time(run: _Run) -> bool:
862 """The first decision approved and no correction was authorized before it."""
863 first = run.decisions[0]
864 if first.decision not in _APPROVALS: 864 ↛ 865line 864 didn't jump to line 865 because the condition on line 864 was never true
865 return False
866 return not any(
867 event.event_type == _CORRECTION
868 and datetime.fromisoformat(event.occurred_at) <= first.decided_at
869 for event in run.events
870 )
873def _over_target(target: FrictionTarget | None, interactions: int, minutes: float) -> list[str]:
874 over: list[str] = []
875 if target is None: 875 ↛ 876line 875 didn't jump to line 876 because the condition on line 875 was never true
876 return over
877 if target.interactions is not None and interactions > target.interactions: 877 ↛ 878line 877 didn't jump to line 878 because the condition on line 877 was never true
878 over.append("interactions")
879 if target.minutes is not None and minutes > target.minutes: 879 ↛ 880line 879 didn't jump to line 880 because the condition on line 879 was never true
880 over.append("minutes")
881 return over
884def _friction(
885 tasks: dict[str, dict[str, Any]], targets: Mapping[Any, FrictionTarget]
886) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]:
887 """One friction row per task against its size's target, and the means per size."""
888 friction_rows = []
889 by_size: dict[str, dict[str, Any]] = {}
890 for entry in tasks.values():
891 wall_minutes = round((entry["end"] - entry["start"]).total_seconds() / 60, 2)
892 target = targets.get(entry["size"])
893 over = _over_target(target, entry["interactions"], wall_minutes)
894 friction_rows.append(
895 {
896 "projectId": entry["projectId"],
897 "taskId": entry["taskId"],
898 "title": entry["title"],
899 "size": entry["size"],
900 "lanes": entry["lanes"],
901 "runs": entry["runs"],
902 "interactions": entry["interactions"],
903 "approvals": entry["approvals"],
904 "corrections": entry["corrections"],
905 "wallMinutes": wall_minutes,
906 "humanWaitMinutes": round(entry["humanWaitMs"] / 60000, 2),
907 "agentMinutes": round(entry["agentMs"] / 60000, 2),
908 "tokens": entry["tokens"],
909 "costUsd": round(entry["costUsd"], 6),
910 "phasesMs": entry["phasesMs"],
911 "target": target.model_dump(mode="json", by_alias=True) if target else None,
912 "overTarget": over,
913 }
914 )
915 size_row = by_size.setdefault(
916 entry["size"], {"size": entry["size"], "tasks": 0, "overTarget": 0, "_i": [], "_m": []}
917 )
918 size_row["tasks"] += 1
919 size_row["overTarget"] += 1 if over else 0
920 size_row["_i"].append(entry["interactions"])
921 size_row["_m"].append(wall_minutes)
922 for size_row in by_size.values():
923 size_row["meanInteractions"] = round(statistics.fmean(size_row.pop("_i")), 2)
924 size_row["meanWallMinutes"] = round(statistics.fmean(size_row.pop("_m")), 2)
925 return friction_rows, by_size
928def build_report(
929 runs: list[_Run],
930 *,
931 filters: Filters,
932 prices: Mapping[str, ModelPrice],
933 targets: Mapping[Any, FrictionTarget] | None,
934 projects: list[dict[str, Any]],
935 now: datetime | None = None,
936) -> dict[str, Any]:
937 """The report of ``runs``: deterministic for the same records, filters, prices and
938 ``now``."""
939 generated = now or datetime.now(UTC)
940 effective_targets = {
941 size: FrictionTarget.model_validate(values)
942 for size, values in DEFAULT_FRICTION_TARGETS.items()
943 }
944 targets_source = "default"
945 if targets: 945 ↛ 946line 945 didn't jump to line 946 because the condition on line 945 was never true
946 effective_targets.update(targets)
947 targets_source = "configured"
948 _price_calls(runs, prices)
949 tables = _TokenTables()
950 aggregate = _Aggregate(quality=_empty_quality(len(runs)))
951 for run in runs:
952 tables.add(run)
953 for run in runs:
954 aggregate.add(run)
955 friction_rows, by_size = _friction(aggregate.tasks, effective_targets)
956 totals = _finish_rows({"total": tables.totals})[0]
957 quality = aggregate.quality
958 decided = quality["decided"]
959 durations = tables.durations
960 phase_times = aggregate.phase_times
961 run_rows = aggregate.run_rows
962 report: dict[str, Any] = {
963 "schemaVersion": SCHEMA_VERSION,
964 "generatedAt": generated.isoformat(),
965 "harnessVersion": __version__,
966 "filters": filters.as_dict(),
967 "projects": projects,
968 "notes": [
969 "Computed from the run records and event chains only; no model was called.",
970 (
971 "Costs: reportedCostUsd is what providers reported; estimatedCostUsd is computed "
972 "from the price table for calls that reported tokens without a cost (estimated)."
973 ),
974 (
975 "Tokens: input includes cache tokens when the provider reports both; reasoning "
976 "tokens are shown apart and not priced separately."
977 ),
978 (
979 "No per-person indicator: interactions, approvals and waits are counted per run, "
980 "task and project."
981 ),
982 ],
983 "totals": {
984 "runs": len(runs),
985 "tasks": len(aggregate.tasks),
986 "agentCalls": totals["calls"],
987 "tokens": {
988 "input": totals["input"],
989 "output": totals["output"],
990 "reasoning": totals["reasoning"],
991 "cache": totals["cache"],
992 "cacheRead": totals["cacheRead"],
993 "total": totals["tokens"],
994 },
995 "cost": {
996 "reportedUsd": totals["reportedCostUsd"],
997 "estimatedUsd": totals["estimatedCostUsd"],
998 "totalUsd": totals["costUsd"],
999 "unpricedCalls": totals["unpricedCalls"],
1000 },
1001 "linesAdded": aggregate.lines["added"],
1002 "linesRemoved": aggregate.lines["removed"],
1003 "humanInteractions": sum(row["interactions"] for row in run_rows),
1004 "approvals": sum(row["approvals"] for row in run_rows),
1005 "humanWaitMs": sum(row["humanWaitMs"] for row in run_rows),
1006 "issuesResolved": len(aggregate.issues),
1007 "featuresDelivered": len(aggregate.features),
1008 "costPerFeatureUsd": round(totals["costUsd"] / len(aggregate.features), 6)
1009 if aggregate.features
1010 else None,
1011 },
1012 "tokens": {
1013 "byAgent": _finish_rows(tables.by["agent"]),
1014 "byModel": _finish_rows(tables.by["model"]),
1015 "byTask": _finish_rows(tables.by["task"]),
1016 "byPhase": _finish_rows(tables.by["phase"]),
1017 "byCallKind": _finish_rows(tables.by["callKind"]),
1018 "byProject": _finish_rows(tables.by["project"]),
1019 },
1020 "models": tables.model_rows(),
1021 "lines": aggregate.lines,
1022 "delivery": {"issuesResolved": aggregate.issues, "featuresDelivered": aggregate.features},
1023 "time": {
1024 "perPhase": [
1025 {
1026 "phase": phase,
1027 "attempts": len(phase_times[phase]),
1028 "totalMs": sum(phase_times[phase]),
1029 "meanMs": int(statistics.fmean(phase_times[phase])),
1030 }
1031 for phase in _PHASES
1032 if phase_times.get(phase)
1033 ],
1034 "agentCalls": {
1035 "count": len(durations),
1036 "totalMs": sum(durations),
1037 "meanMs": int(statistics.fmean(durations)) if durations else None,
1038 "p50Ms": _percentile(durations, 0.5),
1039 "p90Ms": _percentile(durations, 0.9),
1040 },
1041 "humanWaitMs": sum(row["humanWaitMs"] for row in run_rows),
1042 },
1043 "quality": {
1044 **quality,
1045 "approvedFirstTimeRate": round(quality["approvedFirstTime"] / decided, 4)
1046 if decided
1047 else None,
1048 "findingsByRule": sorted(
1049 aggregate.findings_by_rule.values(),
1050 key=lambda item: (-item["count"], item["ruleId"]),
1051 ),
1052 "findingsBySeverity": aggregate.findings_by_severity,
1053 "findingsByFile": sorted(
1054 aggregate.findings_by_file.values(),
1055 key=lambda item: (-item["count"], item["path"]),
1056 )[:_TOP_FILES],
1057 "openBlockingFindings": aggregate.open_blocking,
1058 "gateResults": {
1059 "latestByRun": dict(sorted(aggregate.gate_results.items())),
1060 "openFailed": aggregate.open_failed_gates,
1061 },
1062 "limitsReached": aggregate.limits,
1063 "blockedRunsDetail": aggregate.blocked_detail,
1064 },
1065 "friction": {
1066 "targets": {
1067 size: target.model_dump(mode="json", by_alias=True)
1068 for size, target in sorted(effective_targets.items())
1069 },
1070 "targetsSource": targets_source,
1071 "perTask": sorted(friction_rows, key=lambda item: (item["projectId"], item["taskId"])),
1072 "bySize": [by_size[size] for size in ("S", "M", "L") if size in by_size],
1073 },
1074 "trends": {
1075 "daily": [aggregate.daily[key] for key in sorted(aggregate.daily)],
1076 "weekly": [aggregate.weekly[key] for key in sorted(aggregate.weekly)],
1077 },
1078 "runs": run_rows,
1079 }
1080 report["summary"] = period_status(
1081 {
1082 "blockedRuns": quality["blockedRuns"],
1083 "rejectedRuns": quality["rejectedRuns"],
1084 "openBlockingFindings": aggregate.open_blocking,
1085 "openFailedGates": aggregate.open_failed_gates,
1086 "waitingDecision": quality["waitingDecision"],
1087 },
1088 len(runs),
1089 )
1090 return report
1093__all__ = [
1094 "SCHEMA_VERSION",
1095 "Filters",
1096 "ProjectSource",
1097 "build_report",
1098 "collect",
1099 "parse_since",
1100 "task_issues",
1101]