Coverage for src/governed_harness/application/metrics.py: 90%
94 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""``harness metrics`` and the metrics tab of the dashboard (#58, items 8 to 12)."""
3from __future__ import annotations
5import json
6import subprocess # nosec B404 - the narrative command is the person's configured CLI
7from datetime import UTC, datetime
8from pathlib import Path
9from typing import Any
11from governed_harness.configuration import ConfigurationResolver
12from governed_harness.configuration.friction import ModelPrice, NarrativeConfig
13from governed_harness.domain.errors import ConfigurationError, HarnessError
14from governed_harness.events.sqlite_store import SQLiteEventStore
15from governed_harness.forges import origin_url, parse_remote
16from governed_harness.metrics import (
17 Filters,
18 ProjectSource,
19 build_report,
20 collect,
21 load_prices,
22 render,
23)
24from governed_harness.runtime.process_runner import resolve_executable
25from governed_harness.storage.sqlite import SQLiteStateStore
27NARRATIVE_PROMPT = (
28 "You summarise the activity of a software team's governed agent runs for a period. Read "
29 "the JSON metrics below (computed deterministically by the harness) and write a short "
30 "narrative in plain prose: what was delivered, what it cost (say which costs are "
31 "estimated), where people waited and where the friction was over target, and the quality "
32 "signals. Use only the numbers given; do not invent figures. No per-person judgement.\n\n"
33)
34MAX_NARRATIVE_CHARS = 20_000
37def _forge(workspace: Path) -> dict[str, str] | None:
38 try:
39 url = origin_url(workspace)
40 location = parse_remote(url) if url else None
41 except Exception: # noqa: BLE001 - a missing remote only means no links
42 return None
43 if location is None: 43 ↛ 45line 43 didn't jump to line 45 because the condition on line 43 was always true
44 return None
45 return {"kind": location.kind, "host": location.host, "repository": location.repository}
48def _registry_sources(entries: list[dict[str, Any]], current: Path) -> list[ProjectSource]:
49 """The other projects of the run registry, each database once (the current one apart)."""
50 sources: list[ProjectSource] = []
51 seen = {str(current.resolve())}
52 for entry in entries:
53 database = Path(entry["database"])
54 if not database.is_file() or str(database.resolve()) in seen:
55 continue
56 seen.add(str(database.resolve()))
57 workspace = Path(str(entry.get("workspace") or ""))
58 sources.append(
59 ProjectSource(
60 project_id=str(entry["projectId"]),
61 workspace=str(workspace),
62 state=SQLiteStateStore(database),
63 events=SQLiteEventStore(database),
64 forge=_forge(workspace) if workspace.is_dir() else None,
65 )
66 )
67 return sources
70def _sources(path: Path, all_repos: bool) -> tuple[list[ProjectSource], dict[str, Any]]:
71 """The current project (with its configuration) and, with ``all_repos``, every project of
72 the run registry (``runtime.stateDir``)."""
73 from governed_harness.orchestration.engine_types import EnginePaths
74 from governed_harness.runtime.state_location import registered_projects
76 sources: list[ProjectSource] = []
77 resolved = ConfigurationResolver().resolve(path)
78 project = resolved.project
79 paths = EnginePaths.for_project(resolved)
80 friction = project.friction
81 current = ProjectSource(
82 project_id=project.project_id,
83 workspace=str(resolved.workspace_root),
84 state=SQLiteStateStore(paths.database),
85 events=SQLiteEventStore(paths.database),
86 targets=friction.targets if friction else None,
87 forge=_forge(resolved.workspace_root),
88 )
89 sources.append(current)
90 if all_repos:
91 sources.extend(_registry_sources(registered_projects(), paths.database))
92 settings: dict[str, Any] = {
93 "prices": dict(project.metrics.prices or {}) if project.metrics else {},
94 "narrative": project.metrics.narrative if project.metrics else None,
95 "targets": friction.targets if friction else None,
96 "harnessDir": paths.harness_dir,
97 }
98 return sources, settings
101def metrics_report(
102 path: Path,
103 *,
104 filters: Filters,
105 prices_file: Path | None = None,
106 now: datetime | None = None,
107) -> tuple[dict[str, Any], dict[str, Any]]:
108 """The report and the settings it was computed with (zero model calls)."""
109 sources, settings = _sources(path, filters.all_repos)
110 try:
111 prices: dict[str, ModelPrice] = dict(settings["prices"])
112 if prices_file is not None:
113 prices.update(load_prices(prices_file))
114 runs = collect(sources, filters)
115 projects = [
116 {
117 "projectId": source.project_id,
118 "workspace": source.workspace,
119 "runs": sum(1 for run in runs if run.project_id == source.project_id),
120 }
121 for source in sources
122 ]
123 report = build_report(
124 runs,
125 filters=filters,
126 prices=prices,
127 targets=settings["targets"],
128 projects=projects,
129 now=now,
130 )
131 report["prices"] = {
132 key: value.model_dump(mode="json", by_alias=True)
133 for key, value in sorted(prices.items())
134 }
135 return report, settings
136 finally:
137 for source in sources:
138 source.state.close()
139 source.events.close()
142def run_narrative(
143 report: dict[str, Any], narrative: NarrativeConfig, workspace: Path
144) -> dict[str, Any]:
145 """The optional narrative summary: one call, on demand, to the configured command, which
146 reads the prompt and the metrics on standard input and prints the summary."""
147 compact = {key: report[key] for key in ("totals", "quality", "friction", "models", "trends")}
148 payload = NARRATIVE_PROMPT + json.dumps(compact, ensure_ascii=False)
149 started = datetime.now(UTC)
150 try:
151 result = subprocess.run( # nosec B603 - argv from project.yaml, no shell
152 [resolve_executable(narrative.command[0]), *narrative.command[1:]],
153 input=payload.encode("utf-8"),
154 capture_output=True,
155 cwd=workspace,
156 timeout=narrative.timeout,
157 check=False,
158 )
159 except (OSError, subprocess.TimeoutExpired) as error:
160 raise HarnessError(f"the narrative command did not run: {error}") from error
161 if result.returncode != 0: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true
162 tail = result.stderr.decode("utf-8", "replace").strip()[-500:]
163 raise HarnessError(f"the narrative command exited with {result.returncode}: {tail}")
164 text = result.stdout.decode("utf-8", "replace").strip()[:MAX_NARRATIVE_CHARS]
165 if not text: 165 ↛ 166line 165 didn't jump to line 166 because the condition on line 165 was never true
166 raise HarnessError("the narrative command printed nothing")
167 return {
168 "command": " ".join(narrative.command),
169 "generatedAt": started.isoformat(),
170 "text": text,
171 "promptCharacters": len(payload),
172 }
175def write_reports(
176 report: dict[str, Any], directory: Path, formats: tuple[str, ...] = ("json", "md", "html")
177) -> dict[str, str]:
178 directory.mkdir(parents=True, exist_ok=True)
179 names = {
180 "json": "metrics.json",
181 "md": "metrics.md",
182 "html": "metrics.html",
183 "csv": "metrics-runs.csv",
184 "prometheus": "metrics.prom",
185 }
186 written: dict[str, str] = {}
187 for fmt in formats:
188 target = directory / names[fmt]
189 target.write_text(render(report, fmt), encoding="utf-8")
190 written[fmt] = str(target)
191 return written
194def narrative_settings(settings: dict[str, Any]) -> NarrativeConfig:
195 narrative = settings.get("narrative")
196 if not isinstance(narrative, NarrativeConfig):
197 raise ConfigurationError(
198 "--narrative needs metrics.narrative.command in project.yaml (a command that reads "
199 "the prompt on standard input and prints the summary)"
200 )
201 return narrative
204__all__ = ["metrics_report", "narrative_settings", "run_narrative", "write_reports"]