Coverage for src/governed_harness/application/metrics.py: 90%

94 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""``harness metrics`` and the metrics tab of the dashboard (#58, items 8 to 12).""" 

2 

3from __future__ import annotations 

4 

5import json 

6import subprocess # nosec B404 - the narrative command is the person's configured CLI 

7from datetime import UTC, datetime 

8from pathlib import Path 

9from typing import Any 

10 

11from governed_harness.configuration import ConfigurationResolver 

12from governed_harness.configuration.friction import ModelPrice, NarrativeConfig 

13from governed_harness.domain.errors import ConfigurationError, HarnessError 

14from governed_harness.events.sqlite_store import SQLiteEventStore 

15from governed_harness.forges import origin_url, parse_remote 

16from governed_harness.metrics import ( 

17 Filters, 

18 ProjectSource, 

19 build_report, 

20 collect, 

21 load_prices, 

22 render, 

23) 

24from governed_harness.runtime.process_runner import resolve_executable 

25from governed_harness.storage.sqlite import SQLiteStateStore 

26 

27NARRATIVE_PROMPT = ( 

28 "You summarise the activity of a software team's governed agent runs for a period. Read " 

29 "the JSON metrics below (computed deterministically by the harness) and write a short " 

30 "narrative in plain prose: what was delivered, what it cost (say which costs are " 

31 "estimated), where people waited and where the friction was over target, and the quality " 

32 "signals. Use only the numbers given; do not invent figures. No per-person judgement.\n\n" 

33) 

34MAX_NARRATIVE_CHARS = 20_000 

35 

36 

37def _forge(workspace: Path) -> dict[str, str] | None: 

38 try: 

39 url = origin_url(workspace) 

40 location = parse_remote(url) if url else None 

41 except Exception: # noqa: BLE001 - a missing remote only means no links 

42 return None 

43 if location is None: 43 ↛ 45line 43 didn't jump to line 45 because the condition on line 43 was always true

44 return None 

45 return {"kind": location.kind, "host": location.host, "repository": location.repository} 

46 

47 

48def _registry_sources(entries: list[dict[str, Any]], current: Path) -> list[ProjectSource]: 

49 """The other projects of the run registry, each database once (the current one apart).""" 

50 sources: list[ProjectSource] = [] 

51 seen = {str(current.resolve())} 

52 for entry in entries: 

53 database = Path(entry["database"]) 

54 if not database.is_file() or str(database.resolve()) in seen: 

55 continue 

56 seen.add(str(database.resolve())) 

57 workspace = Path(str(entry.get("workspace") or "")) 

58 sources.append( 

59 ProjectSource( 

60 project_id=str(entry["projectId"]), 

61 workspace=str(workspace), 

62 state=SQLiteStateStore(database), 

63 events=SQLiteEventStore(database), 

64 forge=_forge(workspace) if workspace.is_dir() else None, 

65 ) 

66 ) 

67 return sources 

68 

69 

70def _sources(path: Path, all_repos: bool) -> tuple[list[ProjectSource], dict[str, Any]]: 

71 """The current project (with its configuration) and, with ``all_repos``, every project of 

72 the run registry (``runtime.stateDir``).""" 

73 from governed_harness.orchestration.engine_types import EnginePaths 

74 from governed_harness.runtime.state_location import registered_projects 

75 

76 sources: list[ProjectSource] = [] 

77 resolved = ConfigurationResolver().resolve(path) 

78 project = resolved.project 

79 paths = EnginePaths.for_project(resolved) 

80 friction = project.friction 

81 current = ProjectSource( 

82 project_id=project.project_id, 

83 workspace=str(resolved.workspace_root), 

84 state=SQLiteStateStore(paths.database), 

85 events=SQLiteEventStore(paths.database), 

86 targets=friction.targets if friction else None, 

87 forge=_forge(resolved.workspace_root), 

88 ) 

89 sources.append(current) 

90 if all_repos: 

91 sources.extend(_registry_sources(registered_projects(), paths.database)) 

92 settings: dict[str, Any] = { 

93 "prices": dict(project.metrics.prices or {}) if project.metrics else {}, 

94 "narrative": project.metrics.narrative if project.metrics else None, 

95 "targets": friction.targets if friction else None, 

96 "harnessDir": paths.harness_dir, 

97 } 

98 return sources, settings 

99 

100 

101def metrics_report( 

102 path: Path, 

103 *, 

104 filters: Filters, 

105 prices_file: Path | None = None, 

106 now: datetime | None = None, 

107) -> tuple[dict[str, Any], dict[str, Any]]: 

108 """The report and the settings it was computed with (zero model calls).""" 

109 sources, settings = _sources(path, filters.all_repos) 

110 try: 

111 prices: dict[str, ModelPrice] = dict(settings["prices"]) 

112 if prices_file is not None: 

113 prices.update(load_prices(prices_file)) 

114 runs = collect(sources, filters) 

115 projects = [ 

116 { 

117 "projectId": source.project_id, 

118 "workspace": source.workspace, 

119 "runs": sum(1 for run in runs if run.project_id == source.project_id), 

120 } 

121 for source in sources 

122 ] 

123 report = build_report( 

124 runs, 

125 filters=filters, 

126 prices=prices, 

127 targets=settings["targets"], 

128 projects=projects, 

129 now=now, 

130 ) 

131 report["prices"] = { 

132 key: value.model_dump(mode="json", by_alias=True) 

133 for key, value in sorted(prices.items()) 

134 } 

135 return report, settings 

136 finally: 

137 for source in sources: 

138 source.state.close() 

139 source.events.close() 

140 

141 

142def run_narrative( 

143 report: dict[str, Any], narrative: NarrativeConfig, workspace: Path 

144) -> dict[str, Any]: 

145 """The optional narrative summary: one call, on demand, to the configured command, which 

146 reads the prompt and the metrics on standard input and prints the summary.""" 

147 compact = {key: report[key] for key in ("totals", "quality", "friction", "models", "trends")} 

148 payload = NARRATIVE_PROMPT + json.dumps(compact, ensure_ascii=False) 

149 started = datetime.now(UTC) 

150 try: 

151 result = subprocess.run( # nosec B603 - argv from project.yaml, no shell 

152 [resolve_executable(narrative.command[0]), *narrative.command[1:]], 

153 input=payload.encode("utf-8"), 

154 capture_output=True, 

155 cwd=workspace, 

156 timeout=narrative.timeout, 

157 check=False, 

158 ) 

159 except (OSError, subprocess.TimeoutExpired) as error: 

160 raise HarnessError(f"the narrative command did not run: {error}") from error 

161 if result.returncode != 0: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true

162 tail = result.stderr.decode("utf-8", "replace").strip()[-500:] 

163 raise HarnessError(f"the narrative command exited with {result.returncode}: {tail}") 

164 text = result.stdout.decode("utf-8", "replace").strip()[:MAX_NARRATIVE_CHARS] 

165 if not text: 165 ↛ 166line 165 didn't jump to line 166 because the condition on line 165 was never true

166 raise HarnessError("the narrative command printed nothing") 

167 return { 

168 "command": " ".join(narrative.command), 

169 "generatedAt": started.isoformat(), 

170 "text": text, 

171 "promptCharacters": len(payload), 

172 } 

173 

174 

175def write_reports( 

176 report: dict[str, Any], directory: Path, formats: tuple[str, ...] = ("json", "md", "html") 

177) -> dict[str, str]: 

178 directory.mkdir(parents=True, exist_ok=True) 

179 names = { 

180 "json": "metrics.json", 

181 "md": "metrics.md", 

182 "html": "metrics.html", 

183 "csv": "metrics-runs.csv", 

184 "prometheus": "metrics.prom", 

185 } 

186 written: dict[str, str] = {} 

187 for fmt in formats: 

188 target = directory / names[fmt] 

189 target.write_text(render(report, fmt), encoding="utf-8") 

190 written[fmt] = str(target) 

191 return written 

192 

193 

194def narrative_settings(settings: dict[str, Any]) -> NarrativeConfig: 

195 narrative = settings.get("narrative") 

196 if not isinstance(narrative, NarrativeConfig): 

197 raise ConfigurationError( 

198 "--narrative needs metrics.narrative.command in project.yaml (a command that reads " 

199 "the prompt on standard input and prints the summary)" 

200 ) 

201 return narrative 

202 

203 

204__all__ = ["metrics_report", "narrative_settings", "run_narrative", "write_reports"]