Coverage for src/governed_harness/metrics/render.py: 89%

90 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""The formats of ``harness metrics``: JSON (``metrics.json``), a Markdown summary, CSV (one 

2row per run), the Prometheus text exposition format and one self-contained HTML file.""" 

3 

4from __future__ import annotations 

5 

6import csv 

7import io 

8import json 

9from collections.abc import Callable 

10from typing import Any 

11 

12from governed_harness.metrics.html import render_html 

13 

14FORMATS = ("json", "md", "html", "csv", "prometheus") 

15 

16 

17def _fmt(value: Any) -> str: 

18 if value is None: 18 ↛ 19line 18 didn't jump to line 19 because the condition on line 18 was never true

19 return "-" 

20 if isinstance(value, bool): 20 ↛ 21line 20 didn't jump to line 21 because the condition on line 20 was never true

21 return "yes" if value else "no" 

22 if isinstance(value, float): 

23 return f"{value:,.4f}".rstrip("0").rstrip(".") if value else "0" 

24 if isinstance(value, int): 

25 return f"{value:,}" 

26 return str(value).replace("|", "\\|").replace("\n", " ") 

27 

28 

29def _table(headers: list[str], rows: list[list[Any]]) -> list[str]: 

30 lines = ["| " + " | ".join(headers) + " |", "|" + "|".join("---" for _ in headers) + "|"] 

31 lines.extend("| " + " | ".join(_fmt(cell) for cell in row) + " |" for row in rows) 

32 return lines 

33 

34 

35def render_json(report: dict[str, Any]) -> str: 

36 return json.dumps(report, indent=2, ensure_ascii=False, sort_keys=False) + "\n" 

37 

38 

39def render_markdown(report: dict[str, Any]) -> str: 

40 totals = report["totals"] 

41 lines = [ 

42 "# Harness metrics", 

43 "", 

44 f"Generated {report['generatedAt']} by harness {report['harnessVersion']} from " 

45 f"{totals['runs']} run(s) of {len(report['projects'])} project(s). " 

46 "Deterministic: no model was called.", 

47 "", 

48 "## Status of the period", 

49 "", 

50 f"**{report['summary']['status']}**. {report['summary']['rule']}", 

51 "", 

52 *_table( 

53 ["Condition", "Count", "Met"], 

54 [ 

55 [item["label"], item["count"], "yes" if item["met"] else "no"] 

56 for item in report["summary"]["conditions"] 

57 ], 

58 ), 

59 "", 

60 "## Totals", 

61 "", 

62 *_table( 

63 ["Measure", "Value"], 

64 [ 

65 ["Runs", totals["runs"]], 

66 ["Tasks", totals["tasks"]], 

67 ["Agent calls", totals["agentCalls"]], 

68 ["Input tokens", totals["tokens"]["input"]], 

69 ["Output tokens", totals["tokens"]["output"]], 

70 ["Cache tokens (in input)", totals["tokens"]["cache"]], 

71 ["Cache-read tokens (in cache)", totals["tokens"]["cacheRead"]], 

72 ["Reported cost (USD)", totals["cost"]["reportedUsd"]], 

73 ["Estimated cost (USD, estimated)", totals["cost"]["estimatedUsd"]], 

74 ["Calls without a price", totals["cost"]["unpricedCalls"]], 

75 ["Lines added", totals["linesAdded"]], 

76 ["Lines removed", totals["linesRemoved"]], 

77 ["Human interactions", totals["humanInteractions"]], 

78 ["Approvals", totals["approvals"]], 

79 ["Human wait (minutes)", round(totals["humanWaitMs"] / 60000, 2)], 

80 ["Issues resolved", totals["issuesResolved"]], 

81 ["Features delivered", totals["featuresDelivered"]], 

82 ["Cost per delivered feature (USD)", totals["costPerFeatureUsd"]], 

83 ], 

84 ), 

85 "", 

86 ] 

87 for title, key in ( 

88 ("Tokens by agent", "byAgent"), 

89 ("Tokens by model", "byModel"), 

90 ("Tokens by phase", "byPhase"), 

91 ("Tokens by task", "byTask"), 

92 ): 

93 rows = report["tokens"][key] 

94 lines += [f"## {title}", ""] 

95 if rows: 95 ↛ 112line 95 didn't jump to line 112 because the condition on line 95 was always true

96 lines += _table( 

97 ["Key", "Calls", "Input", "Output", "Cache", "Reported USD", "Estimated USD"], 

98 [ 

99 [ 

100 row["key"], 

101 row["calls"], 

102 row["input"], 

103 row["output"], 

104 row["cache"], 

105 row["reportedCostUsd"], 

106 row["estimatedCostUsd"], 

107 ] 

108 for row in rows 

109 ], 

110 ) 

111 else: 

112 lines.append("No agent call.") 

113 lines.append("") 

114 agent, person = report["lines"]["agent"], report["lines"]["person"] 

115 lines += [ 

116 "## Lines", 

117 "", 

118 *_table( 

119 ["Author", "Added", "Removed", "Files"], 

120 [ 

121 ["Agent invocation", agent["added"], agent["removed"], agent["files"]], 

122 ["Person (out of band)", person["added"], person["removed"], person["files"]], 

123 [ 

124 "Not attributed (no provenance)", 

125 report["lines"]["unattributed"]["added"], 

126 report["lines"]["unattributed"]["removed"], 

127 report["lines"]["unattributed"]["files"], 

128 ], 

129 ], 

130 ), 

131 "", 

132 "## Quality", 

133 "", 

134 ] 

135 quality = report["quality"] 

136 lines += _table( 

137 ["Measure", "Value"], 

138 [ 

139 ["Runs decided", quality["decided"]], 

140 ["Approved the first time", quality["approvedFirstTime"]], 

141 ["Approved the first time (rate)", quality["approvedFirstTimeRate"]], 

142 ["Corrections", quality["corrections"]], 

143 ["Requested changes", quality["requestedChanges"]], 

144 ["Blocked runs", quality["blockedRuns"]], 

145 ["Waiting for a decision", quality["waitingDecision"]], 

146 ["Rejected runs", quality["rejectedRuns"]], 

147 ["Closed runs", quality["closedRuns"]], 

148 ["Pre-authorised approvals applied", quality["preAuthorizedApprovals"]], 

149 ["Open HIGH or CRITICAL findings", quality["openBlockingFindings"]], 

150 *[[f"Findings {name}", count] for name, count in quality["findingsBySeverity"].items()], 

151 *[ 

152 [f"Limit reached: {name}", count] 

153 for name, count in quality["limitsReached"].items() 

154 ], 

155 ], 

156 ) 

157 lines += ["", "## Friction per task", ""] 

158 friction = report["friction"]["perTask"] 

159 if friction: 159 ↛ 188line 159 didn't jump to line 188 because the condition on line 159 was always true

160 lines += _table( 

161 [ 

162 "Task", 

163 "Size", 

164 "Lane", 

165 "Runs", 

166 "Interactions", 

167 "Approvals", 

168 "Minutes", 

169 "Wait minutes", 

170 "Over target", 

171 ], 

172 [ 

173 [ 

174 row["taskId"], 

175 row["size"], 

176 ", ".join(row["lanes"]) or "-", 

177 row["runs"], 

178 row["interactions"], 

179 row["approvals"], 

180 row["wallMinutes"], 

181 row["humanWaitMinutes"], 

182 ", ".join(row["overTarget"]) or "no", 

183 ] 

184 for row in friction 

185 ], 

186 ) 

187 else: 

188 lines.append("No task.") 

189 delivery = report["delivery"] 

190 lines += ["", "## Delivered", ""] 

191 if delivery["featuresDelivered"]: 191 ↛ 202line 191 didn't jump to line 202 because the condition on line 191 was always true

192 lines += [ 

193 f"- `{item['taskId']}` {item['title']} ({item['closedAt'][:10]})" 

194 + ( 

195 f", issues {', '.join(issue['issue'] for issue in item['issues'])}" 

196 if item["issues"] 

197 else "" 

198 ) 

199 for item in delivery["featuresDelivered"] 

200 ] 

201 else: 

202 lines.append("Nothing closed in the period.") 

203 lines += ["", "## Notes", "", *[f"- {note}" for note in report["notes"]], ""] 

204 narrative = report.get("narrative") 

205 if narrative: 

206 lines += [ 

207 "## Narrative summary", 

208 "", 

209 f"Generated on demand by `{narrative['command']}`; not verified by the harness.", 

210 "", 

211 narrative["text"].strip(), 

212 "", 

213 ] 

214 return "\n".join(lines) 

215 

216 

217def render_csv(report: dict[str, Any]) -> str: 

218 buffer = io.StringIO() 

219 rows = report["runs"] 

220 columns = [ 

221 "projectId", 

222 "runId", 

223 "taskId", 

224 "status", 

225 "phase", 

226 "size", 

227 "lane", 

228 "createdAt", 

229 "wallMs", 

230 "humanWaitMs", 

231 "agentCalls", 

232 "inputTokens", 

233 "outputTokens", 

234 "cacheTokens", 

235 "reportedCostUsd", 

236 "estimatedCostUsd", 

237 "interactions", 

238 "approvals", 

239 "corrections", 

240 "linesAdded", 

241 "linesRemoved", 

242 ] 

243 writer = csv.DictWriter(buffer, fieldnames=columns, lineterminator="\n") 

244 writer.writeheader() 

245 for row in rows: 

246 writer.writerow({column: row.get(column) for column in columns}) 

247 return buffer.getvalue() 

248 

249 

250def _label(value: Any) -> str: 

251 return str(value).replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ") 

252 

253 

254def render_prometheus(report: dict[str, Any]) -> str: 

255 lines: list[str] = [] 

256 

257 def metric( 

258 name: str, kind: str, help_text: str, samples: list[tuple[dict[str, Any], Any]] 

259 ) -> None: 

260 lines.append(f"# HELP {name} {help_text}") 

261 lines.append(f"# TYPE {name} {kind}") 

262 for labels, value in samples: 

263 rendered = ",".join(f'{key}="{_label(item)}"' for key, item in labels.items()) 

264 lines.append(f"{name}{{{rendered}}} {value}" if rendered else f"{name} {value}") 

265 

266 totals = report["totals"] 

267 metric("harness_runs", "gauge", "Runs in the report.", [({}, totals["runs"])]) 

268 metric("harness_tasks", "gauge", "Tasks in the report.", [({}, totals["tasks"])]) 

269 metric( 

270 "harness_runs_by_status", 

271 "gauge", 

272 "Runs by status.", 

273 [ 

274 ({"status": status}, sum(1 for row in report["runs"] if row["status"] == status)) 

275 for status in sorted({row["status"] for row in report["runs"]}) 

276 ], 

277 ) 

278 metric( 

279 "harness_agent_calls", 

280 "gauge", 

281 "Agent calls by model.", 

282 [({"model": row["key"]}, row["calls"]) for row in report["tokens"]["byModel"]], 

283 ) 

284 metric( 

285 "harness_tokens", 

286 "gauge", 

287 "Tokens by model and kind (cache is part of input).", 

288 [ 

289 ({"model": row["key"], "kind": kind}, row[kind]) 

290 for row in report["tokens"]["byModel"] 

291 for kind in ("input", "output", "reasoning", "cache") 

292 ], 

293 ) 

294 metric( 

295 "harness_cost_usd", 

296 "gauge", 

297 "Cost in USD by model; source=estimated comes from the price table.", 

298 [ 

299 ({"model": row["key"], "source": source}, row[key]) 

300 for row in report["tokens"]["byModel"] 

301 for source, key in (("reported", "reportedCostUsd"), ("estimated", "estimatedCostUsd")) 

302 ], 

303 ) 

304 metric( 

305 "harness_lines", 

306 "gauge", 

307 "Lines of the latest ChangeSet of each run by author kind.", 

308 [ 

309 ({"author": author, "change": change}, report["lines"][author][change]) 

310 for author in ("agent", "person", "unattributed") 

311 for change in ("added", "removed") 

312 ], 

313 ) 

314 metric( 

315 "harness_human_interactions", 

316 "gauge", 

317 "Human interactions over the runs (no per-person breakdown).", 

318 [({}, totals["humanInteractions"])], 

319 ) 

320 metric( 

321 "harness_human_wait_seconds", 

322 "gauge", 

323 "Time the runs waited for a person.", 

324 [({}, round(totals["humanWaitMs"] / 1000, 3))], 

325 ) 

326 metric( 

327 "harness_tasks_over_friction_target", 

328 "gauge", 

329 "Tasks over their friction target by size.", 

330 [({"size": row["size"]}, row["overTarget"]) for row in report["friction"]["bySize"]], 

331 ) 

332 return "\n".join(lines) + "\n" 

333 

334 

335RENDERERS: dict[str, Callable[[dict[str, Any]], str]] = { 

336 "json": render_json, 

337 "md": render_markdown, 

338 "html": render_html, 

339 "csv": render_csv, 

340 "prometheus": render_prometheus, 

341} 

342 

343 

344def render(report: dict[str, Any], fmt: str) -> str: 

345 if fmt not in RENDERERS: 345 ↛ 346line 345 didn't jump to line 346 because the condition on line 345 was never true

346 raise ValueError(f"--format is one of {', '.join(FORMATS)}, got {fmt!r}") 

347 return RENDERERS[fmt](report) 

348 

349 

350__all__ = [ 

351 "FORMATS", 

352 "render", 

353 "render_csv", 

354 "render_json", 

355 "render_markdown", 

356 "render_prometheus", 

357]