Coverage for src/governed_harness/metrics/html.py: 93%
174 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""One self-contained HTML file of the metrics (#58, item 8): inline CSS and inline SVG charts,
2no script, no font, no image and nothing fetched from the network, so it opens offline from
3the file system. Every chart has a text title, a description and a table with the same numbers;
4identity is never carried by color alone (legends and labels), light and dark color schemes
5are both defined. Links to issues and pull requests are plain anchors: following one is the
6reader's choice, the page itself loads nothing."""
8from __future__ import annotations
10import html
11from typing import Any
13_SECTION_END = "</section>"
14_COST_TITLE = "Cost and tokens"
15_LIMITS_TITLE = "Limits reached"
16_CSS = """
17:root { color-scheme: light; --surface: #fcfcfb; --panel: #ffffff; --ink: #0b0b0b;
18 --ink-2: #52514e; --grid: #d9d8d3; --s1: #2a78d6; --s2: #eb6834; --s3: #1baf7a;
19 --over: #b42318; --ok: #1a7f4b; }
20@media (prefers-color-scheme: dark) { :root { color-scheme: dark; --surface: #1a1a19;
21 --panel: #232322; --ink: #ffffff; --ink-2: #c3c2b7; --grid: #3d3c39; --s1: #3987e5;
22 --s2: #d95926; --s3: #199e70; --over: #ff8a80; --ok: #4cc38a; } }
23* { box-sizing: border-box; }
24body { margin: 0; background: var(--surface); color: var(--ink);
25 font: 15px/1.5 system-ui, -apple-system, "Segoe UI", sans-serif; }
26header, main { max-width: 1120px; margin: 0 auto; padding: 16px; }
27h1 { font-size: 26px; margin: 8px 0 4px; } h2 { font-size: 19px; margin: 28px 0 8px; }
28p.lead, .muted { color: var(--ink-2); }
29.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 12px; }
30.card { background: var(--panel); border: 1px solid var(--grid); border-radius: 10px; padding: 12px; }
31.card .value { font-size: 24px; font-weight: 650; font-variant-numeric: tabular-nums; }
32.card .label { color: var(--ink-2); font-size: 13px; }
33section { background: var(--panel); border: 1px solid var(--grid); border-radius: 10px;
34 padding: 12px 16px; margin-top: 16px; overflow-x: auto; }
35table { border-collapse: collapse; width: 100%; font-size: 14px; font-variant-numeric: tabular-nums; }
36caption { text-align: left; color: var(--ink-2); padding: 4px 0; }
37th, td { text-align: left; padding: 4px 8px; border-bottom: 1px solid var(--grid); vertical-align: top; }
38td.num, th.num { text-align: right; }
39.over { color: var(--over); font-weight: 650; }
40svg { display: block; max-width: 100%; height: auto; }
41svg text { fill: var(--ink-2); font: 12px system-ui, sans-serif; }
42svg .value { fill: var(--ink); }
43.legend { display: flex; gap: 16px; flex-wrap: wrap; font-size: 13px; color: var(--ink-2); margin: 4px 0; }
44.swatch { display: inline-block; width: 12px; height: 12px; border-radius: 3px; margin-right: 6px;
45 vertical-align: -1px; }
46details { margin-top: 8px; } summary { cursor: pointer; color: var(--ink-2); }
47a { color: var(--s1); }
48.status { font-size: 22px; margin: 4px 0; padding: 6px 12px; border-radius: 8px; display: inline-block;
49 border: 2px solid var(--grid); }
50.status-ok { border-color: var(--ok); color: var(--ok); }
51.status-attn { border-color: var(--over); color: var(--over); }
52nav ul { list-style: none; display: flex; flex-wrap: wrap; gap: 8px 16px; padding: 0; margin: 12px 0 0; }
53nav a { text-decoration: none; border-bottom: 1px solid var(--grid); }
54.dimension { margin-top: 12px; } .dim-title { font-size: 15px; margin: 8px 0; }
55.dim-title a { color: var(--ink); text-decoration: none; }
56.card .hint { color: var(--ink-2); font-size: 12px; }
57.sub { margin-top: 16px; } h3 { font-size: 16px; margin: 12px 0 6px; }
58@media (max-width: 600px) { header, main { padding: 12px; } .card .value { font-size: 20px; } }
59@media (forced-colors: active) { .swatch { forced-color-adjust: none; border: 1px solid CanvasText; } }
60"""
63def _e(value: Any) -> str:
64 return html.escape("-" if value is None else str(value), quote=True)
67def _num(value: Any) -> str:
68 if value is None: 68 ↛ 69line 68 didn't jump to line 69 because the condition on line 68 was never true
69 return "-"
70 if isinstance(value, float):
71 return f"{value:,.4f}".rstrip("0").rstrip(".") if value else "0"
72 if isinstance(value, int): 72 ↛ 74line 72 didn't jump to line 74 because the condition on line 72 was always true
73 return f"{value:,}"
74 return _e(value)
77def _table(caption: str, headers: list[str], rows: list[list[Any]], numeric: set[int]) -> str:
78 head = "".join(
79 f'<th scope="col"{' class="num"' if index in numeric else ""}>{_e(item)}</th>'
80 for index, item in enumerate(headers)
81 )
82 body = "".join(
83 "<tr>"
84 + "".join(
85 f'<td class="num">{_num(cell)}</td>' if index in numeric else f"<td>{cell}</td>"
86 for index, cell in enumerate(row)
87 )
88 + "</tr>"
89 for row in rows
90 )
91 if not rows:
92 body = f'<tr><td colspan="{len(headers)}">No data in the period.</td></tr>'
93 return f"<table><caption>{_e(caption)}</caption><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table>"
96def _bars(
97 chart_id: str,
98 title: str,
99 description: str,
100 rows: list[tuple[str, list[float]]],
101 series: list[tuple[str, str]],
102 unit: str,
103) -> str:
104 """Horizontal stacked bars (one row per key), labelled with the total; a native hover
105 title on every segment."""
106 if not rows:
107 return f'<p class="muted">{_e(title)}: no data in the period.</p>'
108 rows = rows[:12]
109 width, label_w, bar_h, gap = 760, 190, 22, 10
110 plot_w = width - label_w - 90
111 peak = max(sum(values) for _, values in rows) or 1
112 height = len(rows) * (bar_h + gap) + 8
113 parts = [
114 f'<svg viewBox="0 0 {width} {height}" role="img" aria-labelledby="{chart_id}-t '
115 f'{chart_id}-d" width="{width}" height="{height}">',
116 f'<title id="{chart_id}-t">{_e(title)}</title><desc id="{chart_id}-d">{_e(description)}</desc>',
117 f'<line x1="{label_w}" y1="0" x2="{label_w}" y2="{height}" stroke="var(--grid)"/>',
118 ]
119 for index, (key, values) in enumerate(rows):
120 y = 4 + index * (bar_h + gap)
121 label = key if len(key) <= 28 else key[:27] + "…"
122 parts.append(
123 f'<text x="{label_w - 8}" y="{y + bar_h * 0.7:.1f}" text-anchor="end">{_e(label)}</text>'
124 )
125 x = float(label_w)
126 for (name, color), value in zip(series, values, strict=True):
127 if value <= 0:
128 continue
129 w = max(1.0, plot_w * value / peak)
130 parts.append(
131 f'<rect x="{x:.1f}" y="{y}" width="{max(1.0, w - 2):.1f}" height="{bar_h}" '
132 f'rx="4" fill="var({color})"><title>{_e(key)}: {_e(name)} {_num(value)} '
133 f"{_e(unit)}</title></rect>"
134 )
135 x += w
136 parts.append(
137 f'<text class="value" x="{x + 6:.1f}" y="{y + bar_h * 0.7:.1f}">'
138 f"{_num(sum(values))}</text>"
139 )
140 parts.append("</svg>")
141 legend = ""
142 if len(series) > 1:
143 legend = (
144 '<div class="legend">'
145 + "".join(
146 f'<span><span class="swatch" style="background:var({color})"></span>{_e(name)}</span>'
147 for name, color in series
148 )
149 + "</div>"
150 )
151 return legend + "".join(parts)
154def _columns(chart_id: str, title: str, description: str, points: list[tuple[str, float]]) -> str:
155 """Vertical columns over time (one series)."""
156 if not points: 156 ↛ 157line 156 didn't jump to line 157 because the condition on line 156 was never true
157 return f'<p class="muted">{_e(title)}: no data in the period.</p>'
158 points = points[-60:]
159 width, height, left, bottom = 760, 200, 40, 36
160 plot_w, plot_h = width - left - 10, height - bottom - 10
161 peak = max(value for _, value in points) or 1
162 step = plot_w / len(points)
163 bar = max(2.0, min(28.0, step - 4))
164 parts = [
165 f'<svg viewBox="0 0 {width} {height}" role="img" aria-labelledby="{chart_id}-t '
166 f'{chart_id}-d" width="{width}" height="{height}">',
167 f'<title id="{chart_id}-t">{_e(title)}</title><desc id="{chart_id}-d">{_e(description)}</desc>',
168 f'<line x1="{left}" y1="{10 + plot_h}" x2="{width - 10}" y2="{10 + plot_h}" stroke="var(--grid)"/>',
169 f'<text x="{left - 6}" y="16" text-anchor="end">{_num(peak)}</text>',
170 f'<text x="{left - 6}" y="{10 + plot_h}" text-anchor="end">0</text>',
171 ]
172 every = max(1, len(points) // 8)
173 for index, (label, value) in enumerate(points):
174 h = plot_h * value / peak
175 x = left + index * step + (step - bar) / 2
176 parts.append(
177 f'<rect x="{x:.1f}" y="{10 + plot_h - h:.1f}" width="{bar:.1f}" height="{max(h, 0):.1f}" '
178 f'rx="3" fill="var(--s1)"><title>{_e(label)}: {_num(value)}</title></rect>'
179 )
180 if index % every == 0: 180 ↛ 173line 180 didn't jump to line 173 because the condition on line 180 was always true
181 parts.append(
182 f'<text x="{x + bar / 2:.1f}" y="{height - 14}" text-anchor="middle">'
183 f"{_e(label[5:] if len(label) == 10 else label)}</text>"
184 )
185 parts.append("</svg>")
186 return "".join(parts)
189def _link(text: str, url: str | None) -> str:
190 if not url: 190 ↛ 192line 190 didn't jump to line 192 because the condition on line 190 was always true
191 return _e(text)
192 return f'<a href="{_e(url)}" rel="noopener noreferrer">{_e(text)}</a>'
195def _sub(section_id: str, title: str, body: str) -> str:
196 """A subsection of a group: a heading with an anchor and its content."""
197 return f'<div class="sub"><h3 id="{section_id}">{_e(title)}</h3>{body}</div>'
200def _group(section_id: str, title: str, intro: str, body: list[str]) -> str:
201 return (
202 f'<section aria-labelledby="{section_id}"><h2 id="{section_id}">{_e(title)}</h2>'
203 f'<p class="muted">{_e(intro)}</p>' + "".join(body) + _SECTION_END
204 )
207def _card(label: str, value: Any, hint: str = "") -> str:
208 small = f'<div class="hint">{_e(hint)}</div>' if hint else ""
209 return (
210 f'<div class="card"><div class="value">{_num(value)}</div>'
211 f'<div class="label">{_e(label)}</div>{small}</div>'
212 )
215_STATUS_TEXT = {
216 "CLEAR": ("Clear", "ok", "✔"),
217 "ATTENTION": ("Needs attention", "attn", "▲"),
218 "NO_DATA": ("No data", "none", "–"),
219}
222def _overview(report: dict[str, Any]) -> str:
223 """Status of the period (derived from the report's own numbers) and its conditions."""
224 summary = report["summary"]
225 label, kind, mark = _STATUS_TEXT[summary["status"]]
226 rows = [
227 [
228 _e(item["label"]),
229 item["count"],
230 "met" if item["met"] else '<span class="over">not met</span>',
231 ]
232 for item in summary["conditions"]
233 ]
234 waiting = summary["waitingDecision"]
235 return (
236 '<section aria-labelledby="s-overview"><h2 id="s-overview">Status of the period</h2>'
237 f'<p class="status status-{kind}"><span aria-hidden="true">{mark}</span> '
238 f"<strong>{_e(label)}</strong></p>"
239 f'<p class="muted">{_e(summary["rule"])} {waiting} run(s) waiting for a person are '
240 "not counted as a problem.</p>"
241 + _table("Conditions of the status", ["Condition", "Count", "Result"], rows, {1})
242 + _SECTION_END
243 )
246def _dimension_cards(report: dict[str, Any]) -> str:
247 """Summary cards per dimension, like the measures of a project overview."""
248 totals, quality = report["totals"], report["quality"]
249 time, friction = report["time"], report["friction"]
250 over = sum(1 for row in friction["perTask"] if row["overTarget"])
251 rate = quality["approvedFirstTimeRate"]
252 groups = [
253 (
254 _COST_TITLE,
255 "#s-cost",
256 [
257 _card("Agent calls", totals["agentCalls"]),
258 _card("Tokens (input + output)", totals["tokens"]["total"]),
259 _card("Reported cost (USD)", totals["cost"]["reportedUsd"]),
260 _card("Estimated cost (USD, estimated)", totals["cost"]["estimatedUsd"]),
261 _card("Cost per delivered feature (USD)", totals["costPerFeatureUsd"]),
262 ],
263 ),
264 (
265 "Time",
266 "#s-time",
267 [
268 _card("Human wait (minutes)", round(totals["humanWaitMs"] / 60000, 2)),
269 _card("Agent call, median (ms)", time["agentCalls"]["p50Ms"]),
270 _card("Agent call, 90th percentile (ms)", time["agentCalls"]["p90Ms"]),
271 ],
272 ),
273 (
274 "Friction",
275 "#s-friction",
276 [
277 _card("Runs", totals["runs"]),
278 _card("Tasks", totals["tasks"]),
279 _card("Human interactions", totals["humanInteractions"]),
280 _card("Tasks over their target", over),
281 _card("Corrections", quality["corrections"]),
282 ],
283 ),
284 (
285 "Quality",
286 "#s-quality",
287 [
288 _card("Approved the first time (rate)", rate),
289 _card("Blocked runs", quality["blockedRuns"]),
290 _card("Rejected runs", quality["rejectedRuns"]),
291 _card("Open HIGH or CRITICAL findings", quality["openBlockingFindings"]),
292 _card(_LIMITS_TITLE, sum(quality["limitsReached"].values())),
293 ],
294 ),
295 (
296 "Delivery",
297 "#s-delivery",
298 [
299 _card("Features delivered", totals["featuresDelivered"]),
300 _card("Issues resolved", totals["issuesResolved"]),
301 _card("Lines added", totals["linesAdded"]),
302 _card("Lines removed", totals["linesRemoved"]),
303 ],
304 ),
305 ]
306 body = "".join(
307 f'<div class="dimension"><h3 class="dim-title"><a href="{target}">{_e(title)}</a></h3>'
308 f'<div class="cards">{"".join(cards)}</div></div>'
309 for title, target, cards in groups
310 )
311 return (
312 '<section aria-labelledby="s-summary"><h2 id="s-summary">Summary by dimension</h2>'
313 + body
314 + _SECTION_END
315 )
318_NAV = (
319 ("#s-overview", "Status"),
320 ("#s-summary", "Summary"),
321 ("#s-cost", _COST_TITLE),
322 ("#s-time", "Time"),
323 ("#s-friction", "Friction"),
324 ("#s-quality", "Quality"),
325 ("#s-delivery", "Delivery"),
326 ("#s-trends", "Trends"),
327 ("#s-notes", "Data quality"),
328)
331def _nav(narrative: bool) -> str:
332 items = list(_NAV)
333 if narrative: 333 ↛ 334line 333 didn't jump to line 334 because the condition on line 333 was never true
334 items.insert(-1, ("#s-narrative", "Narrative"))
335 links = "".join(f'<li><a href="{target}">{_e(text)}</a></li>' for target, text in items)
336 return f'<nav aria-label="Sections of the report"><ul>{links}</ul></nav>'
339def _token_block(report: dict[str, Any], chart_id: str, title: str, key: str) -> str:
340 rows = report["tokens"][key]
341 chart = _bars(
342 f"chart-{chart_id}",
343 title,
344 f"Input and output tokens of the agent calls, by {chart_id}; the table below has "
345 "the same numbers with cache tokens and costs.",
346 [(row["key"], [row["input"], row["output"]]) for row in rows],
347 [("input", "--s1"), ("output", "--s2")],
348 "tokens",
349 )
350 table = _table(
351 title,
352 [
353 chart_id.capitalize(),
354 "Calls",
355 "Input",
356 "Output",
357 "Cache",
358 "Reported USD",
359 "Estimated USD",
360 "Unpriced calls",
361 ],
362 [
363 [
364 _e(row["key"]),
365 row["calls"],
366 row["input"],
367 row["output"],
368 row["cache"],
369 row["reportedCostUsd"],
370 row["estimatedCostUsd"],
371 row["unpricedCalls"],
372 ]
373 for row in rows
374 ],
375 {1, 2, 3, 4, 5, 6, 7},
376 )
377 return _sub(
378 f"h-{chart_id}", title, f"{chart}<details><summary>Table</summary>{table}</details>"
379 )
382def _cost_group(report: dict[str, Any]) -> str:
383 tokens = report["tokens"]
384 blocks = [
385 _token_block(report, chart_id, title, key)
386 for chart_id, title, key in (
387 ("model", "Tokens by model", "byModel"),
388 ("agent", "Tokens by agent", "byAgent"),
389 ("task", "Tokens by task", "byTask"),
390 ("phase", "Tokens by phase", "byPhase"),
391 )
392 ]
393 cost_rows = [
394 (row["key"], [row["reportedCostUsd"], row["estimatedCostUsd"]])
395 for row in tokens["byAgent"]
396 if row["reportedCostUsd"] or row["estimatedCostUsd"]
397 ]
398 blocks.append(
399 _sub(
400 "h-cost",
401 "Cost by agent",
402 '<p class="muted">Estimated costs come from the price table for calls that '
403 "reported tokens without a cost; they are estimates, not charges.</p>"
404 + _bars(
405 "chart-cost",
406 "Cost by agent in US dollars",
407 "Reported and estimated cost per agent provider.",
408 cost_rows,
409 [("reported", "--s1"), ("estimated", "--s2")],
410 "USD",
411 ),
412 )
413 )
414 blocks.append(
415 _sub(
416 "h-models",
417 "Models used",
418 _table(
419 "Models used",
420 ["Provider", "Model", "Calls", "Cost source"],
421 [
422 [_e(m["provider"]), _e(m["model"]), m["calls"], _e(m["cost"])]
423 for m in report["models"]
424 ],
425 {2},
426 ),
427 )
428 )
429 return _group(
430 "s-cost",
431 _COST_TITLE,
432 "What the agent calls consumed, by model, agent, task and phase.",
433 blocks,
434 )
437def _time_group(report: dict[str, Any]) -> str:
438 time = report["time"]
439 calls = time["agentCalls"]
440 body = _table(
441 "Time per phase",
442 ["Phase", "Attempts", "Total ms", "Mean ms"],
443 [
444 [_e(row["phase"]), row["attempts"], row["totalMs"], row["meanMs"]]
445 for row in time["perPhase"]
446 ],
447 {1, 2, 3},
448 ) + _table(
449 "Agent calls",
450 ["Calls", "Total ms", "Mean ms", "Median ms", "90th percentile ms"],
451 [[calls["count"], calls["totalMs"], calls["meanMs"], calls["p50Ms"], calls["p90Ms"]]],
452 {0, 1, 2, 3, 4},
453 )
454 return _group(
455 "s-time",
456 "Time",
457 "Where the elapsed time went: phases and agent calls.",
458 [_sub("h-time", "Time", body)],
459 )
462def _friction_group(report: dict[str, Any]) -> str:
463 friction = report["friction"]
464 rows = []
465 for row in friction["perTask"]:
466 over = set(row["overTarget"])
467 target = row["target"] or {}
468 rows.append(
469 [
470 f"<code>{_e(row['taskId'])}</code> {_e(row['title'])}",
471 _e(row["size"]),
472 _e(", ".join(row["lanes"]) or "-"),
473 row["runs"],
474 f'<span class="over">{row["interactions"]} (over {target.get("interactions")})</span>'
475 if "interactions" in over
476 else row["interactions"],
477 row["approvals"],
478 row["corrections"],
479 f'<span class="over">{row["wallMinutes"]} (over {target.get("minutes")})</span>'
480 if "minutes" in over
481 else row["wallMinutes"],
482 row["humanWaitMinutes"],
483 ]
484 )
485 note = (
486 f'<p class="muted">Targets ({_e(friction["targetsSource"])}): '
487 + "; ".join(
488 f"{_e(size)}: {_e(values.get('interactions'))} interaction(s), "
489 f"{_e(values.get('minutes'))} minutes"
490 for size, values in friction["targets"].items()
491 )
492 + ". A value over its target is marked “over”.</p>"
493 )
494 table = _table(
495 "Friction per task",
496 [
497 "Task",
498 "Size",
499 "Lane",
500 "Runs",
501 "Interactions",
502 "Approvals",
503 "Corrections",
504 "Minutes",
505 "Wait minutes",
506 ],
507 rows,
508 {3, 5, 6, 8},
509 )
510 return _group(
511 "s-friction",
512 "Friction",
513 "How much the people had to intervene, task by task, against the target of each size.",
514 [_sub("h-friction", "Friction per task", note + table)],
515 )
518def _findings_blocks(quality: dict[str, Any]) -> list[str]:
519 severity = quality["findingsBySeverity"]
520 chart = _bars(
521 "chart-severity",
522 "Findings by severity",
523 "Number of findings recorded by the validators, by severity.",
524 [(name, [float(count)]) for name, count in severity.items()],
525 [("findings", "--s2")],
526 "findings",
527 )
528 by_rule = _table(
529 "Findings by rule",
530 ["Rule", "Count", "By severity"],
531 [
532 [
533 f"<code>{_e(row['ruleId'])}</code>",
534 row["count"],
535 _e(", ".join(f"{k} {v}" for k, v in sorted(row["bySeverity"].items()))),
536 ]
537 for row in quality["findingsByRule"][:40]
538 ],
539 {1},
540 )
541 by_file = _table(
542 "Findings by file (the 20 with most findings)",
543 ["File", "Count", "By severity"],
544 [
545 [
546 f"<code>{_e(row['path'])}</code>",
547 row["count"],
548 _e(", ".join(f"{k} {v}" for k, v in sorted(row["bySeverity"].items()))),
549 ]
550 for row in quality["findingsByFile"]
551 ],
552 {1},
553 )
554 return [
555 _sub("h-severity", "Findings by severity", chart),
556 _sub("h-rules", "Findings by rule", by_rule),
557 _sub("h-files", "Findings by file", by_file),
558 ]
561def _stops_blocks(quality: dict[str, Any]) -> list[str]:
562 limits = quality["limitsReached"]
563 gates = quality["gateResults"]
564 limit_table = _table(
565 _LIMITS_TITLE,
566 ["Limit", "Times"],
567 [
568 ["Budget exceeded", limits["budget"]],
569 ["Phase attempts exhausted", limits["phaseAttempts"]],
570 ["Correction cycles exhausted", limits["corrections"]],
571 ],
572 {1},
573 )
574 gate_table = _table(
575 "Latest gate result of each run",
576 ["Result", "Runs"],
577 [[_e(name), count] for name, count in gates["latestByRun"].items()],
578 {1},
579 )
580 blocked = _table(
581 "Runs stopped before a decision",
582 ["Run", "Task", "Phase", "Reason"],
583 [
584 [
585 f"<code>{_e(row['runId'])}</code>",
586 f"<code>{_e(row['taskId'])}</code>",
587 _e(row["phase"]),
588 _e(row["reason"]),
589 ]
590 for row in quality["blockedRunsDetail"]
591 ],
592 set(),
593 )
594 return [
595 _sub("h-limits", _LIMITS_TITLE, limit_table),
596 _sub("h-gates", "Gate results", gate_table),
597 _sub("h-blocked", "Blocked runs and their reason", blocked),
598 ]
601def _quality_group(report: dict[str, Any]) -> str:
602 quality = report["quality"]
603 measures = _table(
604 "Quality of the runs",
605 ["Measure", "Value"],
606 [
607 ["Runs decided", quality["decided"]],
608 ["Approved the first time", quality["approvedFirstTime"]],
609 ["Approved the first time (rate)", quality["approvedFirstTimeRate"]],
610 ["Corrections", quality["corrections"]],
611 ["Requested changes", quality["requestedChanges"]],
612 ["Blocked runs", quality["blockedRuns"]],
613 ["Waiting for a decision", quality["waitingDecision"]],
614 ["Rejected runs", quality["rejectedRuns"]],
615 ["Closed runs", quality["closedRuns"]],
616 ["Pre-authorised approvals applied", quality["preAuthorizedApprovals"]],
617 ["Open HIGH or CRITICAL findings", quality["openBlockingFindings"]],
618 ],
619 {1},
620 )
621 return _group(
622 "s-quality",
623 "Quality",
624 "What the validators found and how the runs ended. Findings are listed by severity, "
625 "rule and file.",
626 [
627 _sub("h-quality", "Quality of the runs", measures),
628 *_findings_blocks(quality),
629 *_stops_blocks(quality),
630 ],
631 )
634def _delivery_group(report: dict[str, Any]) -> str:
635 delivery, lines = report["delivery"], report["lines"]
636 features = [
637 [
638 f"<code>{_e(item['taskId'])}</code> {_e(item['title'])}",
639 _e(item["closedAt"][:10]),
640 ", ".join(_link(issue["issue"], issue.get("url")) for issue in item["issues"]) or "-",
641 _link(f"#{item['pullRequest']['number']}", item["pullRequest"].get("url"))
642 if item.get("pullRequest") and item["pullRequest"].get("number") is not None
643 else "-",
644 ]
645 for item in delivery["featuresDelivered"]
646 ]
647 delivered = _table(
648 f"{len(delivery['featuresDelivered'])} feature(s) delivered, "
649 f"{len(delivery['issuesResolved'])} issue(s) resolved",
650 ["Task", "Closed", "Issues", "Pull request"],
651 features,
652 set(),
653 )
654 chart = _bars(
655 "chart-lines",
656 "Lines added and removed by author kind",
657 "Lines of the latest ChangeSet of each run, attributed by provenance to an agent "
658 "invocation or to a person (out of band).",
659 [
660 ("Agent invocation", [lines["agent"]["added"], lines["agent"]["removed"]]),
661 ("Person (out of band)", [lines["person"]["added"], lines["person"]["removed"]]),
662 ("Not attributed", [lines["unattributed"]["added"], lines["unattributed"]["removed"]]),
663 ],
664 [("added", "--s3"), ("removed", "--s2")],
665 "lines",
666 )
667 return _group(
668 "s-delivery",
669 "Delivery",
670 "What the closed runs delivered and how many lines changed.",
671 [_sub("h-delivery", "Delivered", delivered), _sub("h-lines", "Lines changed", chart)],
672 )
675def _trend_table(caption: str, first: str, rows: list[dict[str, Any]]) -> str:
676 return _table(
677 caption,
678 [first, "Runs", "Tokens", "Cost USD", "Interactions", "Approved", "Closed"],
679 [
680 [
681 _e(row["period"]),
682 row["runs"],
683 row["tokens"],
684 row["costUsd"],
685 row["interactions"],
686 row["approved"],
687 row["closed"],
688 ]
689 for row in rows
690 ],
691 {1, 2, 3, 4, 5, 6},
692 )
695def _trends_group(report: dict[str, Any]) -> str:
696 daily, weekly = report["trends"]["daily"], report["trends"]["weekly"]
697 body = (
698 _columns(
699 "chart-runs",
700 "Runs per day",
701 "Number of runs created each day.",
702 [(row["period"], float(row["runs"])) for row in daily],
703 )
704 + _columns(
705 "chart-tokens",
706 "Tokens per day",
707 "Input and output tokens of the agent calls of the runs created each day.",
708 [(row["period"], float(row["tokens"])) for row in daily],
709 )
710 + f"<details><summary>Daily table</summary>{_trend_table('Daily trend', 'Day', daily)}"
711 "</details>"
712 f"<details><summary>Weekly table</summary>{_trend_table('Weekly trend', 'Week', weekly)}"
713 "</details>"
714 )
715 return _group(
716 "s-trends",
717 "Trends",
718 "Activity per day and per week.",
719 [_sub("h-trend", "Trends", body)],
720 )
723_DATA_QUALITY = (
724 ("Tokens", "reported", "Counts the provider returned for each call; none is guessed."),
725 ("Reported cost", "reported", "The cost the provider returned with the call."),
726 (
727 "Estimated cost",
728 "estimated",
729 "Computed from the price table you configured for calls that reported tokens without "
730 "a cost; never charged and never counted against a budget.",
731 ),
732 ("Time, interactions, waits", "observed", "Taken from the timestamps of the event chain."),
733 ("Findings, gates, limits", "observed", "Records the harness wrote while the run happened."),
734 (
735 "Lines by author",
736 "observed",
737 "From the ChangeSet and, when enabled, provenance; otherwise not attributed.",
738 ),
739)
742def _notes_group(report: dict[str, Any]) -> str:
743 notes = "".join(f"<li>{_e(note)}</li>" for note in report["notes"])
744 quality = _table(
745 "Origin of each number",
746 ["Number", "Kind", "Meaning"],
747 [[_e(a), _e(b), _e(c)] for a, b, c in _DATA_QUALITY],
748 set(),
749 )
750 return _group(
751 "s-notes",
752 "Data quality",
753 "How each family of numbers was obtained: observed, reported by a provider or estimated.",
754 [
755 _sub("h-quality-kinds", "Origin of each number", quality),
756 _sub("h-notes", "Notes", f"<ul>{notes}</ul>"),
757 ],
758 )
761def _narrative(narrative: dict[str, Any]) -> str:
762 return _group(
763 "s-narrative",
764 "Narrative summary",
765 "Generated on demand; not verified by the harness.",
766 [
767 f'<p class="muted">Generated by <code>{_e(narrative["command"])}</code>.</p>'
768 f"<p>{_e(narrative['text']).replace(chr(10), '<br>')}</p>"
769 ],
770 )
773def render_html(report: dict[str, Any]) -> str:
774 totals = report["totals"]
775 narrative = report.get("narrative")
776 projects = ", ".join(_e(item["projectId"]) for item in report["projects"]) or "none"
777 sections = [
778 _overview(report),
779 _dimension_cards(report),
780 _cost_group(report),
781 _time_group(report),
782 _friction_group(report),
783 _quality_group(report),
784 _delivery_group(report),
785 _trends_group(report),
786 ]
787 if narrative: 787 ↛ 788line 787 didn't jump to line 788 because the condition on line 787 was never true
788 sections.append(_narrative(narrative))
789 sections.append(_notes_group(report))
790 return (
791 '<!doctype html>\n<html lang="en">\n<head>\n<meta charset="utf-8">\n'
792 '<meta name="viewport" content="width=device-width,initial-scale=1">\n'
793 "<title>Harness metrics</title>\n"
794 f"<style>{_CSS}</style>\n</head>\n<body>\n<header>\n<h1>Harness metrics</h1>\n"
795 f'<p class="lead">{totals["runs"]} run(s) of {projects}. Generated '
796 f"{_e(report['generatedAt'])} by harness {_e(report['harnessVersion'])}; computed from "
797 "the records only, no model was called. This file loads nothing from the network.</p>\n"
798 f"{_nav(bool(narrative))}\n</header>\n<main>\n"
799 + "\n".join(sections)
800 + "\n</main>\n</body>\n</html>\n"
801 )
804__all__ = ["render_html"]