Coverage for src/governed_harness/ladder/external.py: 84%
117 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Reading external evidence that closes a deferred verification (#55, item 2).
3Three formats, recognized by content or named explicitly:
5* JUnit XML: passes when it reports at least one test case and none failed or errored
6 (``case`` narrows it to the test cases whose name or class name contains a text);
7* SARIF 2.1.0: passes when no result has the level ``error`` (results without a level count
8 as ``warning``, as the format says);
9* a CI status JSON: ``state`` (or ``conclusion``) of a commit status or check run, with the
10 commit it is about (``sha``, ``commit`` or ``head_sha``). ``success``/``passed`` pass;
11 ``failure``, ``error``, ``cancelled``, ``timed_out`` fail; a pending status is not evidence.
13XML is parsed without DTDs or entity expansion, as the output parsers do."""
15from __future__ import annotations
17import json
18import xml.etree.ElementTree as ElementTree # nosec B405 - DTDs and entities are refused below
19from dataclasses import dataclass
20from typing import Any, Literal
22EvidenceKind = Literal["junit", "sarif", "ci-status"]
23_PASSING = {"success", "passed", "pass", "succeeded"}
24_FAILING = {"failure", "failed", "error", "cancelled", "canceled", "timed_out", "action_required"}
27class EvidenceFormatError(ValueError):
28 """The file is not evidence the harness can read."""
31@dataclass(frozen=True)
32class EvidenceVerdict:
33 kind: EvidenceKind
34 passed: bool
35 summary: str
36 commit: str | None = None
37 details: dict[str, Any] | None = None
40def detect_kind(data: bytes) -> EvidenceKind:
41 text = data.decode("utf-8", "replace").lstrip()
42 if text.startswith("<"):
43 return "junit"
44 try:
45 value = json.loads(text)
46 except ValueError as error:
47 raise EvidenceFormatError("the file is neither JUnit XML nor JSON") from error
48 if isinstance(value, dict) and "runs" in value:
49 return "sarif"
50 if isinstance(value, dict): 50 ↛ 52line 50 didn't jump to line 52 because the condition on line 50 was always true
51 return "ci-status"
52 raise EvidenceFormatError("the JSON file is neither SARIF nor a CI status object")
55def _junit_root(data: bytes) -> ElementTree.Element:
56 text = data.decode("utf-8", "replace")
57 if "<!DOCTYPE" in text or "<!ENTITY" in text:
58 raise EvidenceFormatError("JUnit XML with a DTD or entities is refused")
59 try:
60 root = ElementTree.fromstring(text) # nosec B314 - DTDs and entities refused above
61 except ElementTree.ParseError as error:
62 raise EvidenceFormatError(f"not well-formed XML: {error}") from error
63 if root.tag not in {"testsuites", "testsuite"}: 63 ↛ 64line 63 didn't jump to line 64 because the condition on line 63 was never true
64 raise EvidenceFormatError("not a JUnit report (no testsuites or testsuite element)")
65 return root
68def _junit_counts(root: ElementTree.Element, case: str | None) -> tuple[int, int, int, list[str]]:
69 """``(total, failed, skipped, failing ids)`` of the test cases whose id contains
70 ``case``."""
71 total = failed = skipped = 0
72 failing: list[str] = []
73 for item in root.iter("testcase"):
74 name = item.get("name") or "test"
75 classname = item.get("classname") or ""
76 identifier = f"{classname}::{name}" if classname else name
77 if case is not None and case not in identifier:
78 continue
79 total += 1
80 if item.find("skipped") is not None: 80 ↛ 81line 80 didn't jump to line 81 because the condition on line 80 was never true
81 skipped += 1
82 if item.find("failure") is not None or item.find("error") is not None:
83 failed += 1
84 failing.append(identifier)
85 return total, failed, skipped, failing
88def read_junit(data: bytes, case: str | None = None) -> EvidenceVerdict:
89 total, failed, skipped, failing = _junit_counts(_junit_root(data), case)
90 executed = total - skipped
91 passed = executed > 0 and failed == 0
92 scope = f" matching {case!r}" if case is not None else ""
93 summary = (
94 f"JUnit: {total} test case(s){scope}, {failed} failed or errored, {skipped} skipped"
95 + ("" if executed else "; nothing ran")
96 )
97 return EvidenceVerdict(
98 "junit",
99 passed,
100 summary,
101 details={"total": total, "failed": failed, "skipped": skipped, "failing": failing[:20]},
102 )
105def _sarif_runs(data: bytes) -> list[Any]:
106 try:
107 value = json.loads(data.decode("utf-8", "replace"))
108 except ValueError as error:
109 raise EvidenceFormatError("not JSON") from error
110 if not isinstance(value, dict) or not isinstance(value.get("runs"), list): 110 ↛ 111line 110 didn't jump to line 111 because the condition on line 110 was never true
111 raise EvidenceFormatError("not a SARIF log (no runs)")
112 runs: list[Any] = value["runs"]
113 return runs
116def _sarif_levels(run: dict[str, Any]) -> tuple[int, int]:
117 """``(errors, others)`` of a run's results; a result without a level is a warning."""
118 levels = [
119 result.get("level") or "warning"
120 for result in run.get("results") or []
121 if isinstance(result, dict)
122 ]
123 errors = levels.count("error")
124 return errors, len(levels) - errors
127def read_sarif(data: bytes) -> EvidenceVerdict:
128 errors = warnings = 0
129 tools: list[str] = []
130 for run in _sarif_runs(data):
131 if not isinstance(run, dict): 131 ↛ 132line 131 didn't jump to line 132 because the condition on line 131 was never true
132 continue
133 driver = (run.get("tool") or {}).get("driver") or {}
134 if isinstance(driver, dict) and driver.get("name"): 134 ↛ 136line 134 didn't jump to line 136 because the condition on line 134 was always true
135 tools.append(str(driver["name"]))
136 run_errors, run_others = _sarif_levels(run)
137 errors += run_errors
138 warnings += run_others
139 summary = f"SARIF ({', '.join(tools) or 'unknown tool'}): {errors} error(s), {warnings} other"
140 return EvidenceVerdict(
141 "sarif", errors == 0, summary, details={"errors": errors, "others": warnings}
142 )
145def read_ci_status(data: bytes) -> EvidenceVerdict:
146 try:
147 value = json.loads(data.decode("utf-8", "replace"))
148 except ValueError as error:
149 raise EvidenceFormatError("not JSON") from error
150 if not isinstance(value, dict): 150 ↛ 151line 150 didn't jump to line 151 because the condition on line 150 was never true
151 raise EvidenceFormatError("a CI status is a JSON object")
152 raw = value.get("conclusion") or value.get("state") or value.get("status")
153 state = str(raw or "").strip().lower()
154 commit = value.get("sha") or value.get("commit") or value.get("head_sha")
155 context = value.get("context") or value.get("name") or "CI"
156 if state in _PASSING:
157 passed = True
158 elif state in _FAILING: 158 ↛ 159line 158 didn't jump to line 159 because the condition on line 158 was never true
159 passed = False
160 else:
161 raise EvidenceFormatError(
162 f"the CI status {state or 'without a state'!r} is not final (success or failure)"
163 )
164 return EvidenceVerdict(
165 "ci-status",
166 passed,
167 f"CI status {context}: {state}",
168 commit=str(commit) if commit else None,
169 details={"context": str(context), "state": state, "targetUrl": value.get("target_url")},
170 )
173def read_evidence(data: bytes, kind: str = "auto", case: str | None = None) -> EvidenceVerdict:
174 chosen = detect_kind(data) if kind == "auto" else kind
175 if chosen == "junit":
176 return read_junit(data, case)
177 if chosen == "sarif":
178 return read_sarif(data)
179 if chosen == "ci-status": 179 ↛ 181line 179 didn't jump to line 181 because the condition on line 179 was always true
180 return read_ci_status(data)
181 raise EvidenceFormatError(f"unknown evidence format: {kind}")
184__all__ = [
185 "EvidenceFormatError",
186 "EvidenceKind",
187 "EvidenceVerdict",
188 "detect_kind",
189 "read_ci_status",
190 "read_evidence",
191 "read_junit",
192 "read_sarif",
193]