Coverage for src/governed_harness/ladder/external.py: 84%

117 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Reading external evidence that closes a deferred verification (#55, item 2). 

2 

3Three formats, recognized by content or named explicitly: 

4 

5* JUnit XML: passes when it reports at least one test case and none failed or errored 

6 (``case`` narrows it to the test cases whose name or class name contains a text); 

7* SARIF 2.1.0: passes when no result has the level ``error`` (results without a level count 

8 as ``warning``, as the format says); 

9* a CI status JSON: ``state`` (or ``conclusion``) of a commit status or check run, with the 

10 commit it is about (``sha``, ``commit`` or ``head_sha``). ``success``/``passed`` pass; 

11 ``failure``, ``error``, ``cancelled``, ``timed_out`` fail; a pending status is not evidence. 

12 

13XML is parsed without DTDs or entity expansion, as the output parsers do.""" 

14 

15from __future__ import annotations 

16 

17import json 

18import xml.etree.ElementTree as ElementTree # nosec B405 - DTDs and entities are refused below 

19from dataclasses import dataclass 

20from typing import Any, Literal 

21 

22EvidenceKind = Literal["junit", "sarif", "ci-status"] 

23_PASSING = {"success", "passed", "pass", "succeeded"} 

24_FAILING = {"failure", "failed", "error", "cancelled", "canceled", "timed_out", "action_required"} 

25 

26 

27class EvidenceFormatError(ValueError): 

28 """The file is not evidence the harness can read.""" 

29 

30 

31@dataclass(frozen=True) 

32class EvidenceVerdict: 

33 kind: EvidenceKind 

34 passed: bool 

35 summary: str 

36 commit: str | None = None 

37 details: dict[str, Any] | None = None 

38 

39 

40def detect_kind(data: bytes) -> EvidenceKind: 

41 text = data.decode("utf-8", "replace").lstrip() 

42 if text.startswith("<"): 

43 return "junit" 

44 try: 

45 value = json.loads(text) 

46 except ValueError as error: 

47 raise EvidenceFormatError("the file is neither JUnit XML nor JSON") from error 

48 if isinstance(value, dict) and "runs" in value: 

49 return "sarif" 

50 if isinstance(value, dict): 50 ↛ 52line 50 didn't jump to line 52 because the condition on line 50 was always true

51 return "ci-status" 

52 raise EvidenceFormatError("the JSON file is neither SARIF nor a CI status object") 

53 

54 

55def _junit_root(data: bytes) -> ElementTree.Element: 

56 text = data.decode("utf-8", "replace") 

57 if "<!DOCTYPE" in text or "<!ENTITY" in text: 

58 raise EvidenceFormatError("JUnit XML with a DTD or entities is refused") 

59 try: 

60 root = ElementTree.fromstring(text) # nosec B314 - DTDs and entities refused above 

61 except ElementTree.ParseError as error: 

62 raise EvidenceFormatError(f"not well-formed XML: {error}") from error 

63 if root.tag not in {"testsuites", "testsuite"}: 63 ↛ 64line 63 didn't jump to line 64 because the condition on line 63 was never true

64 raise EvidenceFormatError("not a JUnit report (no testsuites or testsuite element)") 

65 return root 

66 

67 

68def _junit_counts(root: ElementTree.Element, case: str | None) -> tuple[int, int, int, list[str]]: 

69 """``(total, failed, skipped, failing ids)`` of the test cases whose id contains 

70 ``case``.""" 

71 total = failed = skipped = 0 

72 failing: list[str] = [] 

73 for item in root.iter("testcase"): 

74 name = item.get("name") or "test" 

75 classname = item.get("classname") or "" 

76 identifier = f"{classname}::{name}" if classname else name 

77 if case is not None and case not in identifier: 

78 continue 

79 total += 1 

80 if item.find("skipped") is not None: 80 ↛ 81line 80 didn't jump to line 81 because the condition on line 80 was never true

81 skipped += 1 

82 if item.find("failure") is not None or item.find("error") is not None: 

83 failed += 1 

84 failing.append(identifier) 

85 return total, failed, skipped, failing 

86 

87 

88def read_junit(data: bytes, case: str | None = None) -> EvidenceVerdict: 

89 total, failed, skipped, failing = _junit_counts(_junit_root(data), case) 

90 executed = total - skipped 

91 passed = executed > 0 and failed == 0 

92 scope = f" matching {case!r}" if case is not None else "" 

93 summary = ( 

94 f"JUnit: {total} test case(s){scope}, {failed} failed or errored, {skipped} skipped" 

95 + ("" if executed else "; nothing ran") 

96 ) 

97 return EvidenceVerdict( 

98 "junit", 

99 passed, 

100 summary, 

101 details={"total": total, "failed": failed, "skipped": skipped, "failing": failing[:20]}, 

102 ) 

103 

104 

105def _sarif_runs(data: bytes) -> list[Any]: 

106 try: 

107 value = json.loads(data.decode("utf-8", "replace")) 

108 except ValueError as error: 

109 raise EvidenceFormatError("not JSON") from error 

110 if not isinstance(value, dict) or not isinstance(value.get("runs"), list): 110 ↛ 111line 110 didn't jump to line 111 because the condition on line 110 was never true

111 raise EvidenceFormatError("not a SARIF log (no runs)") 

112 runs: list[Any] = value["runs"] 

113 return runs 

114 

115 

116def _sarif_levels(run: dict[str, Any]) -> tuple[int, int]: 

117 """``(errors, others)`` of a run's results; a result without a level is a warning.""" 

118 levels = [ 

119 result.get("level") or "warning" 

120 for result in run.get("results") or [] 

121 if isinstance(result, dict) 

122 ] 

123 errors = levels.count("error") 

124 return errors, len(levels) - errors 

125 

126 

127def read_sarif(data: bytes) -> EvidenceVerdict: 

128 errors = warnings = 0 

129 tools: list[str] = [] 

130 for run in _sarif_runs(data): 

131 if not isinstance(run, dict): 131 ↛ 132line 131 didn't jump to line 132 because the condition on line 131 was never true

132 continue 

133 driver = (run.get("tool") or {}).get("driver") or {} 

134 if isinstance(driver, dict) and driver.get("name"): 134 ↛ 136line 134 didn't jump to line 136 because the condition on line 134 was always true

135 tools.append(str(driver["name"])) 

136 run_errors, run_others = _sarif_levels(run) 

137 errors += run_errors 

138 warnings += run_others 

139 summary = f"SARIF ({', '.join(tools) or 'unknown tool'}): {errors} error(s), {warnings} other" 

140 return EvidenceVerdict( 

141 "sarif", errors == 0, summary, details={"errors": errors, "others": warnings} 

142 ) 

143 

144 

145def read_ci_status(data: bytes) -> EvidenceVerdict: 

146 try: 

147 value = json.loads(data.decode("utf-8", "replace")) 

148 except ValueError as error: 

149 raise EvidenceFormatError("not JSON") from error 

150 if not isinstance(value, dict): 150 ↛ 151line 150 didn't jump to line 151 because the condition on line 150 was never true

151 raise EvidenceFormatError("a CI status is a JSON object") 

152 raw = value.get("conclusion") or value.get("state") or value.get("status") 

153 state = str(raw or "").strip().lower() 

154 commit = value.get("sha") or value.get("commit") or value.get("head_sha") 

155 context = value.get("context") or value.get("name") or "CI" 

156 if state in _PASSING: 

157 passed = True 

158 elif state in _FAILING: 158 ↛ 159line 158 didn't jump to line 159 because the condition on line 158 was never true

159 passed = False 

160 else: 

161 raise EvidenceFormatError( 

162 f"the CI status {state or 'without a state'!r} is not final (success or failure)" 

163 ) 

164 return EvidenceVerdict( 

165 "ci-status", 

166 passed, 

167 f"CI status {context}: {state}", 

168 commit=str(commit) if commit else None, 

169 details={"context": str(context), "state": state, "targetUrl": value.get("target_url")}, 

170 ) 

171 

172 

173def read_evidence(data: bytes, kind: str = "auto", case: str | None = None) -> EvidenceVerdict: 

174 chosen = detect_kind(data) if kind == "auto" else kind 

175 if chosen == "junit": 

176 return read_junit(data, case) 

177 if chosen == "sarif": 

178 return read_sarif(data) 

179 if chosen == "ci-status": 179 ↛ 181line 179 didn't jump to line 181 because the condition on line 179 was always true

180 return read_ci_status(data) 

181 raise EvidenceFormatError(f"unknown evidence format: {kind}") 

182 

183 

184__all__ = [ 

185 "EvidenceFormatError", 

186 "EvidenceKind", 

187 "EvidenceVerdict", 

188 "detect_kind", 

189 "read_ci_status", 

190 "read_evidence", 

191 "read_junit", 

192 "read_sarif", 

193]