Coverage for src/governed_harness/review/contract.py: 92%

156 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""The output contract of a reviewer and the verdict the harness recomputes (#57). 

2 

3A reviewer answers with one JSON object (the ``result`` of the provider protocol):: 

4 

5 {"verdict": "PASS" | "PASS_WARN" | "FAIL", 

6 "findings": [{"file": "src/a.py", "side": "new" | "old", "line": 12, 

7 "rule": "tests.mock-of-subject", "severity": "error" | "suggestion", 

8 "issue": "...", "evidence": "the changed line it is about"}], 

9 "summary": "..."} 

10 

11An answer that does not follow it is ``UNKNOWN``. The harness never takes the reviewer's verdict: 

12 

13* a finding outside the reportable locations is dropped and counted; 

14* a finding on a rule of the catalog keeps the rule's severity at most (a ``warning`` rule 

15 cannot block); 

16* a finding on a rule outside the catalog blocks only with concrete evidence: its ``evidence`` 

17 must quote a changed line of the file near the reported line; otherwise it is a suggestion; 

18* at most N findings are kept, errors first, then by rule priority, file, line and rule; 

19* the verdict is ``FAIL`` with any error, ``PASS_WARN`` with suggestions only, else ``PASS``.""" 

20 

21from __future__ import annotations 

22 

23import re 

24from collections.abc import Mapping 

25from dataclasses import dataclass, field 

26from typing import Any, Literal 

27 

28from governed_harness.review.diff import FileChange, Locations 

29from governed_harness.review.rules import DEFAULT_PRIORITY, Rule 

30 

31Verdict = Literal["PASS", "PASS_WARN", "FAIL", "UNKNOWN"] 

32FindingSeverity = Literal["error", "suggestion"] 

33VERDICTS: tuple[str, ...] = ("PASS", "PASS_WARN", "FAIL") 

34SEVERITIES: tuple[str, ...] = ("error", "suggestion") 

35OUT_OF_CATALOG_PRIORITY = 100 

36_EVIDENCE_WINDOW = 3 

37_MAX_TEXT = 1000 

38_RULE_TEXT = re.compile(r"[^a-z0-9.\-_]") 

39 

40 

41class ContractError(ValueError): 

42 """The answer does not follow the output contract: the reviewer's result is UNKNOWN.""" 

43 

44 

45@dataclass(frozen=True) 

46class ReviewFinding: 

47 reviewer: str 

48 file: str 

49 side: str 

50 line: int 

51 rule: str 

52 severity: FindingSeverity 

53 issue: str 

54 evidence: str = "" 

55 in_catalog: bool = True 

56 priority: int = DEFAULT_PRIORITY 

57 source: str = "ai" 

58 """``ai``, ``tool:harness:CHECK``, ``tool:TOOL`` or ``consistency``.""" 

59 note: str = "" 

60 

61 @property 

62 def blocking(self) -> bool: 

63 return self.severity == "error" 

64 

65 @property 

66 def key(self) -> tuple[str, str, int, str]: 

67 return (self.file, self.side, self.line, self.rule) 

68 

69 def sort_key(self) -> tuple[Any, ...]: 

70 return (not self.blocking, self.priority, self.file, self.side, self.line, self.rule) 

71 

72 def as_dict(self) -> dict[str, Any]: 

73 value: dict[str, Any] = { 

74 "reviewer": self.reviewer, 

75 "file": self.file, 

76 "side": self.side, 

77 "line": self.line, 

78 "rule": self.rule, 

79 "severity": self.severity, 

80 "issue": self.issue, 

81 "inCatalog": self.in_catalog, 

82 "priority": self.priority, 

83 "source": self.source, 

84 } 

85 if self.evidence: 

86 value["evidence"] = self.evidence 

87 if self.note: 

88 value["note"] = self.note 

89 return value 

90 

91 @classmethod 

92 def from_dict(cls, value: Mapping[str, Any]) -> ReviewFinding: 

93 return cls( 

94 reviewer=str(value["reviewer"]), 

95 file=str(value["file"]), 

96 side=str(value["side"]), 

97 line=int(value["line"]), 

98 rule=str(value["rule"]), 

99 severity="error" if value["severity"] == "error" else "suggestion", 

100 issue=str(value["issue"]), 

101 evidence=str(value.get("evidence") or ""), 

102 in_catalog=bool(value.get("inCatalog", True)), 

103 priority=int(value.get("priority", DEFAULT_PRIORITY)), 

104 source=str(value.get("source") or "ai"), 

105 note=str(value.get("note") or ""), 

106 ) 

107 

108 

109@dataclass 

110class Normalized: 

111 """A reviewer's answer after the harness applied the contract.""" 

112 

113 findings: list[ReviewFinding] = field(default_factory=list) 

114 stated_verdict: str = "" 

115 summary: str = "" 

116 dropped_outside: int = 0 

117 downgraded: int = 0 

118 truncated: int = 0 

119 model: str | None = None 

120 

121 

122def _text(value: Any) -> str: 

123 return " ".join(str(value).split())[:_MAX_TEXT] 

124 

125 

126def parse_answer(result: Any) -> tuple[str, list[dict[str, Any]], str, str | None]: 

127 """Check the shape of an answer; raise ``ContractError`` when it does not follow it.""" 

128 if not isinstance(result, dict): 

129 raise ContractError("the answer is not a JSON object") 

130 verdict = result.get("verdict") 

131 if verdict not in VERDICTS: 

132 raise ContractError(f"verdict must be one of {', '.join(VERDICTS)}, got {verdict!r}") 

133 summary = result.get("summary") 

134 if not isinstance(summary, str): 134 ↛ 135line 134 didn't jump to line 135 because the condition on line 134 was never true

135 raise ContractError("summary must be a string") 

136 raw = result.get("findings") 

137 if not isinstance(raw, list): 137 ↛ 138line 137 didn't jump to line 138 because the condition on line 137 was never true

138 raise ContractError("findings must be a list") 

139 items = [_checked_finding(index, entry) for index, entry in enumerate(raw, start=1)] 

140 model = result.get("model") 

141 return str(verdict), items, summary, model if isinstance(model, str) and model else None 

142 

143 

144def _checked_finding(index: int, entry: Any) -> dict[str, Any]: 

145 """One finding of an answer, checked against the contract.""" 

146 if not isinstance(entry, dict): 146 ↛ 147line 146 didn't jump to line 147 because the condition on line 146 was never true

147 raise ContractError(f"finding {index} is not an object") 

148 for name in ("file", "rule", "issue"): 

149 if not isinstance(entry.get(name), str) or not entry[name].strip(): 

150 raise ContractError(f"finding {index} needs a non-empty {name}") 

151 if entry.get("side") not in ("new", "old"): 

152 raise ContractError(f"finding {index}: side must be new or old") 

153 line = entry.get("line") 

154 if not isinstance(line, int) or isinstance(line, bool) or line < 1: 

155 raise ContractError(f"finding {index}: line must be a positive integer") 

156 if entry.get("severity") not in SEVERITIES: 156 ↛ 157line 156 didn't jump to line 157 because the condition on line 156 was never true

157 raise ContractError(f"finding {index}: severity must be error or suggestion") 

158 evidence = entry.get("evidence", "") 

159 if evidence is not None and not isinstance(evidence, str): 159 ↛ 160line 159 didn't jump to line 160 because the condition on line 159 was never true

160 raise ContractError(f"finding {index}: evidence must be a string") 

161 return entry 

162 

163 

164def _rule_id(value: str) -> str: 

165 return _RULE_TEXT.sub("-", value.strip().lower())[:80] or "finding" 

166 

167 

168def _near(changes: Mapping[str, FileChange], path: str, side: str, line: int) -> list[str]: 

169 item = changes.get(path) 

170 if item is None: 170 ↛ 171line 170 didn't jump to line 171 because the condition on line 170 was never true

171 return [] 

172 lines = item.added if side == "new" else item.removed 

173 return [ 

174 entry.text 

175 for entry in lines 

176 if abs(entry.number - line) <= _EVIDENCE_WINDOW and entry.text.strip() 

177 ] 

178 

179 

180def concrete(evidence: str, candidates: list[str]) -> bool: 

181 """Whether ``evidence`` quotes one of the changed lines (whitespace-insensitive).""" 

182 quoted = " ".join(evidence.split()).strip("`'\" ") 

183 if len(quoted) < 4: 

184 return False 

185 for text in candidates: 185 ↛ 189line 185 didn't jump to line 189 because the loop on line 185 didn't complete

186 line = " ".join(text.split()) 

187 if quoted in line or (len(line) >= 4 and line in quoted): 

188 return True 

189 return False 

190 

191 

192def normalize( 

193 reviewer: str, 

194 result: Any, 

195 *, 

196 rules: Mapping[str, Rule], 

197 locations: Locations, 

198 changes: Mapping[str, FileChange], 

199 limit: int, 

200) -> Normalized: 

201 """Apply the contract to an answer (``rules``: the reviewer's catalog rules by id).""" 

202 verdict, items, summary, model = parse_answer(result) 

203 normalized = Normalized(stated_verdict=verdict, summary=_text(summary), model=model) 

204 seen: set[tuple[str, str, int, str]] = set() 

205 for entry in items: 

206 path = entry["file"].strip().replace("\\", "/").removeprefix("./") 

207 side, line = entry["side"], int(entry["line"]) 

208 if not locations.allows(path, side, line): 

209 normalized.dropped_outside += 1 

210 continue 

211 finding = _finding(reviewer, entry, path, rules, changes) 

212 if finding.note: 

213 normalized.downgraded += 1 

214 if finding.key in seen: 214 ↛ 215line 214 didn't jump to line 215 because the condition on line 214 was never true

215 continue 

216 seen.add(finding.key) 

217 normalized.findings.append(finding) 

218 normalized.findings.sort(key=ReviewFinding.sort_key) 

219 if len(normalized.findings) > limit: 

220 normalized.truncated = len(normalized.findings) - limit 

221 normalized.findings = normalized.findings[:limit] 

222 return normalized 

223 

224 

225def _finding( 

226 reviewer: str, 

227 entry: dict[str, Any], 

228 path: str, 

229 rules: Mapping[str, Rule], 

230 changes: Mapping[str, FileChange], 

231) -> ReviewFinding: 

232 """One reportable finding of an answer, with its catalog priority; a ``note`` says why an 

233 error was downgraded to a suggestion.""" 

234 side, line = entry["side"], int(entry["line"]) 

235 rule_id = _rule_id(entry["rule"]) 

236 evidence = _text(entry.get("evidence") or "") 

237 rule = rules.get(rule_id) 

238 severity: FindingSeverity = "error" if entry["severity"] == "error" else "suggestion" 

239 note = "" 

240 if severity == "error": 

241 note = _downgrade(rule, evidence, _near(changes, path, side, line)) 

242 return ReviewFinding( 

243 reviewer=reviewer, 

244 file=path, 

245 side=side, 

246 line=line, 

247 rule=rule_id, 

248 severity="suggestion" if note else severity, 

249 issue=_text(entry["issue"]), 

250 evidence=evidence, 

251 in_catalog=rule is not None, 

252 priority=rule.priority if rule is not None else OUT_OF_CATALOG_PRIORITY, 

253 note=note, 

254 ) 

255 

256 

257def _downgrade(rule: Rule | None, evidence: str, near: list[str]) -> str: 

258 """Why an error is reported as a suggestion: its catalog rule is a warning, or it is 

259 outside the catalog and its evidence quotes no changed line; empty when it stays an 

260 error.""" 

261 if rule is not None: 

262 return "" if rule.blocking else "the rule is a warning" 

263 if concrete(evidence, near): 

264 return "" 

265 return "outside the catalog without concrete evidence" 

266 

267 

268def verdict_of(findings: list[ReviewFinding]) -> Verdict: 

269 if any(item.blocking for item in findings): 

270 return "FAIL" 

271 if findings: 

272 return "PASS_WARN" 

273 return "PASS" 

274 

275 

276def contract_text(limit: int) -> str: 

277 """The fixed output contract every reviewer request carries.""" 

278 return ( 

279 "Answer with exactly one JSON object on standard output: " 

280 '{"status": "PASSED", "summary": "<one sentence>", "result": {"verdict": "PASS" | ' 

281 '"PASS_WARN" | "FAIL", "findings": [{"file": "<path>", "side": "new" | "old", ' 

282 '"line": 1, "rule": "<rule id>", "severity": "error" | "suggestion", "issue": ' 

283 '"<what is wrong>", "evidence": "<the changed line it is about>"}], "summary": ' 

284 '"<one sentence>"}}. Report only on the reportable locations of the request: side new ' 

285 "is a line the change added (its number in the new file), side old a line it removed " 

286 "(its number in the old file); anything else is dropped. Use the rule ids listed above; " 

287 "a finding on another rule is an error only when its evidence quotes the changed line. " 

288 f"At most {limit} findings, errors first in rule priority order. Prefer false " 

289 "negatives: report only what you confirmed within the rule's budget. The harness " 

290 "recomputes the verdict from the findings. This request is read-only: do not change " 

291 "any file." 

292 ) 

293 

294 

295__all__ = [ 

296 "OUT_OF_CATALOG_PRIORITY", 

297 "SEVERITIES", 

298 "VERDICTS", 

299 "ContractError", 

300 "Normalized", 

301 "ReviewFinding", 

302 "Verdict", 

303 "concrete", 

304 "contract_text", 

305 "normalize", 

306 "parse_answer", 

307 "verdict_of", 

308]