Coverage for src/governed_harness/review/contract.py: 92%
156 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""The output contract of a reviewer and the verdict the harness recomputes (#57).
3A reviewer answers with one JSON object (the ``result`` of the provider protocol)::
5 {"verdict": "PASS" | "PASS_WARN" | "FAIL",
6 "findings": [{"file": "src/a.py", "side": "new" | "old", "line": 12,
7 "rule": "tests.mock-of-subject", "severity": "error" | "suggestion",
8 "issue": "...", "evidence": "the changed line it is about"}],
9 "summary": "..."}
11An answer that does not follow it is ``UNKNOWN``. The harness never takes the reviewer's verdict:
13* a finding outside the reportable locations is dropped and counted;
14* a finding on a rule of the catalog keeps the rule's severity at most (a ``warning`` rule
15 cannot block);
16* a finding on a rule outside the catalog blocks only with concrete evidence: its ``evidence``
17 must quote a changed line of the file near the reported line; otherwise it is a suggestion;
18* at most N findings are kept, errors first, then by rule priority, file, line and rule;
19* the verdict is ``FAIL`` with any error, ``PASS_WARN`` with suggestions only, else ``PASS``."""
21from __future__ import annotations
23import re
24from collections.abc import Mapping
25from dataclasses import dataclass, field
26from typing import Any, Literal
28from governed_harness.review.diff import FileChange, Locations
29from governed_harness.review.rules import DEFAULT_PRIORITY, Rule
31Verdict = Literal["PASS", "PASS_WARN", "FAIL", "UNKNOWN"]
32FindingSeverity = Literal["error", "suggestion"]
33VERDICTS: tuple[str, ...] = ("PASS", "PASS_WARN", "FAIL")
34SEVERITIES: tuple[str, ...] = ("error", "suggestion")
35OUT_OF_CATALOG_PRIORITY = 100
36_EVIDENCE_WINDOW = 3
37_MAX_TEXT = 1000
38_RULE_TEXT = re.compile(r"[^a-z0-9.\-_]")
41class ContractError(ValueError):
42 """The answer does not follow the output contract: the reviewer's result is UNKNOWN."""
45@dataclass(frozen=True)
46class ReviewFinding:
47 reviewer: str
48 file: str
49 side: str
50 line: int
51 rule: str
52 severity: FindingSeverity
53 issue: str
54 evidence: str = ""
55 in_catalog: bool = True
56 priority: int = DEFAULT_PRIORITY
57 source: str = "ai"
58 """``ai``, ``tool:harness:CHECK``, ``tool:TOOL`` or ``consistency``."""
59 note: str = ""
61 @property
62 def blocking(self) -> bool:
63 return self.severity == "error"
65 @property
66 def key(self) -> tuple[str, str, int, str]:
67 return (self.file, self.side, self.line, self.rule)
69 def sort_key(self) -> tuple[Any, ...]:
70 return (not self.blocking, self.priority, self.file, self.side, self.line, self.rule)
72 def as_dict(self) -> dict[str, Any]:
73 value: dict[str, Any] = {
74 "reviewer": self.reviewer,
75 "file": self.file,
76 "side": self.side,
77 "line": self.line,
78 "rule": self.rule,
79 "severity": self.severity,
80 "issue": self.issue,
81 "inCatalog": self.in_catalog,
82 "priority": self.priority,
83 "source": self.source,
84 }
85 if self.evidence:
86 value["evidence"] = self.evidence
87 if self.note:
88 value["note"] = self.note
89 return value
91 @classmethod
92 def from_dict(cls, value: Mapping[str, Any]) -> ReviewFinding:
93 return cls(
94 reviewer=str(value["reviewer"]),
95 file=str(value["file"]),
96 side=str(value["side"]),
97 line=int(value["line"]),
98 rule=str(value["rule"]),
99 severity="error" if value["severity"] == "error" else "suggestion",
100 issue=str(value["issue"]),
101 evidence=str(value.get("evidence") or ""),
102 in_catalog=bool(value.get("inCatalog", True)),
103 priority=int(value.get("priority", DEFAULT_PRIORITY)),
104 source=str(value.get("source") or "ai"),
105 note=str(value.get("note") or ""),
106 )
109@dataclass
110class Normalized:
111 """A reviewer's answer after the harness applied the contract."""
113 findings: list[ReviewFinding] = field(default_factory=list)
114 stated_verdict: str = ""
115 summary: str = ""
116 dropped_outside: int = 0
117 downgraded: int = 0
118 truncated: int = 0
119 model: str | None = None
122def _text(value: Any) -> str:
123 return " ".join(str(value).split())[:_MAX_TEXT]
126def parse_answer(result: Any) -> tuple[str, list[dict[str, Any]], str, str | None]:
127 """Check the shape of an answer; raise ``ContractError`` when it does not follow it."""
128 if not isinstance(result, dict):
129 raise ContractError("the answer is not a JSON object")
130 verdict = result.get("verdict")
131 if verdict not in VERDICTS:
132 raise ContractError(f"verdict must be one of {', '.join(VERDICTS)}, got {verdict!r}")
133 summary = result.get("summary")
134 if not isinstance(summary, str): 134 ↛ 135line 134 didn't jump to line 135 because the condition on line 134 was never true
135 raise ContractError("summary must be a string")
136 raw = result.get("findings")
137 if not isinstance(raw, list): 137 ↛ 138line 137 didn't jump to line 138 because the condition on line 137 was never true
138 raise ContractError("findings must be a list")
139 items = [_checked_finding(index, entry) for index, entry in enumerate(raw, start=1)]
140 model = result.get("model")
141 return str(verdict), items, summary, model if isinstance(model, str) and model else None
144def _checked_finding(index: int, entry: Any) -> dict[str, Any]:
145 """One finding of an answer, checked against the contract."""
146 if not isinstance(entry, dict): 146 ↛ 147line 146 didn't jump to line 147 because the condition on line 146 was never true
147 raise ContractError(f"finding {index} is not an object")
148 for name in ("file", "rule", "issue"):
149 if not isinstance(entry.get(name), str) or not entry[name].strip():
150 raise ContractError(f"finding {index} needs a non-empty {name}")
151 if entry.get("side") not in ("new", "old"):
152 raise ContractError(f"finding {index}: side must be new or old")
153 line = entry.get("line")
154 if not isinstance(line, int) or isinstance(line, bool) or line < 1:
155 raise ContractError(f"finding {index}: line must be a positive integer")
156 if entry.get("severity") not in SEVERITIES: 156 ↛ 157line 156 didn't jump to line 157 because the condition on line 156 was never true
157 raise ContractError(f"finding {index}: severity must be error or suggestion")
158 evidence = entry.get("evidence", "")
159 if evidence is not None and not isinstance(evidence, str): 159 ↛ 160line 159 didn't jump to line 160 because the condition on line 159 was never true
160 raise ContractError(f"finding {index}: evidence must be a string")
161 return entry
164def _rule_id(value: str) -> str:
165 return _RULE_TEXT.sub("-", value.strip().lower())[:80] or "finding"
168def _near(changes: Mapping[str, FileChange], path: str, side: str, line: int) -> list[str]:
169 item = changes.get(path)
170 if item is None: 170 ↛ 171line 170 didn't jump to line 171 because the condition on line 170 was never true
171 return []
172 lines = item.added if side == "new" else item.removed
173 return [
174 entry.text
175 for entry in lines
176 if abs(entry.number - line) <= _EVIDENCE_WINDOW and entry.text.strip()
177 ]
180def concrete(evidence: str, candidates: list[str]) -> bool:
181 """Whether ``evidence`` quotes one of the changed lines (whitespace-insensitive)."""
182 quoted = " ".join(evidence.split()).strip("`'\" ")
183 if len(quoted) < 4:
184 return False
185 for text in candidates: 185 ↛ 189line 185 didn't jump to line 189 because the loop on line 185 didn't complete
186 line = " ".join(text.split())
187 if quoted in line or (len(line) >= 4 and line in quoted):
188 return True
189 return False
192def normalize(
193 reviewer: str,
194 result: Any,
195 *,
196 rules: Mapping[str, Rule],
197 locations: Locations,
198 changes: Mapping[str, FileChange],
199 limit: int,
200) -> Normalized:
201 """Apply the contract to an answer (``rules``: the reviewer's catalog rules by id)."""
202 verdict, items, summary, model = parse_answer(result)
203 normalized = Normalized(stated_verdict=verdict, summary=_text(summary), model=model)
204 seen: set[tuple[str, str, int, str]] = set()
205 for entry in items:
206 path = entry["file"].strip().replace("\\", "/").removeprefix("./")
207 side, line = entry["side"], int(entry["line"])
208 if not locations.allows(path, side, line):
209 normalized.dropped_outside += 1
210 continue
211 finding = _finding(reviewer, entry, path, rules, changes)
212 if finding.note:
213 normalized.downgraded += 1
214 if finding.key in seen: 214 ↛ 215line 214 didn't jump to line 215 because the condition on line 214 was never true
215 continue
216 seen.add(finding.key)
217 normalized.findings.append(finding)
218 normalized.findings.sort(key=ReviewFinding.sort_key)
219 if len(normalized.findings) > limit:
220 normalized.truncated = len(normalized.findings) - limit
221 normalized.findings = normalized.findings[:limit]
222 return normalized
225def _finding(
226 reviewer: str,
227 entry: dict[str, Any],
228 path: str,
229 rules: Mapping[str, Rule],
230 changes: Mapping[str, FileChange],
231) -> ReviewFinding:
232 """One reportable finding of an answer, with its catalog priority; a ``note`` says why an
233 error was downgraded to a suggestion."""
234 side, line = entry["side"], int(entry["line"])
235 rule_id = _rule_id(entry["rule"])
236 evidence = _text(entry.get("evidence") or "")
237 rule = rules.get(rule_id)
238 severity: FindingSeverity = "error" if entry["severity"] == "error" else "suggestion"
239 note = ""
240 if severity == "error":
241 note = _downgrade(rule, evidence, _near(changes, path, side, line))
242 return ReviewFinding(
243 reviewer=reviewer,
244 file=path,
245 side=side,
246 line=line,
247 rule=rule_id,
248 severity="suggestion" if note else severity,
249 issue=_text(entry["issue"]),
250 evidence=evidence,
251 in_catalog=rule is not None,
252 priority=rule.priority if rule is not None else OUT_OF_CATALOG_PRIORITY,
253 note=note,
254 )
257def _downgrade(rule: Rule | None, evidence: str, near: list[str]) -> str:
258 """Why an error is reported as a suggestion: its catalog rule is a warning, or it is
259 outside the catalog and its evidence quotes no changed line; empty when it stays an
260 error."""
261 if rule is not None:
262 return "" if rule.blocking else "the rule is a warning"
263 if concrete(evidence, near):
264 return ""
265 return "outside the catalog without concrete evidence"
268def verdict_of(findings: list[ReviewFinding]) -> Verdict:
269 if any(item.blocking for item in findings):
270 return "FAIL"
271 if findings:
272 return "PASS_WARN"
273 return "PASS"
276def contract_text(limit: int) -> str:
277 """The fixed output contract every reviewer request carries."""
278 return (
279 "Answer with exactly one JSON object on standard output: "
280 '{"status": "PASSED", "summary": "<one sentence>", "result": {"verdict": "PASS" | '
281 '"PASS_WARN" | "FAIL", "findings": [{"file": "<path>", "side": "new" | "old", '
282 '"line": 1, "rule": "<rule id>", "severity": "error" | "suggestion", "issue": '
283 '"<what is wrong>", "evidence": "<the changed line it is about>"}], "summary": '
284 '"<one sentence>"}}. Report only on the reportable locations of the request: side new '
285 "is a line the change added (its number in the new file), side old a line it removed "
286 "(its number in the old file); anything else is dropped. Use the rule ids listed above; "
287 "a finding on another rule is an error only when its evidence quotes the changed line. "
288 f"At most {limit} findings, errors first in rule priority order. Prefer false "
289 "negatives: report only what you confirmed within the rule's budget. The harness "
290 "recomputes the verdict from the findings. This request is read-only: do not change "
291 "any file."
292 )
295__all__ = [
296 "OUT_OF_CATALOG_PRIORITY",
297 "SEVERITIES",
298 "VERDICTS",
299 "ContractError",
300 "Normalized",
301 "ReviewFinding",
302 "Verdict",
303 "concrete",
304 "contract_text",
305 "normalize",
306 "parse_answer",
307 "verdict_of",
308]