Coverage for src/governed_harness/orchestration/agent_review.py: 88%
102 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Second-agent review in INDEPENDENT_REVIEW (#38).
3Validators check what tools can check; blind reviews of the evaluated implementations found
4defects no validator saw (unrestricted pickle of stored state, wrong fare formulas, broken public
5signatures, card data stored in full). Under ``review.agentReview`` the harness asks a provider
6(call kind ``review``, read-only, optionally another provider or model through
7``review.reviewer``) to compare the ChangeSet with the task's requirements, criteria and
8constraints and return findings with severity and location.
10Cost rule: the review runs only after every blocking deterministic validator passed (no reviewer
11call while tests, lint or types fail) and again only when the ChangeSet digest changed since the
12last review. Under ``enforce`` its HIGH and CRITICAL findings block the gate and, while the
13correction budget (``runtime.verificationCorrections``) lasts, return the run to IMPLEMENTATION
14with the findings as feedback; under ``warn`` they are recorded as MEDIUM at most. The request,
15the reviewer's identity and its output are evidence."""
17from __future__ import annotations
19from typing import TYPE_CHECKING, Any
21from governed_harness.agents.requests import REVIEW_SEVERITIES
22from governed_harness.domain.enums import (
23 ActorType,
24 FindingSeverity,
25 ResultStatus,
26 ValidationKind,
27)
28from governed_harness.domain.models import (
29 Actor,
30 ChangeSet,
31 Execution,
32 Finding,
33 PhaseExecution,
34)
35from governed_harness.orchestration.engine_types import ReviewOutcome as ReviewOutcome
37if TYPE_CHECKING:
38 from governed_harness.orchestration.engine_types import AgentCallOutcome
39 from governed_harness.orchestration.hosts import ResultsHost
41AGENT_REVIEW_ID = "review.agent"
42MAX_DIFF_CHARS = 200_000
43MAX_FINDINGS = 50
44_RANK = {"INFO": 0, "LOW": 1, "MEDIUM": 2, "HIGH": 3, "CRITICAL": 4}
47class AgentReview:
48 def __init__(self, results: ResultsHost) -> None:
49 self.results = results
51 @property
52 def policy(self) -> str:
53 review = self.results.project.review
54 if review is not None and review.panel is not None and review.panel.mode is not None:
55 # #57: the review panel replaces the single reviewer under its own policy.
56 return review.panel.mode
57 return (review.agent_review if review else None) or "off"
59 def blocking_severities(self) -> set[FindingSeverity]:
60 names = self.results.s.resolved.effective_policies.get(
61 "findingBlockSeverities", ["HIGH", "CRITICAL"]
62 )
63 return {FindingSeverity(str(name)) for name in names}
65 def run(
66 self, execution: Execution, phase: PhaseExecution, change_set: ChangeSet
67 ) -> ReviewOutcome:
68 if self.policy == "off":
69 return ReviewOutcome(False)
70 key = f"agentreview:{execution.execution_id}"
71 if self._skipped(execution, change_set, key):
72 return ReviewOutcome(False)
73 if self.results.panel.configured:
74 return self.results.panel.run(execution, phase, change_set)
75 return self._review(execution, phase, change_set, key)
77 def _skipped(self, execution: Execution, change_set: ChangeSet, key: str) -> bool:
78 """Whether the review is skipped (and the event recorded): the ChangeSet was reviewed
79 already, or a blocking deterministic check fails."""
80 results = self.results
81 engine = results.engine
82 last = results.flag_json(key)
83 if isinstance(last, dict) and last.get("digest") == change_set.digest:
84 results.s.events.append(
85 execution.execution_id,
86 "review.agent.skipped",
87 {
88 "reason": "ChangeSet digest unchanged since the last review",
89 "digest": change_set.digest,
90 },
91 )
92 return True
93 validations = engine._latest_validations(execution.execution_id, change_set.digest)
94 failing = [
95 item.validator_id
96 for item in validations
97 if item.mandatory and item.status is not ResultStatus.PASSED
98 ]
99 blocking_ids = {finding_id for item in validations for finding_id in item.finding_ids}
100 blocking_findings = [
101 item
102 for item in results.s.state.list(
103 "finding", Finding, execution_id=execution.execution_id
104 )
105 if item.finding_id in blocking_ids and item.severity in self.blocking_severities()
106 ]
107 if failing or blocking_findings: 107 ↛ 109line 107 didn't jump to line 109 because the condition on line 107 was never true
108 # Cost rule: no reviewer call while a blocking deterministic check fails.
109 results.s.events.append(
110 execution.execution_id,
111 "review.agent.skipped",
112 {
113 "reason": "blocking deterministic validators or findings",
114 "failingValidators": failing,
115 "blockingFindings": len(blocking_findings),
116 },
117 )
118 return True
119 return False
121 def _review(
122 self, execution: Execution, phase: PhaseExecution, change_set: ChangeSet, key: str
123 ) -> ReviewOutcome:
124 """The call to the reviewer agent and the record of its answer."""
125 results = self.results
126 engine = results.engine
127 task = engine.run_task(execution)
128 diff = engine._compute_owned_diff(execution).unified_diff.decode("utf-8", "replace")
129 payload: dict[str, Any] = {
130 "changeSet": {
131 "digest": change_set.digest,
132 "files": [item.model_dump(mode="json", by_alias=True) for item in change_set.files],
133 "diff": diff[:MAX_DIFF_CHARS],
134 "diffTruncated": len(diff) > MAX_DIFF_CHARS,
135 },
136 "severities": list(REVIEW_SEVERITIES),
137 }
138 suffix = ""
139 if results.engineering.configured:
140 # The standards cards no tool verifies and the principles checklist ride on this
141 # call (#56): no extra agent call.
142 checklist, suffix = results.engineering.review_extra(change_set)
143 payload.update(checklist)
144 outcome = results.call_agent(
145 execution,
146 phase,
147 "review",
148 payload,
149 task=task,
150 instructions_suffix=suffix,
151 validate=_findings_from,
152 )
153 provider = results.provider_for(execution, "review")
154 actor = Actor(actor_type=ActorType.AGENT, actor_id=f"agent.{provider}", version="1")
155 enforce = self.policy == "enforce"
156 if outcome.status is not ResultStatus.PASSED or outcome.result is None:
157 self._record(
158 execution,
159 change_set,
160 (),
161 ResultStatus.BLOCKED if enforce else ResultStatus.PASSED,
162 f"The agent review did not answer ({outcome.status}): {outcome.summary}",
163 outcome.evidence_refs,
164 enforce,
165 )
166 return ReviewOutcome(True)
167 try:
168 items = _findings_from(outcome.result)
169 except ValueError as error:
170 note = results.record_finding(
171 execution,
172 validator_id=AGENT_REVIEW_ID,
173 rule_id="review.agent-malformed",
174 category="agent-protocol",
175 severity=FindingSeverity.HIGH,
176 message=f"The agent review returned a malformed result: {error}",
177 evidence_refs=outcome.evidence_refs,
178 )
179 self._record(
180 execution,
181 change_set,
182 (note,),
183 ResultStatus.BLOCKED if enforce else ResultStatus.PASSED,
184 f"The agent review was malformed: {error}",
185 outcome.evidence_refs,
186 enforce,
187 )
188 return ReviewOutcome(True)
189 findings = self._record_findings(execution, items, outcome, actor, enforce)
190 blocking = tuple(item for item in findings if item.severity in self.blocking_severities())
191 self._record(
192 execution,
193 change_set,
194 tuple(findings),
195 ResultStatus.PASSED,
196 f"Agent review by {provider}: {len(findings)} finding(s), {len(blocking)} blocking",
197 outcome.evidence_refs,
198 enforce,
199 )
200 results.set_flag_json(key, {"digest": change_set.digest})
201 return ReviewOutcome(True, blocking if enforce else ())
203 def _record_findings(
204 self,
205 execution: Execution,
206 items: list[dict[str, Any]],
207 outcome: AgentCallOutcome,
208 actor: Actor,
209 enforce: bool,
210 ) -> list[Finding]:
211 """Record the reviewer's findings (at most MAX_FINDINGS); outside ``enforce`` none is
212 above MEDIUM."""
213 results = self.results
214 findings: list[Finding] = []
215 for item in items[:MAX_FINDINGS]:
216 severity = FindingSeverity(item["severity"])
217 if not enforce and _RANK[severity.value] > _RANK["MEDIUM"]:
218 severity = FindingSeverity.MEDIUM
219 findings.append(
220 results.record_finding(
221 execution,
222 validator_id=AGENT_REVIEW_ID,
223 rule_id=f"review.agent.{item['rule']}",
224 category="agent-review",
225 severity=severity,
226 message=item["message"],
227 path=item["path"],
228 line=item["line"],
229 evidence_refs=outcome.evidence_refs,
230 recommendation=(
231 f"Evidence given by the reviewer: {item['evidence']}"
232 if item["evidence"]
233 else None
234 ),
235 introduced=True,
236 actor=actor,
237 )
238 )
239 return findings
241 def _record(
242 self,
243 execution: Execution,
244 change_set: ChangeSet,
245 findings: tuple[Finding, ...],
246 status: ResultStatus,
247 summary: str,
248 refs: tuple[str, ...],
249 mandatory: bool,
250 ) -> None:
251 self.results.record_validation(
252 execution,
253 validator_id=AGENT_REVIEW_ID,
254 digest=change_set.digest,
255 status=status,
256 kind=ValidationKind.SUCCESS
257 if status is ResultStatus.PASSED
258 else ValidationKind.PROTOCOL_ERROR,
259 mandatory=mandatory,
260 summary=summary,
261 findings=findings,
262 evidence_refs=refs or (change_set.diff_ref,),
263 )
266def _findings_from(result: dict[str, Any]) -> list[dict[str, Any]]:
267 raw = result.get("findings")
268 if not isinstance(raw, list): 268 ↛ 269line 268 didn't jump to line 269 because the condition on line 268 was never true
269 raise ValueError("the review result needs a 'findings' list")
270 return [_finding_item(index, entry) for index, entry in enumerate(raw, start=1)]
273def _finding_item(index: int, entry: Any) -> dict[str, Any]:
274 """One validated finding of the review result (the ``index`` counts from 1)."""
275 if not isinstance(entry, dict): 275 ↛ 276line 275 didn't jump to line 276 because the condition on line 275 was never true
276 raise ValueError(f"finding {index} is not an object")
277 severity = str(entry.get("severity", "")).upper()
278 if severity not in REVIEW_SEVERITIES: 278 ↛ 279line 278 didn't jump to line 279 because the condition on line 278 was never true
279 raise ValueError(f"finding {index} has an unknown severity {entry.get('severity')!r}")
280 message = entry.get("message")
281 if not isinstance(message, str) or not message.strip(): 281 ↛ 282line 281 didn't jump to line 282 because the condition on line 281 was never true
282 raise ValueError(f"finding {index} has no message")
283 rule = (
284 "".join(
285 char if char.isalnum() or char in "-_." else "-"
286 for char in str(entry.get("rule") or "finding").strip().lower()
287 )[:60]
288 or "finding"
289 )
290 line = entry.get("line")
291 return {
292 "severity": severity,
293 "rule": rule,
294 "path": str(entry["path"]) if entry.get("path") else None,
295 "line": line if isinstance(line, int) and line >= 1 else None,
296 "message": " ".join(message.split())[:1000],
297 "evidence": " ".join(str(entry.get("evidence") or "").split())[:1000],
298 }