Coverage for src/governed_harness/orchestration/agent_review.py: 88%

102 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Second-agent review in INDEPENDENT_REVIEW (#38). 

2 

3Validators check what tools can check; blind reviews of the evaluated implementations found 

4defects no validator saw (unrestricted pickle of stored state, wrong fare formulas, broken public 

5signatures, card data stored in full). Under ``review.agentReview`` the harness asks a provider 

6(call kind ``review``, read-only, optionally another provider or model through 

7``review.reviewer``) to compare the ChangeSet with the task's requirements, criteria and 

8constraints and return findings with severity and location. 

9 

10Cost rule: the review runs only after every blocking deterministic validator passed (no reviewer 

11call while tests, lint or types fail) and again only when the ChangeSet digest changed since the 

12last review. Under ``enforce`` its HIGH and CRITICAL findings block the gate and, while the 

13correction budget (``runtime.verificationCorrections``) lasts, return the run to IMPLEMENTATION 

14with the findings as feedback; under ``warn`` they are recorded as MEDIUM at most. The request, 

15the reviewer's identity and its output are evidence.""" 

16 

17from __future__ import annotations 

18 

19from typing import TYPE_CHECKING, Any 

20 

21from governed_harness.agents.requests import REVIEW_SEVERITIES 

22from governed_harness.domain.enums import ( 

23 ActorType, 

24 FindingSeverity, 

25 ResultStatus, 

26 ValidationKind, 

27) 

28from governed_harness.domain.models import ( 

29 Actor, 

30 ChangeSet, 

31 Execution, 

32 Finding, 

33 PhaseExecution, 

34) 

35from governed_harness.orchestration.engine_types import ReviewOutcome as ReviewOutcome 

36 

37if TYPE_CHECKING: 

38 from governed_harness.orchestration.engine_types import AgentCallOutcome 

39 from governed_harness.orchestration.hosts import ResultsHost 

40 

41AGENT_REVIEW_ID = "review.agent" 

42MAX_DIFF_CHARS = 200_000 

43MAX_FINDINGS = 50 

44_RANK = {"INFO": 0, "LOW": 1, "MEDIUM": 2, "HIGH": 3, "CRITICAL": 4} 

45 

46 

47class AgentReview: 

48 def __init__(self, results: ResultsHost) -> None: 

49 self.results = results 

50 

51 @property 

52 def policy(self) -> str: 

53 review = self.results.project.review 

54 if review is not None and review.panel is not None and review.panel.mode is not None: 

55 # #57: the review panel replaces the single reviewer under its own policy. 

56 return review.panel.mode 

57 return (review.agent_review if review else None) or "off" 

58 

59 def blocking_severities(self) -> set[FindingSeverity]: 

60 names = self.results.s.resolved.effective_policies.get( 

61 "findingBlockSeverities", ["HIGH", "CRITICAL"] 

62 ) 

63 return {FindingSeverity(str(name)) for name in names} 

64 

65 def run( 

66 self, execution: Execution, phase: PhaseExecution, change_set: ChangeSet 

67 ) -> ReviewOutcome: 

68 if self.policy == "off": 

69 return ReviewOutcome(False) 

70 key = f"agentreview:{execution.execution_id}" 

71 if self._skipped(execution, change_set, key): 

72 return ReviewOutcome(False) 

73 if self.results.panel.configured: 

74 return self.results.panel.run(execution, phase, change_set) 

75 return self._review(execution, phase, change_set, key) 

76 

77 def _skipped(self, execution: Execution, change_set: ChangeSet, key: str) -> bool: 

78 """Whether the review is skipped (and the event recorded): the ChangeSet was reviewed 

79 already, or a blocking deterministic check fails.""" 

80 results = self.results 

81 engine = results.engine 

82 last = results.flag_json(key) 

83 if isinstance(last, dict) and last.get("digest") == change_set.digest: 

84 results.s.events.append( 

85 execution.execution_id, 

86 "review.agent.skipped", 

87 { 

88 "reason": "ChangeSet digest unchanged since the last review", 

89 "digest": change_set.digest, 

90 }, 

91 ) 

92 return True 

93 validations = engine._latest_validations(execution.execution_id, change_set.digest) 

94 failing = [ 

95 item.validator_id 

96 for item in validations 

97 if item.mandatory and item.status is not ResultStatus.PASSED 

98 ] 

99 blocking_ids = {finding_id for item in validations for finding_id in item.finding_ids} 

100 blocking_findings = [ 

101 item 

102 for item in results.s.state.list( 

103 "finding", Finding, execution_id=execution.execution_id 

104 ) 

105 if item.finding_id in blocking_ids and item.severity in self.blocking_severities() 

106 ] 

107 if failing or blocking_findings: 107 ↛ 109line 107 didn't jump to line 109 because the condition on line 107 was never true

108 # Cost rule: no reviewer call while a blocking deterministic check fails. 

109 results.s.events.append( 

110 execution.execution_id, 

111 "review.agent.skipped", 

112 { 

113 "reason": "blocking deterministic validators or findings", 

114 "failingValidators": failing, 

115 "blockingFindings": len(blocking_findings), 

116 }, 

117 ) 

118 return True 

119 return False 

120 

121 def _review( 

122 self, execution: Execution, phase: PhaseExecution, change_set: ChangeSet, key: str 

123 ) -> ReviewOutcome: 

124 """The call to the reviewer agent and the record of its answer.""" 

125 results = self.results 

126 engine = results.engine 

127 task = engine.run_task(execution) 

128 diff = engine._compute_owned_diff(execution).unified_diff.decode("utf-8", "replace") 

129 payload: dict[str, Any] = { 

130 "changeSet": { 

131 "digest": change_set.digest, 

132 "files": [item.model_dump(mode="json", by_alias=True) for item in change_set.files], 

133 "diff": diff[:MAX_DIFF_CHARS], 

134 "diffTruncated": len(diff) > MAX_DIFF_CHARS, 

135 }, 

136 "severities": list(REVIEW_SEVERITIES), 

137 } 

138 suffix = "" 

139 if results.engineering.configured: 

140 # The standards cards no tool verifies and the principles checklist ride on this 

141 # call (#56): no extra agent call. 

142 checklist, suffix = results.engineering.review_extra(change_set) 

143 payload.update(checklist) 

144 outcome = results.call_agent( 

145 execution, 

146 phase, 

147 "review", 

148 payload, 

149 task=task, 

150 instructions_suffix=suffix, 

151 validate=_findings_from, 

152 ) 

153 provider = results.provider_for(execution, "review") 

154 actor = Actor(actor_type=ActorType.AGENT, actor_id=f"agent.{provider}", version="1") 

155 enforce = self.policy == "enforce" 

156 if outcome.status is not ResultStatus.PASSED or outcome.result is None: 

157 self._record( 

158 execution, 

159 change_set, 

160 (), 

161 ResultStatus.BLOCKED if enforce else ResultStatus.PASSED, 

162 f"The agent review did not answer ({outcome.status}): {outcome.summary}", 

163 outcome.evidence_refs, 

164 enforce, 

165 ) 

166 return ReviewOutcome(True) 

167 try: 

168 items = _findings_from(outcome.result) 

169 except ValueError as error: 

170 note = results.record_finding( 

171 execution, 

172 validator_id=AGENT_REVIEW_ID, 

173 rule_id="review.agent-malformed", 

174 category="agent-protocol", 

175 severity=FindingSeverity.HIGH, 

176 message=f"The agent review returned a malformed result: {error}", 

177 evidence_refs=outcome.evidence_refs, 

178 ) 

179 self._record( 

180 execution, 

181 change_set, 

182 (note,), 

183 ResultStatus.BLOCKED if enforce else ResultStatus.PASSED, 

184 f"The agent review was malformed: {error}", 

185 outcome.evidence_refs, 

186 enforce, 

187 ) 

188 return ReviewOutcome(True) 

189 findings = self._record_findings(execution, items, outcome, actor, enforce) 

190 blocking = tuple(item for item in findings if item.severity in self.blocking_severities()) 

191 self._record( 

192 execution, 

193 change_set, 

194 tuple(findings), 

195 ResultStatus.PASSED, 

196 f"Agent review by {provider}: {len(findings)} finding(s), {len(blocking)} blocking", 

197 outcome.evidence_refs, 

198 enforce, 

199 ) 

200 results.set_flag_json(key, {"digest": change_set.digest}) 

201 return ReviewOutcome(True, blocking if enforce else ()) 

202 

203 def _record_findings( 

204 self, 

205 execution: Execution, 

206 items: list[dict[str, Any]], 

207 outcome: AgentCallOutcome, 

208 actor: Actor, 

209 enforce: bool, 

210 ) -> list[Finding]: 

211 """Record the reviewer's findings (at most MAX_FINDINGS); outside ``enforce`` none is 

212 above MEDIUM.""" 

213 results = self.results 

214 findings: list[Finding] = [] 

215 for item in items[:MAX_FINDINGS]: 

216 severity = FindingSeverity(item["severity"]) 

217 if not enforce and _RANK[severity.value] > _RANK["MEDIUM"]: 

218 severity = FindingSeverity.MEDIUM 

219 findings.append( 

220 results.record_finding( 

221 execution, 

222 validator_id=AGENT_REVIEW_ID, 

223 rule_id=f"review.agent.{item['rule']}", 

224 category="agent-review", 

225 severity=severity, 

226 message=item["message"], 

227 path=item["path"], 

228 line=item["line"], 

229 evidence_refs=outcome.evidence_refs, 

230 recommendation=( 

231 f"Evidence given by the reviewer: {item['evidence']}" 

232 if item["evidence"] 

233 else None 

234 ), 

235 introduced=True, 

236 actor=actor, 

237 ) 

238 ) 

239 return findings 

240 

241 def _record( 

242 self, 

243 execution: Execution, 

244 change_set: ChangeSet, 

245 findings: tuple[Finding, ...], 

246 status: ResultStatus, 

247 summary: str, 

248 refs: tuple[str, ...], 

249 mandatory: bool, 

250 ) -> None: 

251 self.results.record_validation( 

252 execution, 

253 validator_id=AGENT_REVIEW_ID, 

254 digest=change_set.digest, 

255 status=status, 

256 kind=ValidationKind.SUCCESS 

257 if status is ResultStatus.PASSED 

258 else ValidationKind.PROTOCOL_ERROR, 

259 mandatory=mandatory, 

260 summary=summary, 

261 findings=findings, 

262 evidence_refs=refs or (change_set.diff_ref,), 

263 ) 

264 

265 

266def _findings_from(result: dict[str, Any]) -> list[dict[str, Any]]: 

267 raw = result.get("findings") 

268 if not isinstance(raw, list): 268 ↛ 269line 268 didn't jump to line 269 because the condition on line 268 was never true

269 raise ValueError("the review result needs a 'findings' list") 

270 return [_finding_item(index, entry) for index, entry in enumerate(raw, start=1)] 

271 

272 

273def _finding_item(index: int, entry: Any) -> dict[str, Any]: 

274 """One validated finding of the review result (the ``index`` counts from 1).""" 

275 if not isinstance(entry, dict): 275 ↛ 276line 275 didn't jump to line 276 because the condition on line 275 was never true

276 raise ValueError(f"finding {index} is not an object") 

277 severity = str(entry.get("severity", "")).upper() 

278 if severity not in REVIEW_SEVERITIES: 278 ↛ 279line 278 didn't jump to line 279 because the condition on line 278 was never true

279 raise ValueError(f"finding {index} has an unknown severity {entry.get('severity')!r}") 

280 message = entry.get("message") 

281 if not isinstance(message, str) or not message.strip(): 281 ↛ 282line 281 didn't jump to line 282 because the condition on line 281 was never true

282 raise ValueError(f"finding {index} has no message") 

283 rule = ( 

284 "".join( 

285 char if char.isalnum() or char in "-_." else "-" 

286 for char in str(entry.get("rule") or "finding").strip().lower() 

287 )[:60] 

288 or "finding" 

289 ) 

290 line = entry.get("line") 

291 return { 

292 "severity": severity, 

293 "rule": rule, 

294 "path": str(entry["path"]) if entry.get("path") else None, 

295 "line": line if isinstance(line, int) and line >= 1 else None, 

296 "message": " ".join(message.split())[:1000], 

297 "evidence": " ".join(str(entry.get("evidence") or "").split())[:1000], 

298 }