Coverage for src/governed_harness/orchestration/review_panel.py: 90%

193 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""The review panel in INDEPENDENT_REVIEW (#57), replacing the single reviewer of #38. 

2 

3Under ``review.panel`` the second review of a run is the panel of ``governed_harness.review``: 

4the same cost rule as before (no reviewer while a blocking deterministic validator fails, and 

5again only when the ChangeSet digest changed), then the consistency checks, the deterministic 

6rules and the reviewers by domain over their slices, in parallel and read-only. The rules a 

7linter verifies are not run again here: the validators of VERIFICATION already ran them. 

8 

9Every request, answer, sandbox and the report are evidence of the run; the findings are 

10recorded under ``review.agent`` so the gate, the brief and the corrections treat them as the 

11single reviewer's. Errors are ``HIGH`` under ``enforce`` (they block the gate) and ``MEDIUM`` 

12under ``warn``; suggestions are ``LOW``; a reviewer that never answered (``UNKNOWN``) blocks. 

13 

14Scoped auto-fix (``review.panel.autoFix: scoped``): only errors on lines the agent wrote in this 

15run go back to IMPLEMENTATION: an added line of a file whose provenance is the agent's 

16(``provenance.agentSnapshots``), never a line of a file a person also edited, never a removed 

17line and never a suggestion; at most ``maxAttempts`` times, each one recorded. Other errors stay 

18for the person who decides.""" 

19 

20from __future__ import annotations 

21 

22import threading 

23from collections.abc import Callable, Sequence 

24from typing import TYPE_CHECKING, Any 

25 

26from governed_harness import __version__ 

27from governed_harness.agents import AgentCallResult 

28from governed_harness.agents.routing import RoutingDecision 

29from governed_harness.capabilities import grants_from_rules 

30from governed_harness.configuration.review import ReviewPanelConfig 

31from governed_harness.domain.enums import ( 

32 ActorType, 

33 FindingSeverity, 

34 PhaseId, 

35 ResultStatus, 

36 ValidationKind, 

37) 

38from governed_harness.domain.models import ( 

39 Actor, 

40 ChangeSet, 

41 ComponentProvenance, 

42 Execution, 

43 Finding, 

44 PhaseExecution, 

45) 

46from governed_harness.orchestration.agent_review import AGENT_REVIEW_ID 

47from governed_harness.orchestration.engine_types import READ_ONLY_RULE, ReviewOutcome 

48from governed_harness.review.cache import ReviewCache 

49from governed_harness.review.contract import ReviewFinding 

50from governed_harness.review.invoke import BuiltProvider, InvokerHooks, ProviderInvoker 

51from governed_harness.review.panel import PanelInputs, PanelReport, ReviewerCall, run_panel 

52from governed_harness.review.project import ( 

53 consistency_runner, 

54 panel_context, 

55 review_setup, 

56 reviewer_route, 

57) 

58from governed_harness.review.providers import build_reviewer_provider, mcp_digest, mcp_servers 

59from governed_harness.review.reviewers import Reviewer 

60from governed_harness.runtime.workspace import WorkspaceDiff 

61 

62if TYPE_CHECKING: 

63 from governed_harness.orchestration.hosts import ResultsHost 

64 

65AUTOFIX_FLAG = "reviewfix" 

66PANEL_RULE_PREFIX = "review.panel" 

67 

68 

69class _SerializedObserver: 

70 """The process ledger of a run, written by one thread at a time (reviewers run in 

71 parallel).""" 

72 

73 def __init__(self, inner: Any) -> None: 

74 self.inner = inner 

75 self.lock = threading.Lock() 

76 

77 def started(self, pid: int, argv: tuple[str, ...]) -> None: 

78 with self.lock: 

79 self.inner.started(pid, argv) 

80 

81 def finished(self, pid: int) -> None: 

82 with self.lock: 

83 self.inner.finished(pid) 

84 

85 

86def compact_task(task: Any) -> dict[str, Any]: 

87 """What a reviewer needs of the task: the intent, the requirement and criterion texts and 

88 the constraints (not the whole task document).""" 

89 return { 

90 "intent": task.intent, 

91 "requirements": [item.text for item in task.requirements], 

92 "acceptanceCriteria": [item.text for item in task.acceptance_criteria], 

93 "constraints": list(task.constraints), 

94 } 

95 

96 

97class PanelReview: 

98 def __init__(self, results: ResultsHost) -> None: 

99 self.results = results 

100 

101 @property 

102 def settings(self) -> ReviewPanelConfig | None: 

103 review = self.results.project.review 

104 return review.panel if review is not None else None 

105 

106 def enabled_settings(self) -> ReviewPanelConfig: 

107 """The panel's settings, for the steps that run only when it is configured.""" 

108 settings = self.settings 

109 if settings is None: # not reached: the review runs the panel only when configured 109 ↛ 110line 109 didn't jump to line 110 because the condition on line 109 was never true

110 raise RuntimeError("review.panel is not configured") 

111 return settings 

112 

113 @property 

114 def configured(self) -> bool: 

115 settings = self.settings 

116 return settings is not None and settings.enabled 

117 

118 # ----- routing of the reviewers (#85) ----------------------------------------------------- 

119 def _route_observer( 

120 self, decisions: dict[tuple[str, str], RoutingDecision] 

121 ) -> Callable[[Reviewer, str, RoutingDecision], None]: 

122 """Keeps the routing decision of each reviewer on each provider, recorded with its call 

123 as for any other call: under ``agentRouting``, or for a reviewer's own model.""" 

124 routed = self.results.project.agent_routing is not None 

125 

126 def heard(reviewer: Reviewer, provider_id: str, decision: RoutingDecision) -> None: 

127 if routed or decision.rule.startswith("reviewer:"): 

128 decisions[(reviewer.reviewer_id, provider_id)] = decision 

129 

130 return heard 

131 

132 def _record_routing( 

133 self, 

134 execution: Execution, 

135 phase: PhaseExecution, 

136 call: ReviewerCall, 

137 decision: RoutingDecision, 

138 ) -> None: 

139 """``agent.routing.decided`` of one reviewer call, as every other agent call records it, 

140 with the reviewer and the attempt.""" 

141 results = self.results 

142 record = { 

143 **decision.as_dict(), 

144 "provider": call.provider, 

145 "phase": phase.phase_id, 

146 "reviewer": call.reviewer, 

147 "attempt": call.attempt, 

148 } 

149 ref = results.record_json( 

150 execution, 

151 phase.phase_id, 

152 record, 

153 kind="agent-routing", 

154 summary=( 

155 f"Routing of the {call.reviewer} reviewer: {decision.rule} -> " 

156 f"{decision.model or 'provider default'}/{decision.effort or 'default'}" 

157 ), 

158 ) 

159 results.s.events.append( 

160 execution.execution_id, 

161 "agent.routing.decided", 

162 {**record, "evidenceRef": ref}, 

163 phase_execution_id=phase.phase_execution_id, 

164 ) 

165 

166 # ----- the invoker ----------------------------------------------------------------------- 

167 def _invoker( 

168 self, 

169 execution: Execution, 

170 phase: PhaseExecution, 

171 servers: dict[str, Any], 

172 decisions: dict[tuple[str, str], RoutingDecision], 

173 ) -> ProviderInvoker: 

174 results = self.results 

175 engine = results.engine 

176 resolved = results.s.resolved 

177 workspace = results.s.paths.workspace 

178 runtime = resolved.project.runtime 

179 runner = engine._runner(execution) 

180 if runner.observer is not None: 180 ↛ 183line 180 didn't jump to line 183 because the condition on line 180 was always true

181 runner.observer = _SerializedObserver(runner.observer) 

182 

183 def build(provider_id: str) -> BuiltProvider | str: 

184 built = build_reviewer_provider( 

185 resolved, 

186 provider_id, 

187 workspace=workspace, 

188 protected=engine._protected_paths(), 

189 host=engine.sandbox_host, 

190 allow_network=engine._agent_network_allowed(), 

191 ) 

192 if isinstance(built, str): 192 ↛ 193line 192 didn't jump to line 193 because the condition on line 192 was never true

193 return built 

194 refs: tuple[str, ...] = () 

195 if built.sandbox is not None: 195 ↛ 196line 195 didn't jump to line 196 because the condition on line 195 was never true

196 refs = ( 

197 results.record_json( 

198 execution, 

199 phase.phase_id, 

200 {**built.sandbox.evidence(), "readOnlyWorkspace": True}, 

201 kind="review-sandbox", 

202 summary=f"Read-only sandbox of the reviewers on {provider_id}", 

203 ), 

204 ) 

205 return BuiltProvider(built.built.provider, built.built.actor, refs) 

206 

207 def before(_calls: Sequence[ReviewerCall]) -> str | None: 

208 blocked = results.before_agent_call(execution, phase, "review") 

209 return blocked.summary if blocked is not None else None 

210 

211 def record_request(call: ReviewerCall) -> str | None: 

212 decision = decisions.get((call.reviewer, call.provider)) 

213 if decision is not None: 

214 self._record_routing(execution, phase, call, decision) 

215 return results.record_json( 

216 execution, 

217 phase.phase_id, 

218 call.request, 

219 kind="agent-review-request", 

220 summary=f"Review request of the {call.reviewer} reviewer to {call.provider}", 

221 ) 

222 

223 def after(_call: ReviewerCall, result: AgentCallResult) -> None: 

224 engine._save_agent_result(execution, phase, result.execution) 

225 results.after_agent_call(execution, phase, result.execution) 

226 

227 def on_violation(diff: WorkspaceDiff, restored: int, unrestorable: int) -> None: 

228 shown = ", ".join(item.path for item in diff.changes[:10]) 

229 results.record_finding( 

230 execution, 

231 validator_id="harness.agent-calls", 

232 rule_id=READ_ONLY_RULE, 

233 category="agent-protocol", 

234 severity=FindingSeverity.HIGH, 

235 message=( 

236 f"A reviewer of the review panel changed {len(diff.changes)} path(s): " 

237 f"{shown}; restored {restored}, could not restore {unrestorable}" 

238 ), 

239 path=diff.changes[0].path, 

240 recommendation="Use a provider that honours readOnly requests.", 

241 ) 

242 

243 judge, timeout = self._contract_judge(execution, phase) 

244 return ProviderInvoker( 

245 workspace=workspace, 

246 artifacts=results.s.artifacts, 

247 runner=runner, 

248 provenance=engine._provenance(execution), 

249 build=build, 

250 grants_for=lambda actor: grants_from_rules( 

251 execution.execution_id, actor, resolved.effective_capabilities 

252 ), 

253 execution_id=execution.execution_id, 

254 phase_id=PhaseId.INDEPENDENT_REVIEW, 

255 default_timeout=timeout(engine._bounded_timeout(runtime.command_timeout_seconds)), 

256 max_output_bytes=runtime.max_output_bytes, 

257 hooks=InvokerHooks( 

258 cancelled=lambda: engine.is_cancelled(execution.execution_id), 

259 before=before, 

260 record_request=record_request, 

261 after=after, 

262 on_violation=on_violation, 

263 judge=judge, 

264 ), 

265 mcp_servers=servers, 

266 ) 

267 

268 def _contract_judge( 

269 self, execution: Execution, phase: PhaseExecution 

270 ) -> tuple[ 

271 Callable[[ReviewerCall, AgentCallResult, str | None], str | None] | None, 

272 Callable[[int], int], 

273 ]: 

274 """Under ``agentContract: enforce`` (#149) every reviewer answer is a ``review`` call of 

275 the contract: judged against the token budget, the time limit (which also shortens the 

276 reviewers' timeout) and the result limits, and recorded; ``(None, unchanged)`` without 

277 it, so the panel behaves as before.""" 

278 flow = self.results.agent_contract 

279 if not flow.enabled: 

280 return None, lambda seconds: seconds 

281 limits = flow.limits(execution, self.results.engine.run_task(execution), "review") 

282 

283 def judge( 

284 call: ReviewerCall, result: AgentCallResult, request_ref: str | None 

285 ) -> str | None: 

286 return flow.settle_reviewer( 

287 execution, 

288 phase, 

289 call.reviewer, 

290 result, 

291 request=call.request, 

292 request_ref=request_ref, 

293 limits=limits, 

294 ) 

295 

296 return judge, lambda seconds: flow.timeout(seconds, limits) 

297 

298 # ----- the review -------------------------------------------------------------------------- 

299 def run( 

300 self, execution: Execution, phase: PhaseExecution, change_set: ChangeSet 

301 ) -> ReviewOutcome: 

302 settings = self.enabled_settings() 

303 results = self.results 

304 engine = results.engine 

305 resolved = results.s.resolved 

306 workspace = results.s.paths.workspace 

307 setup = review_setup(resolved, workspace) 

308 servers = mcp_servers(workspace, settings.mcp_servers) 

309 review = resolved.project.review 

310 reviewer_config = review.reviewer if review is not None else None 

311 provider = ( 

312 settings.provider 

313 or (reviewer_config.provider if reviewer_config else None) 

314 or results.provider_for(execution, "review") 

315 ) 

316 cache_settings = settings.effective_cache 

317 diff = engine._compute_owned_diff(execution).unified_diff.decode("utf-8", "replace") 

318 untrusted, notice = panel_context(resolved, workspace) 

319 decisions: dict[tuple[str, str], RoutingDecision] = {} 

320 inputs = PanelInputs( 

321 workspace=workspace, 

322 diff_text=diff, 

323 mode="run", 

324 catalog=setup.catalog, 

325 reviewers=setup.reviewers, 

326 settings=settings, 

327 provider=provider, 

328 invoker=self._invoker(execution, phase, servers, decisions), 

329 route=reviewer_route(resolved, self._route_observer(decisions)), 

330 runner_version=__version__, 

331 base=results.s.state.get_flag(f"baseline:{execution.execution_id}") or None, 

332 head=change_set.digest, 

333 fallback=settings.fallback_provider, 

334 packs=setup.packs or None, 

335 cache=ReviewCache( 

336 results.s.paths.harness_dir / "review" / "cache", 

337 ttl_seconds=cache_settings.ttl_seconds, 

338 max_entries=cache_settings.limit, 

339 ) 

340 if cache_settings.on 

341 else None, 

342 consistency=consistency_runner( 

343 resolved, 

344 workspace, 

345 engine._runner(execution), 

346 execution_id=execution.execution_id, 

347 cancelled=lambda: engine.is_cancelled(execution.execution_id), 

348 ), 

349 extra={"task": compact_task(engine.run_task(execution)), **untrusted}, 

350 untrusted_notice=notice, 

351 mcp={"mcpServers": servers, "digest": mcp_digest(servers)}, 

352 ) 

353 report = run_panel(inputs) 

354 enforce = settings.mode != "warn" 

355 report_ref = results.record_json( 

356 execution, 

357 phase.phase_id, 

358 report.as_dict(), 

359 kind="review-panel-report", 

360 summary=( 

361 f"Review panel: {report.verdict}, {len(report.findings)} finding(s), " 

362 f"{report.tokens} reported token(s), cache {report.cache}" 

363 ), 

364 ) 

365 results.s.events.append( 

366 execution.execution_id, 

367 "review.panel.completed", 

368 { 

369 "verdict": report.verdict, 

370 "digest": report.digest, 

371 "findings": len(report.findings), 

372 "reviewers": {item.reviewer: item.status for item in report.reviewers}, 

373 "tokens": report.tokens, 

374 "cache": report.cache, 

375 "evidenceRef": report_ref, 

376 }, 

377 phase_execution_id=phase.phase_execution_id, 

378 ) 

379 findings, by_key = self._record_findings(execution, report, report_ref, enforce) 

380 unknown = [item.reviewer for item in report.reviewers if item.status == "UNKNOWN"] 

381 status = ResultStatus.BLOCKED if unknown and enforce else ResultStatus.PASSED 

382 results.record_validation( 

383 execution, 

384 validator_id=AGENT_REVIEW_ID, 

385 digest=change_set.digest, 

386 status=status, 

387 kind=ValidationKind.SUCCESS 

388 if status is ResultStatus.PASSED 

389 else ValidationKind.PROTOCOL_ERROR, 

390 mandatory=enforce, 

391 summary=( 

392 f"Review panel by {provider}: {report.verdict}, {len(findings)} finding(s)" 

393 + (f"; no valid answer from {', '.join(unknown)}" if unknown else "") 

394 ), 

395 findings=tuple(findings), 

396 evidence_refs=(report_ref,), 

397 ) 

398 results.set_flag_json( 

399 f"agentreview:{execution.execution_id}", {"digest": change_set.digest} 

400 ) 

401 if not enforce: 401 ↛ 402line 401 didn't jump to line 402 because the condition on line 401 was never true

402 return ReviewOutcome(True) 

403 if unknown: 

404 # A reviewer that never answered blocks; nothing goes back to the agent. 

405 return ReviewOutcome(True) 

406 blocking = [ 

407 (item, by_key.get(item.finding_id)) 

408 for item in findings 

409 if item.severity in results.agent_review.blocking_severities() 

410 ] 

411 if not blocking: 

412 return ReviewOutcome(True) 

413 return ReviewOutcome(True, self._auto_fix(execution, change_set, blocking)) 

414 

415 def _record_findings( 

416 self, execution: Execution, report: PanelReport, report_ref: str, enforce: bool 

417 ) -> tuple[list[Finding], dict[str, ReviewFinding]]: 

418 error = FindingSeverity.HIGH if enforce else FindingSeverity.MEDIUM 

419 recorded = self._record_panel_problems(execution, report, report_ref, error) 

420 by_key: dict[str, ReviewFinding] = {} 

421 for finding in report.findings: 

422 entry = self._record_review_finding(execution, report, finding, report_ref, error) 

423 by_key[entry.finding_id] = finding 

424 recorded.append(entry) 

425 return recorded, by_key 

426 

427 def _record_panel_problems( 

428 self, execution: Execution, report: PanelReport, report_ref: str, error: FindingSeverity 

429 ) -> list[Finding]: 

430 """A finding per failed consistency check and per reviewer without a valid answer.""" 

431 results = self.results 

432 recorded = [ 

433 results.record_finding( 

434 execution, 

435 validator_id=AGENT_REVIEW_ID, 

436 rule_id=f"{PANEL_RULE_PREFIX}.consistency.{item['id']}", 

437 category="agent-review", 

438 severity=error, 

439 message=( 

440 f"The consistency check {item['id']} failed ({item.get('summary', '')}); " 

441 "no reviewer was called" 

442 ), 

443 evidence_refs=(report_ref,), 

444 ) 

445 for item in report.consistency 

446 if item.get("status") != "PASSED" 

447 ] 

448 recorded.extend( 

449 results.record_finding( 

450 execution, 

451 validator_id=AGENT_REVIEW_ID, 

452 rule_id=f"{PANEL_RULE_PREFIX}.unknown", 

453 category="agent-protocol", 

454 severity=error, 

455 message=f"The {outcome.reviewer} reviewer gave no valid answer: {outcome.reason}", 

456 evidence_refs=(report_ref, *outcome.evidence_refs), 

457 ) 

458 for outcome in report.reviewers 

459 if outcome.status == "UNKNOWN" 

460 ) 

461 return recorded 

462 

463 def _record_review_finding( 

464 self, 

465 execution: Execution, 

466 report: PanelReport, 

467 finding: ReviewFinding, 

468 report_ref: str, 

469 error: FindingSeverity, 

470 ) -> Finding: 

471 """One finding of the report, attributed to the agent or the tool that made it.""" 

472 if finding.source == "ai": 472 ↛ 477line 472 didn't jump to line 477 because the condition on line 472 was always true

473 actor = Actor( 

474 actor_type=ActorType.AGENT, actor_id=f"agent.{report.provider}", version="1" 

475 ) 

476 else: 

477 actor = Actor( 

478 actor_type=ActorType.TOOL, 

479 actor_id=f"validator.review.{finding.reviewer}", 

480 version="1", 

481 ) 

482 side = "" if finding.side == "new" else " (a removed line)" 

483 return self.results.record_finding( 

484 execution, 

485 validator_id=AGENT_REVIEW_ID, 

486 rule_id=f"{PANEL_RULE_PREFIX}.{finding.rule}", 

487 category="agent-review", 

488 severity=error if finding.blocking else FindingSeverity.LOW, 

489 message=f"[{finding.reviewer}] {finding.issue}{side}", 

490 path=finding.file, 

491 line=finding.line, 

492 evidence_refs=(report_ref,), 

493 recommendation=(f"Evidence: {finding.evidence}" if finding.evidence else None), 

494 introduced=True, 

495 actor=actor, 

496 ) 

497 

498 # ----- scoped auto-fix --------------------------------------------------------------------- 

499 def agent_files(self, execution: Execution, change_set: ChangeSet) -> set[str] | None: 

500 """The ChangeSet files only the agent wrote in this run (provenance), or ``None`` when 

501 provenance is not recorded.""" 

502 engine = self.results.engine 

503 if not engine.provenance.snapshots_enabled: 503 ↛ 504line 503 didn't jump to line 504 because the condition on line 503 was never true

504 return None 

505 engine.provenance.attribute(execution, change_set, PhaseId.INDEPENDENT_REVIEW) 

506 records = [ 

507 item 

508 for item in self.results.s.state.list( 

509 "component_provenance", ComponentProvenance, execution_id=execution.execution_id 

510 ) 

511 if item.change_set_digest == change_set.digest 

512 ] 

513 if not records: 513 ↛ 514line 513 didn't jump to line 514 because the condition on line 513 was never true

514 return None 

515 return {item.path for item in records[-1].files if item.source == "AGENT"} 

516 

517 def _attempts(self, execution: Execution) -> int: 

518 return sum( 

519 1 

520 for event in self.results.s.events.list(execution.execution_id) 

521 if event.event_type == "review.autofix.requested" 

522 ) 

523 

524 def _auto_fix( 

525 self, 

526 execution: Execution, 

527 change_set: ChangeSet, 

528 blocking: list[tuple[Finding, ReviewFinding | None]], 

529 ) -> tuple[Finding, ...]: 

530 settings = self.enabled_settings() 

531 config = settings.auto_fix 

532 if config is None: 532 ↛ 534line 532 didn't jump to line 534 because the condition on line 532 was never true

533 # Without autoFix the panel behaves as the single reviewer: every error goes back. 

534 return tuple(item for item, _ in blocking) 

535 if config.mode != "scoped": 535 ↛ 536line 535 didn't jump to line 536 because the condition on line 535 was never true

536 return () 

537 attempts = self._attempts(execution) 

538 agent_files = self.agent_files(execution, change_set) or set() 

539 fixable = [ 

540 finding 

541 for finding, source in blocking 

542 if source is not None and source.side == "new" and source.file in agent_files 

543 ] 

544 kept = [finding for finding, _ in blocking if finding not in fixable] 

545 key = f"{AUTOFIX_FLAG}:{execution.execution_id}" 

546 if not fixable or attempts >= config.attempts: 

547 self.results.s.events.append( 

548 execution.execution_id, 

549 "review.autofix.declined", 

550 { 

551 "reason": "no error on a line the agent wrote" 

552 if not fixable 

553 else f"the {config.attempts} attempt(s) are used", 

554 "attempts": attempts, 

555 "notFixable": [item.finding_id for item in kept], 

556 }, 

557 ) 

558 self.results.set_flag_json(key, None) 

559 return () 

560 self.results.set_flag_json( 

561 key, 

562 { 

563 "digest": change_set.digest, 

564 "fixable": [item.finding_id for item in fixable], 

565 "notFixable": [item.finding_id for item in kept], 

566 }, 

567 ) 

568 self.results.s.events.append( 

569 execution.execution_id, 

570 "review.autofix.requested", 

571 { 

572 "attempt": attempts + 1, 

573 "maxAttempts": config.attempts, 

574 "fixable": [item.finding_id for item in fixable], 

575 "notFixable": [item.finding_id for item in kept], 

576 "changeSetDigest": change_set.digest, 

577 }, 

578 ) 

579 return tuple(fixable) 

580 

581 def scoped_findings(self, execution: Execution, findings: list[Finding]) -> list[Finding]: 

582 """The findings a correction carries back under scoped auto-fix: only the fixable ones 

583 of the current digest.""" 

584 value = self.results.flag_json(f"{AUTOFIX_FLAG}:{execution.execution_id}") 

585 if not isinstance(value, dict) or value.get("digest") != execution.change_set_digest: 585 ↛ 586line 585 didn't jump to line 586 because the condition on line 585 was never true

586 return findings 

587 wanted = set(value.get("fixable") or []) 

588 return [item for item in findings if item.finding_id in wanted] 

589 

590 

591__all__ = ["AUTOFIX_FLAG", "PanelReview", "compact_task"]