Coverage for src/governed_harness/orchestration/review_panel.py: 90%
193 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""The review panel in INDEPENDENT_REVIEW (#57), replacing the single reviewer of #38.
3Under ``review.panel`` the second review of a run is the panel of ``governed_harness.review``:
4the same cost rule as before (no reviewer while a blocking deterministic validator fails, and
5again only when the ChangeSet digest changed), then the consistency checks, the deterministic
6rules and the reviewers by domain over their slices, in parallel and read-only. The rules a
7linter verifies are not run again here: the validators of VERIFICATION already ran them.
9Every request, answer, sandbox and the report are evidence of the run; the findings are
10recorded under ``review.agent`` so the gate, the brief and the corrections treat them as the
11single reviewer's. Errors are ``HIGH`` under ``enforce`` (they block the gate) and ``MEDIUM``
12under ``warn``; suggestions are ``LOW``; a reviewer that never answered (``UNKNOWN``) blocks.
14Scoped auto-fix (``review.panel.autoFix: scoped``): only errors on lines the agent wrote in this
15run go back to IMPLEMENTATION: an added line of a file whose provenance is the agent's
16(``provenance.agentSnapshots``), never a line of a file a person also edited, never a removed
17line and never a suggestion; at most ``maxAttempts`` times, each one recorded. Other errors stay
18for the person who decides."""
20from __future__ import annotations
22import threading
23from collections.abc import Callable, Sequence
24from typing import TYPE_CHECKING, Any
26from governed_harness import __version__
27from governed_harness.agents import AgentCallResult
28from governed_harness.agents.routing import RoutingDecision
29from governed_harness.capabilities import grants_from_rules
30from governed_harness.configuration.review import ReviewPanelConfig
31from governed_harness.domain.enums import (
32 ActorType,
33 FindingSeverity,
34 PhaseId,
35 ResultStatus,
36 ValidationKind,
37)
38from governed_harness.domain.models import (
39 Actor,
40 ChangeSet,
41 ComponentProvenance,
42 Execution,
43 Finding,
44 PhaseExecution,
45)
46from governed_harness.orchestration.agent_review import AGENT_REVIEW_ID
47from governed_harness.orchestration.engine_types import READ_ONLY_RULE, ReviewOutcome
48from governed_harness.review.cache import ReviewCache
49from governed_harness.review.contract import ReviewFinding
50from governed_harness.review.invoke import BuiltProvider, InvokerHooks, ProviderInvoker
51from governed_harness.review.panel import PanelInputs, PanelReport, ReviewerCall, run_panel
52from governed_harness.review.project import (
53 consistency_runner,
54 panel_context,
55 review_setup,
56 reviewer_route,
57)
58from governed_harness.review.providers import build_reviewer_provider, mcp_digest, mcp_servers
59from governed_harness.review.reviewers import Reviewer
60from governed_harness.runtime.workspace import WorkspaceDiff
62if TYPE_CHECKING:
63 from governed_harness.orchestration.hosts import ResultsHost
65AUTOFIX_FLAG = "reviewfix"
66PANEL_RULE_PREFIX = "review.panel"
69class _SerializedObserver:
70 """The process ledger of a run, written by one thread at a time (reviewers run in
71 parallel)."""
73 def __init__(self, inner: Any) -> None:
74 self.inner = inner
75 self.lock = threading.Lock()
77 def started(self, pid: int, argv: tuple[str, ...]) -> None:
78 with self.lock:
79 self.inner.started(pid, argv)
81 def finished(self, pid: int) -> None:
82 with self.lock:
83 self.inner.finished(pid)
86def compact_task(task: Any) -> dict[str, Any]:
87 """What a reviewer needs of the task: the intent, the requirement and criterion texts and
88 the constraints (not the whole task document)."""
89 return {
90 "intent": task.intent,
91 "requirements": [item.text for item in task.requirements],
92 "acceptanceCriteria": [item.text for item in task.acceptance_criteria],
93 "constraints": list(task.constraints),
94 }
97class PanelReview:
98 def __init__(self, results: ResultsHost) -> None:
99 self.results = results
101 @property
102 def settings(self) -> ReviewPanelConfig | None:
103 review = self.results.project.review
104 return review.panel if review is not None else None
106 def enabled_settings(self) -> ReviewPanelConfig:
107 """The panel's settings, for the steps that run only when it is configured."""
108 settings = self.settings
109 if settings is None: # not reached: the review runs the panel only when configured 109 ↛ 110line 109 didn't jump to line 110 because the condition on line 109 was never true
110 raise RuntimeError("review.panel is not configured")
111 return settings
113 @property
114 def configured(self) -> bool:
115 settings = self.settings
116 return settings is not None and settings.enabled
118 # ----- routing of the reviewers (#85) -----------------------------------------------------
119 def _route_observer(
120 self, decisions: dict[tuple[str, str], RoutingDecision]
121 ) -> Callable[[Reviewer, str, RoutingDecision], None]:
122 """Keeps the routing decision of each reviewer on each provider, recorded with its call
123 as for any other call: under ``agentRouting``, or for a reviewer's own model."""
124 routed = self.results.project.agent_routing is not None
126 def heard(reviewer: Reviewer, provider_id: str, decision: RoutingDecision) -> None:
127 if routed or decision.rule.startswith("reviewer:"):
128 decisions[(reviewer.reviewer_id, provider_id)] = decision
130 return heard
132 def _record_routing(
133 self,
134 execution: Execution,
135 phase: PhaseExecution,
136 call: ReviewerCall,
137 decision: RoutingDecision,
138 ) -> None:
139 """``agent.routing.decided`` of one reviewer call, as every other agent call records it,
140 with the reviewer and the attempt."""
141 results = self.results
142 record = {
143 **decision.as_dict(),
144 "provider": call.provider,
145 "phase": phase.phase_id,
146 "reviewer": call.reviewer,
147 "attempt": call.attempt,
148 }
149 ref = results.record_json(
150 execution,
151 phase.phase_id,
152 record,
153 kind="agent-routing",
154 summary=(
155 f"Routing of the {call.reviewer} reviewer: {decision.rule} -> "
156 f"{decision.model or 'provider default'}/{decision.effort or 'default'}"
157 ),
158 )
159 results.s.events.append(
160 execution.execution_id,
161 "agent.routing.decided",
162 {**record, "evidenceRef": ref},
163 phase_execution_id=phase.phase_execution_id,
164 )
166 # ----- the invoker -----------------------------------------------------------------------
167 def _invoker(
168 self,
169 execution: Execution,
170 phase: PhaseExecution,
171 servers: dict[str, Any],
172 decisions: dict[tuple[str, str], RoutingDecision],
173 ) -> ProviderInvoker:
174 results = self.results
175 engine = results.engine
176 resolved = results.s.resolved
177 workspace = results.s.paths.workspace
178 runtime = resolved.project.runtime
179 runner = engine._runner(execution)
180 if runner.observer is not None: 180 ↛ 183line 180 didn't jump to line 183 because the condition on line 180 was always true
181 runner.observer = _SerializedObserver(runner.observer)
183 def build(provider_id: str) -> BuiltProvider | str:
184 built = build_reviewer_provider(
185 resolved,
186 provider_id,
187 workspace=workspace,
188 protected=engine._protected_paths(),
189 host=engine.sandbox_host,
190 allow_network=engine._agent_network_allowed(),
191 )
192 if isinstance(built, str): 192 ↛ 193line 192 didn't jump to line 193 because the condition on line 192 was never true
193 return built
194 refs: tuple[str, ...] = ()
195 if built.sandbox is not None: 195 ↛ 196line 195 didn't jump to line 196 because the condition on line 195 was never true
196 refs = (
197 results.record_json(
198 execution,
199 phase.phase_id,
200 {**built.sandbox.evidence(), "readOnlyWorkspace": True},
201 kind="review-sandbox",
202 summary=f"Read-only sandbox of the reviewers on {provider_id}",
203 ),
204 )
205 return BuiltProvider(built.built.provider, built.built.actor, refs)
207 def before(_calls: Sequence[ReviewerCall]) -> str | None:
208 blocked = results.before_agent_call(execution, phase, "review")
209 return blocked.summary if blocked is not None else None
211 def record_request(call: ReviewerCall) -> str | None:
212 decision = decisions.get((call.reviewer, call.provider))
213 if decision is not None:
214 self._record_routing(execution, phase, call, decision)
215 return results.record_json(
216 execution,
217 phase.phase_id,
218 call.request,
219 kind="agent-review-request",
220 summary=f"Review request of the {call.reviewer} reviewer to {call.provider}",
221 )
223 def after(_call: ReviewerCall, result: AgentCallResult) -> None:
224 engine._save_agent_result(execution, phase, result.execution)
225 results.after_agent_call(execution, phase, result.execution)
227 def on_violation(diff: WorkspaceDiff, restored: int, unrestorable: int) -> None:
228 shown = ", ".join(item.path for item in diff.changes[:10])
229 results.record_finding(
230 execution,
231 validator_id="harness.agent-calls",
232 rule_id=READ_ONLY_RULE,
233 category="agent-protocol",
234 severity=FindingSeverity.HIGH,
235 message=(
236 f"A reviewer of the review panel changed {len(diff.changes)} path(s): "
237 f"{shown}; restored {restored}, could not restore {unrestorable}"
238 ),
239 path=diff.changes[0].path,
240 recommendation="Use a provider that honours readOnly requests.",
241 )
243 judge, timeout = self._contract_judge(execution, phase)
244 return ProviderInvoker(
245 workspace=workspace,
246 artifacts=results.s.artifacts,
247 runner=runner,
248 provenance=engine._provenance(execution),
249 build=build,
250 grants_for=lambda actor: grants_from_rules(
251 execution.execution_id, actor, resolved.effective_capabilities
252 ),
253 execution_id=execution.execution_id,
254 phase_id=PhaseId.INDEPENDENT_REVIEW,
255 default_timeout=timeout(engine._bounded_timeout(runtime.command_timeout_seconds)),
256 max_output_bytes=runtime.max_output_bytes,
257 hooks=InvokerHooks(
258 cancelled=lambda: engine.is_cancelled(execution.execution_id),
259 before=before,
260 record_request=record_request,
261 after=after,
262 on_violation=on_violation,
263 judge=judge,
264 ),
265 mcp_servers=servers,
266 )
268 def _contract_judge(
269 self, execution: Execution, phase: PhaseExecution
270 ) -> tuple[
271 Callable[[ReviewerCall, AgentCallResult, str | None], str | None] | None,
272 Callable[[int], int],
273 ]:
274 """Under ``agentContract: enforce`` (#149) every reviewer answer is a ``review`` call of
275 the contract: judged against the token budget, the time limit (which also shortens the
276 reviewers' timeout) and the result limits, and recorded; ``(None, unchanged)`` without
277 it, so the panel behaves as before."""
278 flow = self.results.agent_contract
279 if not flow.enabled:
280 return None, lambda seconds: seconds
281 limits = flow.limits(execution, self.results.engine.run_task(execution), "review")
283 def judge(
284 call: ReviewerCall, result: AgentCallResult, request_ref: str | None
285 ) -> str | None:
286 return flow.settle_reviewer(
287 execution,
288 phase,
289 call.reviewer,
290 result,
291 request=call.request,
292 request_ref=request_ref,
293 limits=limits,
294 )
296 return judge, lambda seconds: flow.timeout(seconds, limits)
298 # ----- the review --------------------------------------------------------------------------
299 def run(
300 self, execution: Execution, phase: PhaseExecution, change_set: ChangeSet
301 ) -> ReviewOutcome:
302 settings = self.enabled_settings()
303 results = self.results
304 engine = results.engine
305 resolved = results.s.resolved
306 workspace = results.s.paths.workspace
307 setup = review_setup(resolved, workspace)
308 servers = mcp_servers(workspace, settings.mcp_servers)
309 review = resolved.project.review
310 reviewer_config = review.reviewer if review is not None else None
311 provider = (
312 settings.provider
313 or (reviewer_config.provider if reviewer_config else None)
314 or results.provider_for(execution, "review")
315 )
316 cache_settings = settings.effective_cache
317 diff = engine._compute_owned_diff(execution).unified_diff.decode("utf-8", "replace")
318 untrusted, notice = panel_context(resolved, workspace)
319 decisions: dict[tuple[str, str], RoutingDecision] = {}
320 inputs = PanelInputs(
321 workspace=workspace,
322 diff_text=diff,
323 mode="run",
324 catalog=setup.catalog,
325 reviewers=setup.reviewers,
326 settings=settings,
327 provider=provider,
328 invoker=self._invoker(execution, phase, servers, decisions),
329 route=reviewer_route(resolved, self._route_observer(decisions)),
330 runner_version=__version__,
331 base=results.s.state.get_flag(f"baseline:{execution.execution_id}") or None,
332 head=change_set.digest,
333 fallback=settings.fallback_provider,
334 packs=setup.packs or None,
335 cache=ReviewCache(
336 results.s.paths.harness_dir / "review" / "cache",
337 ttl_seconds=cache_settings.ttl_seconds,
338 max_entries=cache_settings.limit,
339 )
340 if cache_settings.on
341 else None,
342 consistency=consistency_runner(
343 resolved,
344 workspace,
345 engine._runner(execution),
346 execution_id=execution.execution_id,
347 cancelled=lambda: engine.is_cancelled(execution.execution_id),
348 ),
349 extra={"task": compact_task(engine.run_task(execution)), **untrusted},
350 untrusted_notice=notice,
351 mcp={"mcpServers": servers, "digest": mcp_digest(servers)},
352 )
353 report = run_panel(inputs)
354 enforce = settings.mode != "warn"
355 report_ref = results.record_json(
356 execution,
357 phase.phase_id,
358 report.as_dict(),
359 kind="review-panel-report",
360 summary=(
361 f"Review panel: {report.verdict}, {len(report.findings)} finding(s), "
362 f"{report.tokens} reported token(s), cache {report.cache}"
363 ),
364 )
365 results.s.events.append(
366 execution.execution_id,
367 "review.panel.completed",
368 {
369 "verdict": report.verdict,
370 "digest": report.digest,
371 "findings": len(report.findings),
372 "reviewers": {item.reviewer: item.status for item in report.reviewers},
373 "tokens": report.tokens,
374 "cache": report.cache,
375 "evidenceRef": report_ref,
376 },
377 phase_execution_id=phase.phase_execution_id,
378 )
379 findings, by_key = self._record_findings(execution, report, report_ref, enforce)
380 unknown = [item.reviewer for item in report.reviewers if item.status == "UNKNOWN"]
381 status = ResultStatus.BLOCKED if unknown and enforce else ResultStatus.PASSED
382 results.record_validation(
383 execution,
384 validator_id=AGENT_REVIEW_ID,
385 digest=change_set.digest,
386 status=status,
387 kind=ValidationKind.SUCCESS
388 if status is ResultStatus.PASSED
389 else ValidationKind.PROTOCOL_ERROR,
390 mandatory=enforce,
391 summary=(
392 f"Review panel by {provider}: {report.verdict}, {len(findings)} finding(s)"
393 + (f"; no valid answer from {', '.join(unknown)}" if unknown else "")
394 ),
395 findings=tuple(findings),
396 evidence_refs=(report_ref,),
397 )
398 results.set_flag_json(
399 f"agentreview:{execution.execution_id}", {"digest": change_set.digest}
400 )
401 if not enforce: 401 ↛ 402line 401 didn't jump to line 402 because the condition on line 401 was never true
402 return ReviewOutcome(True)
403 if unknown:
404 # A reviewer that never answered blocks; nothing goes back to the agent.
405 return ReviewOutcome(True)
406 blocking = [
407 (item, by_key.get(item.finding_id))
408 for item in findings
409 if item.severity in results.agent_review.blocking_severities()
410 ]
411 if not blocking:
412 return ReviewOutcome(True)
413 return ReviewOutcome(True, self._auto_fix(execution, change_set, blocking))
415 def _record_findings(
416 self, execution: Execution, report: PanelReport, report_ref: str, enforce: bool
417 ) -> tuple[list[Finding], dict[str, ReviewFinding]]:
418 error = FindingSeverity.HIGH if enforce else FindingSeverity.MEDIUM
419 recorded = self._record_panel_problems(execution, report, report_ref, error)
420 by_key: dict[str, ReviewFinding] = {}
421 for finding in report.findings:
422 entry = self._record_review_finding(execution, report, finding, report_ref, error)
423 by_key[entry.finding_id] = finding
424 recorded.append(entry)
425 return recorded, by_key
427 def _record_panel_problems(
428 self, execution: Execution, report: PanelReport, report_ref: str, error: FindingSeverity
429 ) -> list[Finding]:
430 """A finding per failed consistency check and per reviewer without a valid answer."""
431 results = self.results
432 recorded = [
433 results.record_finding(
434 execution,
435 validator_id=AGENT_REVIEW_ID,
436 rule_id=f"{PANEL_RULE_PREFIX}.consistency.{item['id']}",
437 category="agent-review",
438 severity=error,
439 message=(
440 f"The consistency check {item['id']} failed ({item.get('summary', '')}); "
441 "no reviewer was called"
442 ),
443 evidence_refs=(report_ref,),
444 )
445 for item in report.consistency
446 if item.get("status") != "PASSED"
447 ]
448 recorded.extend(
449 results.record_finding(
450 execution,
451 validator_id=AGENT_REVIEW_ID,
452 rule_id=f"{PANEL_RULE_PREFIX}.unknown",
453 category="agent-protocol",
454 severity=error,
455 message=f"The {outcome.reviewer} reviewer gave no valid answer: {outcome.reason}",
456 evidence_refs=(report_ref, *outcome.evidence_refs),
457 )
458 for outcome in report.reviewers
459 if outcome.status == "UNKNOWN"
460 )
461 return recorded
463 def _record_review_finding(
464 self,
465 execution: Execution,
466 report: PanelReport,
467 finding: ReviewFinding,
468 report_ref: str,
469 error: FindingSeverity,
470 ) -> Finding:
471 """One finding of the report, attributed to the agent or the tool that made it."""
472 if finding.source == "ai": 472 ↛ 477line 472 didn't jump to line 477 because the condition on line 472 was always true
473 actor = Actor(
474 actor_type=ActorType.AGENT, actor_id=f"agent.{report.provider}", version="1"
475 )
476 else:
477 actor = Actor(
478 actor_type=ActorType.TOOL,
479 actor_id=f"validator.review.{finding.reviewer}",
480 version="1",
481 )
482 side = "" if finding.side == "new" else " (a removed line)"
483 return self.results.record_finding(
484 execution,
485 validator_id=AGENT_REVIEW_ID,
486 rule_id=f"{PANEL_RULE_PREFIX}.{finding.rule}",
487 category="agent-review",
488 severity=error if finding.blocking else FindingSeverity.LOW,
489 message=f"[{finding.reviewer}] {finding.issue}{side}",
490 path=finding.file,
491 line=finding.line,
492 evidence_refs=(report_ref,),
493 recommendation=(f"Evidence: {finding.evidence}" if finding.evidence else None),
494 introduced=True,
495 actor=actor,
496 )
498 # ----- scoped auto-fix ---------------------------------------------------------------------
499 def agent_files(self, execution: Execution, change_set: ChangeSet) -> set[str] | None:
500 """The ChangeSet files only the agent wrote in this run (provenance), or ``None`` when
501 provenance is not recorded."""
502 engine = self.results.engine
503 if not engine.provenance.snapshots_enabled: 503 ↛ 504line 503 didn't jump to line 504 because the condition on line 503 was never true
504 return None
505 engine.provenance.attribute(execution, change_set, PhaseId.INDEPENDENT_REVIEW)
506 records = [
507 item
508 for item in self.results.s.state.list(
509 "component_provenance", ComponentProvenance, execution_id=execution.execution_id
510 )
511 if item.change_set_digest == change_set.digest
512 ]
513 if not records: 513 ↛ 514line 513 didn't jump to line 514 because the condition on line 513 was never true
514 return None
515 return {item.path for item in records[-1].files if item.source == "AGENT"}
517 def _attempts(self, execution: Execution) -> int:
518 return sum(
519 1
520 for event in self.results.s.events.list(execution.execution_id)
521 if event.event_type == "review.autofix.requested"
522 )
524 def _auto_fix(
525 self,
526 execution: Execution,
527 change_set: ChangeSet,
528 blocking: list[tuple[Finding, ReviewFinding | None]],
529 ) -> tuple[Finding, ...]:
530 settings = self.enabled_settings()
531 config = settings.auto_fix
532 if config is None: 532 ↛ 534line 532 didn't jump to line 534 because the condition on line 532 was never true
533 # Without autoFix the panel behaves as the single reviewer: every error goes back.
534 return tuple(item for item, _ in blocking)
535 if config.mode != "scoped": 535 ↛ 536line 535 didn't jump to line 536 because the condition on line 535 was never true
536 return ()
537 attempts = self._attempts(execution)
538 agent_files = self.agent_files(execution, change_set) or set()
539 fixable = [
540 finding
541 for finding, source in blocking
542 if source is not None and source.side == "new" and source.file in agent_files
543 ]
544 kept = [finding for finding, _ in blocking if finding not in fixable]
545 key = f"{AUTOFIX_FLAG}:{execution.execution_id}"
546 if not fixable or attempts >= config.attempts:
547 self.results.s.events.append(
548 execution.execution_id,
549 "review.autofix.declined",
550 {
551 "reason": "no error on a line the agent wrote"
552 if not fixable
553 else f"the {config.attempts} attempt(s) are used",
554 "attempts": attempts,
555 "notFixable": [item.finding_id for item in kept],
556 },
557 )
558 self.results.set_flag_json(key, None)
559 return ()
560 self.results.set_flag_json(
561 key,
562 {
563 "digest": change_set.digest,
564 "fixable": [item.finding_id for item in fixable],
565 "notFixable": [item.finding_id for item in kept],
566 },
567 )
568 self.results.s.events.append(
569 execution.execution_id,
570 "review.autofix.requested",
571 {
572 "attempt": attempts + 1,
573 "maxAttempts": config.attempts,
574 "fixable": [item.finding_id for item in fixable],
575 "notFixable": [item.finding_id for item in kept],
576 "changeSetDigest": change_set.digest,
577 },
578 )
579 return tuple(fixable)
581 def scoped_findings(self, execution: Execution, findings: list[Finding]) -> list[Finding]:
582 """The findings a correction carries back under scoped auto-fix: only the fixable ones
583 of the current digest."""
584 value = self.results.flag_json(f"{AUTOFIX_FLAG}:{execution.execution_id}")
585 if not isinstance(value, dict) or value.get("digest") != execution.change_set_digest: 585 ↛ 586line 585 didn't jump to line 586 because the condition on line 585 was never true
586 return findings
587 wanted = set(value.get("fixable") or [])
588 return [item for item in findings if item.finding_id in wanted]
591__all__ = ["AUTOFIX_FLAG", "PanelReview", "compact_task"]