Coverage for src/governed_harness/review/panel.py: 95%
411 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""The review panel (#57): reviewers by domain over diff slices, with a deterministic verdict.
3``run_panel`` is the whole review, independent of where it runs (INDEPENDENT_REVIEW of a
4governed run, or ``harness review-code`` outside one). The caller gives it the diff, the catalog,
5the reviewers and an ``Invoker`` that calls the providers; everything else is decided here, in
6this order, so that no token is spent that a cheaper step could save:
81. the global cache: the same review of the same diff in the same mode is not repeated;
92. the project's consistency checks: when one fails, no model is called;
103. the deterministic rules (``verified_by: tool:...``): the harness's own checks and, outside a
11 run, the linters the repository configures; they never reach a model;
124. the reviewers whose mode matches, whose slice changed and whose signal is present;
135. per reviewer: the per-reviewer cache, else one call with a budget proportional to the slice;
14 an invalid answer is ``UNKNOWN`` and gets one retry with the alternate provider; a persistent
15 ``UNKNOWN`` blocks;
166. the findings: outside the reportable locations dropped, outside the catalog downgraded
17 without concrete evidence, capped, ordered; an optional second opinion on the blocking ones;
187. the verdict, recomputed: ``FAIL`` with any error or failed consistency check, ``UNKNOWN``
19 with a reviewer that never answered, ``PASS_WARN`` with suggestions only, else ``PASS``.
21The report is deterministic: the same inputs and answers give the same report and digest."""
23from __future__ import annotations
25from collections.abc import Callable, Mapping, Sequence
26from dataclasses import dataclass, field, replace
27from pathlib import Path
28from typing import Any, Literal, Protocol
30from governed_harness.agents.requests import REQUEST_SCHEMA_VERSION
31from governed_harness.configuration.review import ReviewPanelConfig
32from governed_harness.evidence.hashing import sha256_bytes, sha256_json
33from governed_harness.review.cache import ReviewCache, global_key, reviewer_key
34from governed_harness.review.checks import CHECKS, run_checks
35from governed_harness.review.contract import (
36 ContractError,
37 ReviewFinding,
38 Verdict,
39 contract_text,
40 normalize,
41 verdict_of,
42)
43from governed_harness.review.diff import (
44 FileChange,
45 Locations,
46 diff_hash,
47 parse_diff,
48 render,
49 reportable_locations,
50)
51from governed_harness.review.reviewers import Reviewer, definitions_hash, drift
52from governed_harness.review.rules import HARNESS_TOOL, Catalog, Rule
53from governed_harness.review.signals import activation, slice_files
55REPORT_SCHEMA_VERSION = "1.0"
56HIT = "hit"
57"""The cache value of an answer served from a cache (no model was called)."""
58SKIPPED = "SKIPPED"
59MAX_SLICE_CHARS = 200_000
60SECOND_OPINION_ID = "second-opinion"
63# ----- what the caller provides -----------------------------------------------------------------
64@dataclass(frozen=True)
65class ReviewerCall:
66 reviewer: str
67 provider: str
68 request: dict[str, Any]
69 model: str | None
70 effort: str | None
71 timeout_seconds: int | None
72 max_tokens: int
73 attempt: int = 1
76@dataclass(frozen=True)
77class ReviewerAnswer:
78 status: Literal["ANSWERED", "ERROR"]
79 result: dict[str, Any] | None
80 summary: str
81 provider: str
82 model: str | None = None
83 """The model the call was configured with (``None``: the provider's default)."""
84 tokens: int | None = None
85 invocation_id: str | None = None
86 evidence_refs: tuple[str, ...] = ()
89class Invoker(Protocol):
90 def invoke(self, calls: Sequence[ReviewerCall], workers: int) -> list[ReviewerAnswer]:
91 """Answer every call (in parallel up to ``workers``), in the order of ``calls``."""
94Route = Callable[[Reviewer, str], tuple[str | None, str | None]]
95"""The model and effort of a reviewer on a provider."""
98@dataclass
99class PanelInputs:
100 workspace: Path
101 diff_text: str
102 mode: str
103 catalog: Catalog
104 reviewers: list[Reviewer]
105 settings: ReviewPanelConfig
106 provider: str
107 invoker: Invoker
108 route: Route
109 runner_version: str
110 base: str | None = None
111 head: str | None = None
112 fallback: str | None = None
113 forced_model: str | None = None
114 skip: tuple[str, ...] = ()
115 packs: tuple[str, ...] | None = None
116 cache: ReviewCache | None = None
117 consistency: Callable[[], list[dict[str, Any]]] | None = None
118 linters: Callable[[list[Rule], Locations], list[ReviewFinding]] | None = None
119 extra: dict[str, Any] = field(default_factory=dict)
120 mcp: dict[str, Any] = field(default_factory=dict)
121 """The MCP configuration reviewers may use (servers of the allowlist only)."""
122 untrusted_notice: str | None = None
123 """Since #5: the notice that repository content is untrusted context, not instructions."""
126# ----- the report -------------------------------------------------------------------------------
127@dataclass
128class ReviewerOutcome:
129 reviewer: str
130 domain: str
131 status: str
132 reason: str
133 activation: dict[str, Any] = field(default_factory=dict)
134 files: list[str] = field(default_factory=list)
135 changed_lines: int = 0
136 slice_hash: str | None = None
137 budget: int | None = None
138 provider: str | None = None
139 configured_model: str | None = None
140 executed_model: str | None = None
141 attempts: int = 0
142 cache: str = "off"
143 tokens: int = 0
144 over_budget: bool = False
145 dropped_outside: int = 0
146 downgraded: int = 0
147 truncated: int = 0
148 summary: str = ""
149 findings: list[ReviewFinding] = field(default_factory=list)
150 invocation_ids: list[str] = field(default_factory=list)
151 evidence_refs: list[str] = field(default_factory=list)
153 def as_dict(self) -> dict[str, Any]:
154 value: dict[str, Any] = {
155 "id": self.reviewer,
156 "domain": self.domain,
157 "status": self.status,
158 "reason": self.reason,
159 "activation": self.activation,
160 "files": self.files,
161 "changedLines": self.changed_lines,
162 "findings": len(self.findings),
163 }
164 if self.status not in {SKIPPED}:
165 value.update(
166 {
167 "sliceHash": self.slice_hash,
168 "budget": self.budget,
169 "provider": self.provider,
170 "model": {
171 "configured": self.configured_model,
172 "executed": self.executed_model,
173 },
174 "attempts": self.attempts,
175 "cache": self.cache,
176 "tokens": self.tokens,
177 "overBudget": self.over_budget,
178 "droppedOutside": self.dropped_outside,
179 "downgraded": self.downgraded,
180 "truncated": self.truncated,
181 "summary": self.summary,
182 }
183 )
184 return value
187@dataclass
188class PanelReport:
189 mode: str
190 base: str | None
191 head: str | None
192 diff_hash: str
193 provider: str
194 fallback: str | None
195 verdict: Verdict
196 blocking: bool
197 findings: list[ReviewFinding]
198 reviewers: list[ReviewerOutcome]
199 catalog: dict[str, Any]
200 consistency: list[dict[str, Any]]
201 deterministic: dict[str, Any]
202 drift: list[str]
203 locations: int
204 second_opinion: dict[str, Any] | None = None
205 cache: str = "off"
206 reviewer_cache_hits: int = 0
207 cached_from: dict[str, int] | None = None
208 """On a global cache hit: the tokens and model calls of the review the report was cached
209 from (this review called no model, #75)."""
211 @property
212 def tokens(self) -> int:
213 return sum(item.tokens for item in self.reviewers) + int(
214 (self.second_opinion or {}).get("tokens") or 0
215 )
217 @property
218 def model_calls(self) -> int:
219 calls = sum(item.attempts for item in self.reviewers if item.cache != HIT)
220 second = self.second_opinion or {}
221 return calls + int(bool(second.get("status")) and second.get("cache") != HIT)
223 def body(self) -> dict[str, Any]:
224 """The deterministic part of the report (what its digest covers)."""
225 return {
226 "schemaVersion": REPORT_SCHEMA_VERSION,
227 "mode": self.mode,
228 "base": self.base,
229 "head": self.head,
230 "diffHash": self.diff_hash,
231 "verdict": self.verdict,
232 "findings": [item.as_dict() for item in self.findings],
233 "reviewers": [{"id": item.reviewer, "status": item.status} for item in self.reviewers],
234 "consistency": [
235 {"id": item["id"], "status": item["status"]} for item in self.consistency
236 ],
237 "catalog": self.catalog.get("digest"),
238 }
240 @property
241 def digest(self) -> str:
242 return sha256_json(self.body())
244 def as_dict(self) -> dict[str, Any]:
245 errors = sum(1 for item in self.findings if item.blocking)
246 tokens: dict[str, Any] = {
247 "total": self.tokens,
248 "byReviewer": {item.reviewer: item.tokens for item in self.reviewers},
249 "modelCalls": self.model_calls,
250 }
251 if self.cached_from is not None:
252 tokens["cachedFrom"] = dict(self.cached_from)
253 return {
254 **self.body(),
255 "digest": self.digest,
256 "provider": self.provider,
257 "fallbackProvider": self.fallback,
258 "blocking": self.blocking,
259 "counts": {
260 "findings": len(self.findings),
261 "errors": errors,
262 "suggestions": len(self.findings) - errors,
263 "reportableLines": self.locations,
264 "droppedOutside": sum(item.dropped_outside for item in self.reviewers),
265 "downgraded": sum(item.downgraded for item in self.reviewers),
266 "truncated": sum(item.truncated for item in self.reviewers),
267 },
268 "reviewers": [item.as_dict() for item in self.reviewers],
269 "consistency": self.consistency,
270 "deterministic": self.deterministic,
271 "catalog": self.catalog,
272 "drift": self.drift,
273 "secondOpinion": self.second_opinion,
274 "cache": {"global": self.cache, "reviewerHits": self.reviewer_cache_hits},
275 "tokens": tokens,
276 }
278 @classmethod
279 def from_dict(cls, value: Mapping[str, Any]) -> PanelReport:
280 reviewers = [_outcome_from_dict(item) for item in value.get("reviewers") or []]
281 return cls(
282 mode=str(value["mode"]),
283 base=value.get("base"),
284 head=value.get("head"),
285 diff_hash=str(value["diffHash"]),
286 provider=str(value.get("provider") or ""),
287 fallback=value.get("fallbackProvider"),
288 verdict=value["verdict"],
289 blocking=bool(value.get("blocking")),
290 findings=[ReviewFinding.from_dict(item) for item in value.get("findings") or []],
291 reviewers=reviewers,
292 catalog=dict(value.get("catalog") or {}),
293 consistency=list(value.get("consistency") or []),
294 deterministic=dict(value.get("deterministic") or {}),
295 drift=list(value.get("drift") or []),
296 locations=int((value.get("counts") or {}).get("reportableLines") or 0),
297 second_opinion=value.get("secondOpinion"),
298 )
301def _outcome_from_dict(item: Mapping[str, Any]) -> ReviewerOutcome:
302 """A reviewer outcome of a cached report (its findings are in the report's list)."""
303 model = item.get("model") or {}
304 return ReviewerOutcome(
305 reviewer=str(item["id"]),
306 domain=str(item.get("domain") or ""),
307 status=str(item["status"]),
308 reason=str(item.get("reason") or ""),
309 activation=dict(item.get("activation") or {}),
310 files=list(item.get("files") or []),
311 changed_lines=int(item.get("changedLines") or 0),
312 slice_hash=item.get("sliceHash"),
313 budget=item.get("budget"),
314 provider=item.get("provider"),
315 configured_model=model.get("configured"),
316 executed_model=model.get("executed"),
317 attempts=int(item.get("attempts") or 0),
318 cache=str(item.get("cache") or "off"),
319 tokens=0,
320 summary=str(item.get("summary") or ""),
321 )
324# ----- helpers ----------------------------------------------------------------------------------
325def options_of(settings: ReviewPanelConfig) -> dict[str, Any]:
326 """The settings that change a review's result (part of both cache keys)."""
327 second = settings.second_opinion
328 return {
329 "maxFindings": settings.finding_limit,
330 "budget": settings.effective_budget.model_dump(mode="json", by_alias=True),
331 "runTools": bool(settings.run_tools),
332 "secondOpinion": second.model_dump(mode="json", by_alias=True) if second else None,
333 "consistency": [
334 item.model_dump(mode="json", by_alias=True)
335 for item in settings.consistency_checks or ()
336 ],
337 "policy": settings.mode or "enforce",
338 }
341def _rules_by_id(rules: Sequence[Rule]) -> dict[str, Rule]:
342 return {rule.rule_id: rule for rule in rules}
345def deterministic_findings(
346 catalog: Catalog, files: list[FileChange], locations: Locations
347) -> tuple[list[ReviewFinding], dict[str, Any]]:
348 """The findings of the rules verified by the harness's own checks."""
349 by_check: dict[str, list[Rule]] = {}
350 unknown: list[str] = []
351 for rule in catalog.tool_rules():
352 for name in (name for tool, name in rule.tools if tool == HARNESS_TOOL):
353 if name in CHECKS: 353 ↛ 356line 353 didn't jump to line 356 because the condition on line 353 was always true
354 by_check.setdefault(name, []).append(rule)
355 else:
356 unknown.append(f"{rule.rule_id}: tool:{HARNESS_TOOL}:{name}")
357 findings = [
358 ReviewFinding(
359 reviewer=HARNESS_TOOL,
360 file=hit.path,
361 side=hit.side,
362 line=hit.line,
363 rule=rule.rule_id,
364 severity="error" if rule.blocking else "suggestion",
365 issue=hit.message,
366 evidence=hit.evidence,
367 priority=rule.priority,
368 source=f"tool:{HARNESS_TOOL}:{hit.check}",
369 )
370 for hit in run_checks(by_check, files)
371 for rule in by_check.get(hit.check, [])
372 if rule.applies(hit.path) and locations.allows(hit.path, hit.side, hit.line)
373 ]
374 return findings, {
375 "checks": sorted(by_check),
376 "unknownChecks": sorted(unknown),
377 "rules": sorted(rule.rule_id for rule in catalog.tool_rules()),
378 }
381def _dedupe(findings: list[ReviewFinding]) -> list[ReviewFinding]:
382 seen: set[tuple[str, str, int, str]] = set()
383 kept: list[ReviewFinding] = []
384 for item in sorted(findings, key=ReviewFinding.sort_key):
385 if item.key in seen: 385 ↛ 386line 385 didn't jump to line 386 because the condition on line 385 was never true
386 continue
387 seen.add(item.key)
388 kept.append(item)
389 return kept
392@dataclass
393class _Planned:
394 reviewer: Reviewer
395 outcome: ReviewerOutcome
396 files: list[FileChange]
397 locations: Locations
398 request: dict[str, Any]
399 key: str | None
400 provider: str
401 model: str | None
402 effort: str | None
403 rules: dict[str, Rule]
406def _request(
407 inputs: PanelInputs,
408 reviewer: Reviewer,
409 files: list[FileChange],
410 locations: Locations,
411 budget: int,
412 model: str | None,
413 effort: str | None,
414 limit: int,
415) -> dict[str, Any]:
416 text = render(files)
417 truncated = len(text) > MAX_SLICE_CHARS
418 notice = f"\n\n{inputs.untrusted_notice}" if inputs.untrusted_notice else ""
419 request: dict[str, Any] = {
420 "schemaVersion": REQUEST_SCHEMA_VERSION,
421 "kind": "review",
422 "readOnly": True,
423 "instructions": reviewer.body.rstrip("\n") + notice + "\n\n" + contract_text(limit),
424 "workspace": str(inputs.workspace),
425 "reviewer": {
426 "id": reviewer.reviewer_id,
427 "domain": reviewer.domain,
428 },
429 "outputContract": {"maxFindings": limit},
430 "slice": {
431 "files": [item.path for item in files],
432 "changedLines": sum(item.changed_lines for item in files),
433 "diff": text[:MAX_SLICE_CHARS],
434 "truncated": truncated,
435 },
436 "reportableLocations": locations.as_ranges(),
437 "budget": {"maxTokens": budget},
438 "isolation": {
439 "readOnly": True,
440 "tools": list(reviewer.spec.tools),
441 "mcpServers": sorted(
442 set(reviewer.spec.mcp_servers) & set(inputs.mcp.get("mcpServers") or {})
443 ),
444 },
445 }
446 if model or effort: 446 ↛ 448line 446 didn't jump to line 448 because the condition on line 446 was always true
447 request["routing"] = {"model": model, "effort": effort}
448 request.update(inputs.extra)
449 return request
452def _message_hash(request: Mapping[str, Any]) -> str:
453 return sha256_json(
454 {key: value for key, value in request.items() if key not in {"instructions", "workspace"}}
455 )
458def _plan(inputs: PanelInputs, files: list[FileChange], limit: int) -> list[_Planned]:
459 settings = inputs.settings
460 planned: list[_Planned] = []
461 for reviewer in sorted(inputs.reviewers, key=lambda item: item.reviewer_id):
462 outcome = ReviewerOutcome(reviewer.reviewer_id, reviewer.domain, SKIPPED, "")
463 if reviewer.reviewer_id in inputs.skip: 463 ↛ 464line 463 didn't jump to line 464 because the condition on line 463 was never true
464 outcome.reason = "skipped by --skip"
465 planned.append(
466 _Planned(reviewer, outcome, [], Locations({}), {}, None, "", None, None, {})
467 )
468 continue
469 if inputs.mode not in reviewer.spec.modes: 469 ↛ 470line 469 didn't jump to line 470 because the condition on line 469 was never true
470 outcome.reason = f"does not run in {inputs.mode} mode"
471 planned.append(
472 _Planned(reviewer, outcome, [], Locations({}), {}, None, "", None, None, {})
473 )
474 continue
475 selected = slice_files(files, reviewer.spec.diff_slice)
476 active = activation(reviewer, selected, inputs.packs)
477 outcome.activation = active.as_dict()
478 outcome.files = [item.path for item in selected]
479 outcome.changed_lines = sum(item.changed_lines for item in selected)
480 rules = inputs.catalog.for_domain(reviewer.domain, ai_only=True)
481 if not active.active:
482 outcome.reason = active.reason
483 planned.append(
484 _Planned(reviewer, outcome, [], Locations({}), {}, None, "", None, None, {})
485 )
486 continue
487 provider = reviewer.spec.provider or inputs.provider
488 model, effort = inputs.route(reviewer, provider)
489 if inputs.forced_model: 489 ↛ 490line 489 didn't jump to line 490 because the condition on line 489 was never true
490 model = inputs.forced_model
491 budget = settings.effective_budget.for_lines(
492 outcome.changed_lines, reviewer.spec.max_budget
493 )
494 locations = reportable_locations(selected)
495 own_limit = min(limit, reviewer.spec.max_findings or limit)
496 request = _request(inputs, reviewer, selected, locations, budget, model, effort, own_limit)
497 outcome.status = "PENDING"
498 outcome.reason = active.reason
499 outcome.slice_hash = diff_hash(request["slice"]["diff"])
500 outcome.budget = budget
501 outcome.provider = provider
502 outcome.configured_model = model
503 key = None
504 if inputs.cache is not None:
505 key = reviewer_key(
506 base=inputs.base,
507 provider=provider,
508 fallback=inputs.fallback,
509 forced_model=inputs.forced_model,
510 reviewer=reviewer.reviewer_id,
511 model=model,
512 slice_hash=outcome.slice_hash,
513 prompt_hash=sha256_bytes(request["instructions"].encode("utf-8")),
514 message_hash=_message_hash(request),
515 definition_hash=reviewer.definition_hash,
516 runner_version=inputs.runner_version,
517 mcp_hash=sha256_json(inputs.mcp),
518 options={**options_of(settings), "maxFindings": own_limit},
519 )
520 planned.append(
521 _Planned(
522 reviewer,
523 outcome,
524 selected,
525 locations,
526 request,
527 key,
528 provider,
529 model,
530 effort,
531 _rules_by_id(rules),
532 )
533 )
534 return planned
537def _apply_answer(item: _Planned, answer: ReviewerAnswer, limit: int) -> bool:
538 """Record an answer on the outcome; ``True`` when it followed the contract."""
539 outcome = item.outcome
540 outcome.attempts += 1
541 outcome.tokens += answer.tokens or 0
542 outcome.provider = answer.provider
543 if answer.invocation_id:
544 outcome.invocation_ids.append(answer.invocation_id)
545 outcome.evidence_refs.extend(answer.evidence_refs)
546 if answer.status != "ANSWERED" or answer.result is None:
547 outcome.reason = f"no valid answer from {answer.provider}: {answer.summary}"
548 return False
549 changes = {change.path: change for change in item.files}
550 try:
551 normalized = normalize(
552 item.reviewer.reviewer_id,
553 answer.result,
554 rules=item.rules,
555 locations=item.locations,
556 changes=changes,
557 limit=min(limit, item.reviewer.spec.max_findings or limit),
558 )
559 except ContractError as error:
560 outcome.reason = f"the answer of {answer.provider} does not follow the contract: {error}"
561 return False
562 outcome.findings = normalized.findings
563 outcome.dropped_outside = normalized.dropped_outside
564 outcome.downgraded = normalized.downgraded
565 outcome.truncated = normalized.truncated
566 outcome.summary = normalized.summary
567 outcome.configured_model = answer.model or outcome.configured_model
568 outcome.executed_model = normalized.model or outcome.configured_model
569 outcome.status = verdict_of(normalized.findings)
570 outcome.over_budget = outcome.budget is not None and outcome.tokens > outcome.budget
571 return True
574def _cached_value(outcome: ReviewerOutcome) -> dict[str, Any]:
575 return {
576 "status": outcome.status,
577 "summary": outcome.summary,
578 "provider": outcome.provider,
579 "configuredModel": outcome.configured_model,
580 "executedModel": outcome.executed_model,
581 "droppedOutside": outcome.dropped_outside,
582 "downgraded": outcome.downgraded,
583 "truncated": outcome.truncated,
584 "attempts": outcome.attempts,
585 "findings": [item.as_dict() for item in outcome.findings],
586 }
589def _from_cache(outcome: ReviewerOutcome, value: Mapping[str, Any]) -> None:
590 outcome.status = str(value["status"])
591 outcome.summary = str(value.get("summary") or "")
592 outcome.provider = value.get("provider") or outcome.provider
593 outcome.configured_model = value.get("configuredModel")
594 outcome.executed_model = value.get("executedModel")
595 outcome.dropped_outside = int(value.get("droppedOutside") or 0)
596 outcome.downgraded = int(value.get("downgraded") or 0)
597 outcome.truncated = int(value.get("truncated") or 0)
598 outcome.attempts = int(value.get("attempts") or 1)
599 outcome.findings = [ReviewFinding.from_dict(item) for item in value.get("findings") or []]
600 outcome.cache = HIT
601 outcome.tokens = 0
604def _call_of(inputs: PanelInputs, item: _Planned, provider: str, attempt: int) -> ReviewerCall:
605 """The call of a planned reviewer on ``provider``; on another provider than planned (the
606 fallback) the model and effort are routed again."""
607 request = item.request
608 model, effort = item.model, item.effort
609 if provider != item.provider:
610 model, effort = inputs.route(item.reviewer, provider)
611 model = inputs.forced_model or model
612 request = {key: value for key, value in request.items() if key != "routing"}
613 if model or effort: 613 ↛ 615line 613 didn't jump to line 615 because the condition on line 613 was always true
614 request["routing"] = {"model": model, "effort": effort}
615 return ReviewerCall(
616 reviewer=item.reviewer.reviewer_id,
617 provider=provider,
618 request=request,
619 model=model,
620 effort=effort,
621 timeout_seconds=item.reviewer.spec.timeout_seconds,
622 max_tokens=item.outcome.budget or 1,
623 attempt=attempt,
624 )
627def _uncached(cache: ReviewCache | None, pending: list[_Planned]) -> list[_Planned]:
628 """The planned reviewers to call: a cached answer fills the others."""
629 calls: list[_Planned] = []
630 for item in pending:
631 cached = cache.get(item.key) if cache is not None and item.key else None
632 if cached is not None:
633 _from_cache(item.outcome, cached)
634 continue
635 item.outcome.cache = "miss" if cache is not None else "off"
636 calls.append(item)
637 return calls
640def _call_reviewers(inputs: PanelInputs, planned: list[_Planned], limit: int) -> None:
641 pending = [item for item in planned if item.outcome.status == "PENDING"]
642 if not pending:
643 return
644 cache = inputs.cache
645 calls = _uncached(cache, pending)
646 workers = inputs.settings.workers
647 answers = inputs.invoker.invoke(
648 [_call_of(inputs, item, item.provider, 1) for item in calls], workers
649 )
650 retry = [
651 item
652 for item, answer in zip(calls, answers, strict=True)
653 if not _apply_answer(item, answer, limit)
654 ]
655 if retry:
656 second = inputs.invoker.invoke(
657 [_call_of(inputs, item, inputs.fallback or item.provider, 2) for item in retry],
658 workers,
659 )
660 for item, answer in zip(retry, second, strict=True):
661 if not _apply_answer(item, answer, limit): 661 ↛ 660line 661 didn't jump to line 660 because the condition on line 661 was always true
662 item.outcome.status = "UNKNOWN"
663 item.outcome.findings = []
664 if cache is None:
665 return
666 for item in calls:
667 if item.key and item.outcome.status != "UNKNOWN":
668 cache.put(item.key, _cached_value(item.outcome))
671def _second_opinion(
672 inputs: PanelInputs, findings: list[ReviewFinding], files: list[FileChange], limit: int
673) -> tuple[list[ReviewFinding], dict[str, Any] | None]:
674 config = inputs.settings.second_opinion
675 blocking = [item for item in findings if item.blocking and item.source == "ai"]
676 if config is None or config.mode != "blocking" or not blocking:
677 return findings, None
678 provider = config.provider or inputs.fallback or inputs.provider
679 touched = sorted({item.file for item in blocking})
680 selected = [item for item in files if item.path in touched]
681 locations = reportable_locations(selected)
682 request: dict[str, Any] = {
683 "schemaVersion": REQUEST_SCHEMA_VERSION,
684 "kind": "review",
685 "readOnly": True,
686 "instructions": (
687 "You give a second opinion on blocking review findings. For each finding below, "
688 "read the code it points to and keep it only if you confirm the defect; return the "
689 "findings you confirm, unchanged, and drop the others.\n\n" + contract_text(limit)
690 ),
691 "workspace": str(inputs.workspace),
692 "reviewer": {"id": SECOND_OPINION_ID, "domain": "review", "title": "Second opinion"},
693 "outputContract": {"maxFindings": limit},
694 "slice": {"files": touched, "diff": render(selected)[:MAX_SLICE_CHARS]},
695 "reportableLocations": locations.as_ranges(),
696 "findings": [item.as_dict() for item in blocking],
697 "isolation": {"readOnly": True, "tools": ["Read", "Grep", "Glob"], "mcpServers": []},
698 **inputs.extra,
699 }
700 answer = inputs.invoker.invoke(
701 [
702 ReviewerCall(
703 reviewer=SECOND_OPINION_ID,
704 provider=provider,
705 request=request,
706 model=inputs.forced_model,
707 effort=None,
708 timeout_seconds=None,
709 max_tokens=inputs.settings.effective_budget.for_lines(
710 sum(item.changed_lines for item in selected)
711 ),
712 )
713 ],
714 1,
715 )[0]
716 record: dict[str, Any] = {"provider": provider, "tokens": answer.tokens or 0}
717 confirmed: set[tuple[str, str, int, str]] = set()
718 try:
719 if answer.status != "ANSWERED" or answer.result is None: 719 ↛ 720line 719 didn't jump to line 720 because the condition on line 719 was never true
720 raise ContractError(answer.summary)
721 normalized = normalize(
722 SECOND_OPINION_ID,
723 answer.result,
724 rules={},
725 locations=locations,
726 changes={item.path: item for item in selected},
727 limit=len(blocking) + limit,
728 )
729 confirmed = {(item.file, item.side, item.line, item.rule) for item in normalized.findings}
730 except ContractError as error:
731 record.update({"status": "UNKNOWN", "reason": str(error), "kept": len(blocking)})
732 return findings, record
733 kept: list[ReviewFinding] = []
734 refuted = 0
735 for item in findings:
736 if item.blocking and item.source == "ai" and item.key not in confirmed:
737 refuted += 1
738 kept.append(
739 replace(item, severity="suggestion", note="not confirmed by the second opinion")
740 )
741 else:
742 kept.append(item)
743 record.update({"status": "ANSWERED", "confirmed": len(blocking) - refuted, "refuted": refuted})
744 return sorted(kept, key=ReviewFinding.sort_key), record
747# ----- the panel --------------------------------------------------------------------------------
748def _global_key(inputs: PanelInputs, hashed: str) -> str:
749 """The cache key of a whole review."""
750 return global_key(
751 mode=inputs.mode,
752 diff_hash=hashed,
753 definitions_hash=sha256_json([definitions_hash(inputs.reviewers), inputs.catalog.digest]),
754 runner_version=inputs.runner_version,
755 skip=inputs.skip,
756 forced_model=inputs.forced_model,
757 provider=inputs.provider,
758 fallback=inputs.fallback,
759 options={**options_of(inputs.settings), "base": inputs.base, "head": inputs.head},
760 )
763def _deterministic(
764 inputs: PanelInputs, files: list[FileChange], locations: Locations, report: PanelReport
765) -> list[ReviewFinding]:
766 """The findings of the harness's checks and of the repository's linters, counted in the
767 report."""
768 findings, report.deterministic = deterministic_findings(inputs.catalog, files, locations)
769 if inputs.linters is not None:
770 linted = inputs.linters(
771 [
772 rule
773 for rule in inputs.catalog.tool_rules()
774 if any(t != HARNESS_TOOL for t, _ in rule.tools)
775 ],
776 locations,
777 )
778 report.deterministic["linterFindings"] = len(linted)
779 findings.extend(linted)
780 report.deterministic["findings"] = len(findings)
781 return findings
784def _verdict(
785 failed: bool, reviewers: list[ReviewerOutcome], findings: list[ReviewFinding]
786) -> Verdict:
787 """FAIL when a consistency check failed, UNKNOWN when a reviewer never answered, else the
788 verdict of the findings."""
789 if failed:
790 return "FAIL"
791 if any(item.status == "UNKNOWN" for item in reviewers):
792 return "UNKNOWN"
793 return verdict_of(findings)
796def _served_from_cache(value: Mapping[str, Any]) -> PanelReport:
797 """The report of a global cache hit: no reviewer and no second opinion was called, so the
798 review reports 0 model calls and 0 tokens; every reviewer that answered is marked as served
799 from the cache, and the cached review's numbers are kept under ``cachedFrom`` (#75)."""
800 report = PanelReport.from_dict(value)
801 report.cache = HIT
802 usage = value.get("tokens") or {}
803 report.cached_from = {
804 "total": int(usage.get("total") or 0),
805 "modelCalls": int(usage.get("modelCalls") or 0),
806 }
807 for outcome in report.reviewers:
808 if outcome.status != SKIPPED:
809 outcome.cache = HIT
810 if report.second_opinion is not None:
811 report.second_opinion = {**report.second_opinion, "tokens": 0, "cache": HIT}
812 return report
815def run_panel(inputs: PanelInputs) -> PanelReport:
816 settings = inputs.settings
817 limit = settings.finding_limit
818 files = parse_diff(inputs.diff_text)
819 locations = reportable_locations(files)
820 hashed = diff_hash(inputs.diff_text)
821 drifted = drift(inputs.reviewers, inputs.catalog)
822 key = _global_key(inputs, hashed) if inputs.cache is not None else None
823 cached = inputs.cache.get(key) if inputs.cache is not None and key is not None else None
824 if cached is not None:
825 return _served_from_cache(cached)
826 report = PanelReport(
827 mode=inputs.mode,
828 base=inputs.base,
829 head=inputs.head,
830 diff_hash=hashed,
831 provider=inputs.provider,
832 fallback=inputs.fallback,
833 verdict="PASS",
834 blocking=False,
835 findings=[],
836 reviewers=[],
837 catalog=inputs.catalog.summary(),
838 consistency=[],
839 deterministic={},
840 drift=drifted,
841 locations=locations.count(),
842 cache="miss" if inputs.cache is not None else "off",
843 )
844 # 2. consistency checks: when one fails no model is called.
845 if inputs.consistency is not None:
846 report.consistency = inputs.consistency()
847 failed = [item for item in report.consistency if item.get("status") != "PASSED"]
848 # 3. deterministic rules.
849 findings = _deterministic(inputs, files, locations, report)
850 if failed:
851 report.reviewers = [
852 ReviewerOutcome(
853 reviewer.reviewer_id,
854 reviewer.domain,
855 SKIPPED,
856 "a consistency check failed: no model was called",
857 )
858 for reviewer in sorted(inputs.reviewers, key=lambda item: item.reviewer_id)
859 ]
860 else:
861 # 4-5. reviewers.
862 planned = _plan(inputs, files, limit)
863 _call_reviewers(inputs, planned, limit)
864 report.reviewers = [item.outcome for item in planned]
865 report.reviewer_cache_hits = sum(1 for item in report.reviewers if item.cache == HIT)
866 findings.extend(finding for outcome in report.reviewers for finding in outcome.findings)
867 # 6. findings.
868 merged = _dedupe(findings)
869 merged, report.second_opinion = _second_opinion(inputs, merged, files, limit)
870 if len(merged) > limit: 870 ↛ 871line 870 didn't jump to line 871 because the condition on line 870 was never true
871 report.deterministic["truncated"] = len(merged) - limit
872 merged = merged[:limit]
873 report.findings = merged
874 # 7. verdict.
875 report.verdict = _verdict(bool(failed), report.reviewers, merged)
876 report.blocking = report.verdict in {"FAIL", "UNKNOWN"} and settings.mode != "warn"
877 if inputs.cache is not None and key is not None and report.verdict != "UNKNOWN":
878 inputs.cache.put(key, report.as_dict())
879 return report
882__all__ = [
883 "REPORT_SCHEMA_VERSION",
884 "Invoker",
885 "PanelInputs",
886 "PanelReport",
887 "ReviewerAnswer",
888 "ReviewerCall",
889 "ReviewerOutcome",
890 "Route",
891 "deterministic_findings",
892 "options_of",
893 "run_panel",
894]