Coverage for src/governed_harness/review/panel.py: 95%

411 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""The review panel (#57): reviewers by domain over diff slices, with a deterministic verdict. 

2 

3``run_panel`` is the whole review, independent of where it runs (INDEPENDENT_REVIEW of a 

4governed run, or ``harness review-code`` outside one). The caller gives it the diff, the catalog, 

5the reviewers and an ``Invoker`` that calls the providers; everything else is decided here, in 

6this order, so that no token is spent that a cheaper step could save: 

7 

81. the global cache: the same review of the same diff in the same mode is not repeated; 

92. the project's consistency checks: when one fails, no model is called; 

103. the deterministic rules (``verified_by: tool:...``): the harness's own checks and, outside a 

11 run, the linters the repository configures; they never reach a model; 

124. the reviewers whose mode matches, whose slice changed and whose signal is present; 

135. per reviewer: the per-reviewer cache, else one call with a budget proportional to the slice; 

14 an invalid answer is ``UNKNOWN`` and gets one retry with the alternate provider; a persistent 

15 ``UNKNOWN`` blocks; 

166. the findings: outside the reportable locations dropped, outside the catalog downgraded 

17 without concrete evidence, capped, ordered; an optional second opinion on the blocking ones; 

187. the verdict, recomputed: ``FAIL`` with any error or failed consistency check, ``UNKNOWN`` 

19 with a reviewer that never answered, ``PASS_WARN`` with suggestions only, else ``PASS``. 

20 

21The report is deterministic: the same inputs and answers give the same report and digest.""" 

22 

23from __future__ import annotations 

24 

25from collections.abc import Callable, Mapping, Sequence 

26from dataclasses import dataclass, field, replace 

27from pathlib import Path 

28from typing import Any, Literal, Protocol 

29 

30from governed_harness.agents.requests import REQUEST_SCHEMA_VERSION 

31from governed_harness.configuration.review import ReviewPanelConfig 

32from governed_harness.evidence.hashing import sha256_bytes, sha256_json 

33from governed_harness.review.cache import ReviewCache, global_key, reviewer_key 

34from governed_harness.review.checks import CHECKS, run_checks 

35from governed_harness.review.contract import ( 

36 ContractError, 

37 ReviewFinding, 

38 Verdict, 

39 contract_text, 

40 normalize, 

41 verdict_of, 

42) 

43from governed_harness.review.diff import ( 

44 FileChange, 

45 Locations, 

46 diff_hash, 

47 parse_diff, 

48 render, 

49 reportable_locations, 

50) 

51from governed_harness.review.reviewers import Reviewer, definitions_hash, drift 

52from governed_harness.review.rules import HARNESS_TOOL, Catalog, Rule 

53from governed_harness.review.signals import activation, slice_files 

54 

55REPORT_SCHEMA_VERSION = "1.0" 

56HIT = "hit" 

57"""The cache value of an answer served from a cache (no model was called).""" 

58SKIPPED = "SKIPPED" 

59MAX_SLICE_CHARS = 200_000 

60SECOND_OPINION_ID = "second-opinion" 

61 

62 

63# ----- what the caller provides ----------------------------------------------------------------- 

64@dataclass(frozen=True) 

65class ReviewerCall: 

66 reviewer: str 

67 provider: str 

68 request: dict[str, Any] 

69 model: str | None 

70 effort: str | None 

71 timeout_seconds: int | None 

72 max_tokens: int 

73 attempt: int = 1 

74 

75 

76@dataclass(frozen=True) 

77class ReviewerAnswer: 

78 status: Literal["ANSWERED", "ERROR"] 

79 result: dict[str, Any] | None 

80 summary: str 

81 provider: str 

82 model: str | None = None 

83 """The model the call was configured with (``None``: the provider's default).""" 

84 tokens: int | None = None 

85 invocation_id: str | None = None 

86 evidence_refs: tuple[str, ...] = () 

87 

88 

89class Invoker(Protocol): 

90 def invoke(self, calls: Sequence[ReviewerCall], workers: int) -> list[ReviewerAnswer]: 

91 """Answer every call (in parallel up to ``workers``), in the order of ``calls``.""" 

92 

93 

94Route = Callable[[Reviewer, str], tuple[str | None, str | None]] 

95"""The model and effort of a reviewer on a provider.""" 

96 

97 

98@dataclass 

99class PanelInputs: 

100 workspace: Path 

101 diff_text: str 

102 mode: str 

103 catalog: Catalog 

104 reviewers: list[Reviewer] 

105 settings: ReviewPanelConfig 

106 provider: str 

107 invoker: Invoker 

108 route: Route 

109 runner_version: str 

110 base: str | None = None 

111 head: str | None = None 

112 fallback: str | None = None 

113 forced_model: str | None = None 

114 skip: tuple[str, ...] = () 

115 packs: tuple[str, ...] | None = None 

116 cache: ReviewCache | None = None 

117 consistency: Callable[[], list[dict[str, Any]]] | None = None 

118 linters: Callable[[list[Rule], Locations], list[ReviewFinding]] | None = None 

119 extra: dict[str, Any] = field(default_factory=dict) 

120 mcp: dict[str, Any] = field(default_factory=dict) 

121 """The MCP configuration reviewers may use (servers of the allowlist only).""" 

122 untrusted_notice: str | None = None 

123 """Since #5: the notice that repository content is untrusted context, not instructions.""" 

124 

125 

126# ----- the report ------------------------------------------------------------------------------- 

127@dataclass 

128class ReviewerOutcome: 

129 reviewer: str 

130 domain: str 

131 status: str 

132 reason: str 

133 activation: dict[str, Any] = field(default_factory=dict) 

134 files: list[str] = field(default_factory=list) 

135 changed_lines: int = 0 

136 slice_hash: str | None = None 

137 budget: int | None = None 

138 provider: str | None = None 

139 configured_model: str | None = None 

140 executed_model: str | None = None 

141 attempts: int = 0 

142 cache: str = "off" 

143 tokens: int = 0 

144 over_budget: bool = False 

145 dropped_outside: int = 0 

146 downgraded: int = 0 

147 truncated: int = 0 

148 summary: str = "" 

149 findings: list[ReviewFinding] = field(default_factory=list) 

150 invocation_ids: list[str] = field(default_factory=list) 

151 evidence_refs: list[str] = field(default_factory=list) 

152 

153 def as_dict(self) -> dict[str, Any]: 

154 value: dict[str, Any] = { 

155 "id": self.reviewer, 

156 "domain": self.domain, 

157 "status": self.status, 

158 "reason": self.reason, 

159 "activation": self.activation, 

160 "files": self.files, 

161 "changedLines": self.changed_lines, 

162 "findings": len(self.findings), 

163 } 

164 if self.status not in {SKIPPED}: 

165 value.update( 

166 { 

167 "sliceHash": self.slice_hash, 

168 "budget": self.budget, 

169 "provider": self.provider, 

170 "model": { 

171 "configured": self.configured_model, 

172 "executed": self.executed_model, 

173 }, 

174 "attempts": self.attempts, 

175 "cache": self.cache, 

176 "tokens": self.tokens, 

177 "overBudget": self.over_budget, 

178 "droppedOutside": self.dropped_outside, 

179 "downgraded": self.downgraded, 

180 "truncated": self.truncated, 

181 "summary": self.summary, 

182 } 

183 ) 

184 return value 

185 

186 

187@dataclass 

188class PanelReport: 

189 mode: str 

190 base: str | None 

191 head: str | None 

192 diff_hash: str 

193 provider: str 

194 fallback: str | None 

195 verdict: Verdict 

196 blocking: bool 

197 findings: list[ReviewFinding] 

198 reviewers: list[ReviewerOutcome] 

199 catalog: dict[str, Any] 

200 consistency: list[dict[str, Any]] 

201 deterministic: dict[str, Any] 

202 drift: list[str] 

203 locations: int 

204 second_opinion: dict[str, Any] | None = None 

205 cache: str = "off" 

206 reviewer_cache_hits: int = 0 

207 cached_from: dict[str, int] | None = None 

208 """On a global cache hit: the tokens and model calls of the review the report was cached 

209 from (this review called no model, #75).""" 

210 

211 @property 

212 def tokens(self) -> int: 

213 return sum(item.tokens for item in self.reviewers) + int( 

214 (self.second_opinion or {}).get("tokens") or 0 

215 ) 

216 

217 @property 

218 def model_calls(self) -> int: 

219 calls = sum(item.attempts for item in self.reviewers if item.cache != HIT) 

220 second = self.second_opinion or {} 

221 return calls + int(bool(second.get("status")) and second.get("cache") != HIT) 

222 

223 def body(self) -> dict[str, Any]: 

224 """The deterministic part of the report (what its digest covers).""" 

225 return { 

226 "schemaVersion": REPORT_SCHEMA_VERSION, 

227 "mode": self.mode, 

228 "base": self.base, 

229 "head": self.head, 

230 "diffHash": self.diff_hash, 

231 "verdict": self.verdict, 

232 "findings": [item.as_dict() for item in self.findings], 

233 "reviewers": [{"id": item.reviewer, "status": item.status} for item in self.reviewers], 

234 "consistency": [ 

235 {"id": item["id"], "status": item["status"]} for item in self.consistency 

236 ], 

237 "catalog": self.catalog.get("digest"), 

238 } 

239 

240 @property 

241 def digest(self) -> str: 

242 return sha256_json(self.body()) 

243 

244 def as_dict(self) -> dict[str, Any]: 

245 errors = sum(1 for item in self.findings if item.blocking) 

246 tokens: dict[str, Any] = { 

247 "total": self.tokens, 

248 "byReviewer": {item.reviewer: item.tokens for item in self.reviewers}, 

249 "modelCalls": self.model_calls, 

250 } 

251 if self.cached_from is not None: 

252 tokens["cachedFrom"] = dict(self.cached_from) 

253 return { 

254 **self.body(), 

255 "digest": self.digest, 

256 "provider": self.provider, 

257 "fallbackProvider": self.fallback, 

258 "blocking": self.blocking, 

259 "counts": { 

260 "findings": len(self.findings), 

261 "errors": errors, 

262 "suggestions": len(self.findings) - errors, 

263 "reportableLines": self.locations, 

264 "droppedOutside": sum(item.dropped_outside for item in self.reviewers), 

265 "downgraded": sum(item.downgraded for item in self.reviewers), 

266 "truncated": sum(item.truncated for item in self.reviewers), 

267 }, 

268 "reviewers": [item.as_dict() for item in self.reviewers], 

269 "consistency": self.consistency, 

270 "deterministic": self.deterministic, 

271 "catalog": self.catalog, 

272 "drift": self.drift, 

273 "secondOpinion": self.second_opinion, 

274 "cache": {"global": self.cache, "reviewerHits": self.reviewer_cache_hits}, 

275 "tokens": tokens, 

276 } 

277 

278 @classmethod 

279 def from_dict(cls, value: Mapping[str, Any]) -> PanelReport: 

280 reviewers = [_outcome_from_dict(item) for item in value.get("reviewers") or []] 

281 return cls( 

282 mode=str(value["mode"]), 

283 base=value.get("base"), 

284 head=value.get("head"), 

285 diff_hash=str(value["diffHash"]), 

286 provider=str(value.get("provider") or ""), 

287 fallback=value.get("fallbackProvider"), 

288 verdict=value["verdict"], 

289 blocking=bool(value.get("blocking")), 

290 findings=[ReviewFinding.from_dict(item) for item in value.get("findings") or []], 

291 reviewers=reviewers, 

292 catalog=dict(value.get("catalog") or {}), 

293 consistency=list(value.get("consistency") or []), 

294 deterministic=dict(value.get("deterministic") or {}), 

295 drift=list(value.get("drift") or []), 

296 locations=int((value.get("counts") or {}).get("reportableLines") or 0), 

297 second_opinion=value.get("secondOpinion"), 

298 ) 

299 

300 

301def _outcome_from_dict(item: Mapping[str, Any]) -> ReviewerOutcome: 

302 """A reviewer outcome of a cached report (its findings are in the report's list).""" 

303 model = item.get("model") or {} 

304 return ReviewerOutcome( 

305 reviewer=str(item["id"]), 

306 domain=str(item.get("domain") or ""), 

307 status=str(item["status"]), 

308 reason=str(item.get("reason") or ""), 

309 activation=dict(item.get("activation") or {}), 

310 files=list(item.get("files") or []), 

311 changed_lines=int(item.get("changedLines") or 0), 

312 slice_hash=item.get("sliceHash"), 

313 budget=item.get("budget"), 

314 provider=item.get("provider"), 

315 configured_model=model.get("configured"), 

316 executed_model=model.get("executed"), 

317 attempts=int(item.get("attempts") or 0), 

318 cache=str(item.get("cache") or "off"), 

319 tokens=0, 

320 summary=str(item.get("summary") or ""), 

321 ) 

322 

323 

324# ----- helpers ---------------------------------------------------------------------------------- 

325def options_of(settings: ReviewPanelConfig) -> dict[str, Any]: 

326 """The settings that change a review's result (part of both cache keys).""" 

327 second = settings.second_opinion 

328 return { 

329 "maxFindings": settings.finding_limit, 

330 "budget": settings.effective_budget.model_dump(mode="json", by_alias=True), 

331 "runTools": bool(settings.run_tools), 

332 "secondOpinion": second.model_dump(mode="json", by_alias=True) if second else None, 

333 "consistency": [ 

334 item.model_dump(mode="json", by_alias=True) 

335 for item in settings.consistency_checks or () 

336 ], 

337 "policy": settings.mode or "enforce", 

338 } 

339 

340 

341def _rules_by_id(rules: Sequence[Rule]) -> dict[str, Rule]: 

342 return {rule.rule_id: rule for rule in rules} 

343 

344 

345def deterministic_findings( 

346 catalog: Catalog, files: list[FileChange], locations: Locations 

347) -> tuple[list[ReviewFinding], dict[str, Any]]: 

348 """The findings of the rules verified by the harness's own checks.""" 

349 by_check: dict[str, list[Rule]] = {} 

350 unknown: list[str] = [] 

351 for rule in catalog.tool_rules(): 

352 for name in (name for tool, name in rule.tools if tool == HARNESS_TOOL): 

353 if name in CHECKS: 353 ↛ 356line 353 didn't jump to line 356 because the condition on line 353 was always true

354 by_check.setdefault(name, []).append(rule) 

355 else: 

356 unknown.append(f"{rule.rule_id}: tool:{HARNESS_TOOL}:{name}") 

357 findings = [ 

358 ReviewFinding( 

359 reviewer=HARNESS_TOOL, 

360 file=hit.path, 

361 side=hit.side, 

362 line=hit.line, 

363 rule=rule.rule_id, 

364 severity="error" if rule.blocking else "suggestion", 

365 issue=hit.message, 

366 evidence=hit.evidence, 

367 priority=rule.priority, 

368 source=f"tool:{HARNESS_TOOL}:{hit.check}", 

369 ) 

370 for hit in run_checks(by_check, files) 

371 for rule in by_check.get(hit.check, []) 

372 if rule.applies(hit.path) and locations.allows(hit.path, hit.side, hit.line) 

373 ] 

374 return findings, { 

375 "checks": sorted(by_check), 

376 "unknownChecks": sorted(unknown), 

377 "rules": sorted(rule.rule_id for rule in catalog.tool_rules()), 

378 } 

379 

380 

381def _dedupe(findings: list[ReviewFinding]) -> list[ReviewFinding]: 

382 seen: set[tuple[str, str, int, str]] = set() 

383 kept: list[ReviewFinding] = [] 

384 for item in sorted(findings, key=ReviewFinding.sort_key): 

385 if item.key in seen: 385 ↛ 386line 385 didn't jump to line 386 because the condition on line 385 was never true

386 continue 

387 seen.add(item.key) 

388 kept.append(item) 

389 return kept 

390 

391 

392@dataclass 

393class _Planned: 

394 reviewer: Reviewer 

395 outcome: ReviewerOutcome 

396 files: list[FileChange] 

397 locations: Locations 

398 request: dict[str, Any] 

399 key: str | None 

400 provider: str 

401 model: str | None 

402 effort: str | None 

403 rules: dict[str, Rule] 

404 

405 

406def _request( 

407 inputs: PanelInputs, 

408 reviewer: Reviewer, 

409 files: list[FileChange], 

410 locations: Locations, 

411 budget: int, 

412 model: str | None, 

413 effort: str | None, 

414 limit: int, 

415) -> dict[str, Any]: 

416 text = render(files) 

417 truncated = len(text) > MAX_SLICE_CHARS 

418 notice = f"\n\n{inputs.untrusted_notice}" if inputs.untrusted_notice else "" 

419 request: dict[str, Any] = { 

420 "schemaVersion": REQUEST_SCHEMA_VERSION, 

421 "kind": "review", 

422 "readOnly": True, 

423 "instructions": reviewer.body.rstrip("\n") + notice + "\n\n" + contract_text(limit), 

424 "workspace": str(inputs.workspace), 

425 "reviewer": { 

426 "id": reviewer.reviewer_id, 

427 "domain": reviewer.domain, 

428 }, 

429 "outputContract": {"maxFindings": limit}, 

430 "slice": { 

431 "files": [item.path for item in files], 

432 "changedLines": sum(item.changed_lines for item in files), 

433 "diff": text[:MAX_SLICE_CHARS], 

434 "truncated": truncated, 

435 }, 

436 "reportableLocations": locations.as_ranges(), 

437 "budget": {"maxTokens": budget}, 

438 "isolation": { 

439 "readOnly": True, 

440 "tools": list(reviewer.spec.tools), 

441 "mcpServers": sorted( 

442 set(reviewer.spec.mcp_servers) & set(inputs.mcp.get("mcpServers") or {}) 

443 ), 

444 }, 

445 } 

446 if model or effort: 446 ↛ 448line 446 didn't jump to line 448 because the condition on line 446 was always true

447 request["routing"] = {"model": model, "effort": effort} 

448 request.update(inputs.extra) 

449 return request 

450 

451 

452def _message_hash(request: Mapping[str, Any]) -> str: 

453 return sha256_json( 

454 {key: value for key, value in request.items() if key not in {"instructions", "workspace"}} 

455 ) 

456 

457 

458def _plan(inputs: PanelInputs, files: list[FileChange], limit: int) -> list[_Planned]: 

459 settings = inputs.settings 

460 planned: list[_Planned] = [] 

461 for reviewer in sorted(inputs.reviewers, key=lambda item: item.reviewer_id): 

462 outcome = ReviewerOutcome(reviewer.reviewer_id, reviewer.domain, SKIPPED, "") 

463 if reviewer.reviewer_id in inputs.skip: 463 ↛ 464line 463 didn't jump to line 464 because the condition on line 463 was never true

464 outcome.reason = "skipped by --skip" 

465 planned.append( 

466 _Planned(reviewer, outcome, [], Locations({}), {}, None, "", None, None, {}) 

467 ) 

468 continue 

469 if inputs.mode not in reviewer.spec.modes: 469 ↛ 470line 469 didn't jump to line 470 because the condition on line 469 was never true

470 outcome.reason = f"does not run in {inputs.mode} mode" 

471 planned.append( 

472 _Planned(reviewer, outcome, [], Locations({}), {}, None, "", None, None, {}) 

473 ) 

474 continue 

475 selected = slice_files(files, reviewer.spec.diff_slice) 

476 active = activation(reviewer, selected, inputs.packs) 

477 outcome.activation = active.as_dict() 

478 outcome.files = [item.path for item in selected] 

479 outcome.changed_lines = sum(item.changed_lines for item in selected) 

480 rules = inputs.catalog.for_domain(reviewer.domain, ai_only=True) 

481 if not active.active: 

482 outcome.reason = active.reason 

483 planned.append( 

484 _Planned(reviewer, outcome, [], Locations({}), {}, None, "", None, None, {}) 

485 ) 

486 continue 

487 provider = reviewer.spec.provider or inputs.provider 

488 model, effort = inputs.route(reviewer, provider) 

489 if inputs.forced_model: 489 ↛ 490line 489 didn't jump to line 490 because the condition on line 489 was never true

490 model = inputs.forced_model 

491 budget = settings.effective_budget.for_lines( 

492 outcome.changed_lines, reviewer.spec.max_budget 

493 ) 

494 locations = reportable_locations(selected) 

495 own_limit = min(limit, reviewer.spec.max_findings or limit) 

496 request = _request(inputs, reviewer, selected, locations, budget, model, effort, own_limit) 

497 outcome.status = "PENDING" 

498 outcome.reason = active.reason 

499 outcome.slice_hash = diff_hash(request["slice"]["diff"]) 

500 outcome.budget = budget 

501 outcome.provider = provider 

502 outcome.configured_model = model 

503 key = None 

504 if inputs.cache is not None: 

505 key = reviewer_key( 

506 base=inputs.base, 

507 provider=provider, 

508 fallback=inputs.fallback, 

509 forced_model=inputs.forced_model, 

510 reviewer=reviewer.reviewer_id, 

511 model=model, 

512 slice_hash=outcome.slice_hash, 

513 prompt_hash=sha256_bytes(request["instructions"].encode("utf-8")), 

514 message_hash=_message_hash(request), 

515 definition_hash=reviewer.definition_hash, 

516 runner_version=inputs.runner_version, 

517 mcp_hash=sha256_json(inputs.mcp), 

518 options={**options_of(settings), "maxFindings": own_limit}, 

519 ) 

520 planned.append( 

521 _Planned( 

522 reviewer, 

523 outcome, 

524 selected, 

525 locations, 

526 request, 

527 key, 

528 provider, 

529 model, 

530 effort, 

531 _rules_by_id(rules), 

532 ) 

533 ) 

534 return planned 

535 

536 

537def _apply_answer(item: _Planned, answer: ReviewerAnswer, limit: int) -> bool: 

538 """Record an answer on the outcome; ``True`` when it followed the contract.""" 

539 outcome = item.outcome 

540 outcome.attempts += 1 

541 outcome.tokens += answer.tokens or 0 

542 outcome.provider = answer.provider 

543 if answer.invocation_id: 

544 outcome.invocation_ids.append(answer.invocation_id) 

545 outcome.evidence_refs.extend(answer.evidence_refs) 

546 if answer.status != "ANSWERED" or answer.result is None: 

547 outcome.reason = f"no valid answer from {answer.provider}: {answer.summary}" 

548 return False 

549 changes = {change.path: change for change in item.files} 

550 try: 

551 normalized = normalize( 

552 item.reviewer.reviewer_id, 

553 answer.result, 

554 rules=item.rules, 

555 locations=item.locations, 

556 changes=changes, 

557 limit=min(limit, item.reviewer.spec.max_findings or limit), 

558 ) 

559 except ContractError as error: 

560 outcome.reason = f"the answer of {answer.provider} does not follow the contract: {error}" 

561 return False 

562 outcome.findings = normalized.findings 

563 outcome.dropped_outside = normalized.dropped_outside 

564 outcome.downgraded = normalized.downgraded 

565 outcome.truncated = normalized.truncated 

566 outcome.summary = normalized.summary 

567 outcome.configured_model = answer.model or outcome.configured_model 

568 outcome.executed_model = normalized.model or outcome.configured_model 

569 outcome.status = verdict_of(normalized.findings) 

570 outcome.over_budget = outcome.budget is not None and outcome.tokens > outcome.budget 

571 return True 

572 

573 

574def _cached_value(outcome: ReviewerOutcome) -> dict[str, Any]: 

575 return { 

576 "status": outcome.status, 

577 "summary": outcome.summary, 

578 "provider": outcome.provider, 

579 "configuredModel": outcome.configured_model, 

580 "executedModel": outcome.executed_model, 

581 "droppedOutside": outcome.dropped_outside, 

582 "downgraded": outcome.downgraded, 

583 "truncated": outcome.truncated, 

584 "attempts": outcome.attempts, 

585 "findings": [item.as_dict() for item in outcome.findings], 

586 } 

587 

588 

589def _from_cache(outcome: ReviewerOutcome, value: Mapping[str, Any]) -> None: 

590 outcome.status = str(value["status"]) 

591 outcome.summary = str(value.get("summary") or "") 

592 outcome.provider = value.get("provider") or outcome.provider 

593 outcome.configured_model = value.get("configuredModel") 

594 outcome.executed_model = value.get("executedModel") 

595 outcome.dropped_outside = int(value.get("droppedOutside") or 0) 

596 outcome.downgraded = int(value.get("downgraded") or 0) 

597 outcome.truncated = int(value.get("truncated") or 0) 

598 outcome.attempts = int(value.get("attempts") or 1) 

599 outcome.findings = [ReviewFinding.from_dict(item) for item in value.get("findings") or []] 

600 outcome.cache = HIT 

601 outcome.tokens = 0 

602 

603 

604def _call_of(inputs: PanelInputs, item: _Planned, provider: str, attempt: int) -> ReviewerCall: 

605 """The call of a planned reviewer on ``provider``; on another provider than planned (the 

606 fallback) the model and effort are routed again.""" 

607 request = item.request 

608 model, effort = item.model, item.effort 

609 if provider != item.provider: 

610 model, effort = inputs.route(item.reviewer, provider) 

611 model = inputs.forced_model or model 

612 request = {key: value for key, value in request.items() if key != "routing"} 

613 if model or effort: 613 ↛ 615line 613 didn't jump to line 615 because the condition on line 613 was always true

614 request["routing"] = {"model": model, "effort": effort} 

615 return ReviewerCall( 

616 reviewer=item.reviewer.reviewer_id, 

617 provider=provider, 

618 request=request, 

619 model=model, 

620 effort=effort, 

621 timeout_seconds=item.reviewer.spec.timeout_seconds, 

622 max_tokens=item.outcome.budget or 1, 

623 attempt=attempt, 

624 ) 

625 

626 

627def _uncached(cache: ReviewCache | None, pending: list[_Planned]) -> list[_Planned]: 

628 """The planned reviewers to call: a cached answer fills the others.""" 

629 calls: list[_Planned] = [] 

630 for item in pending: 

631 cached = cache.get(item.key) if cache is not None and item.key else None 

632 if cached is not None: 

633 _from_cache(item.outcome, cached) 

634 continue 

635 item.outcome.cache = "miss" if cache is not None else "off" 

636 calls.append(item) 

637 return calls 

638 

639 

640def _call_reviewers(inputs: PanelInputs, planned: list[_Planned], limit: int) -> None: 

641 pending = [item for item in planned if item.outcome.status == "PENDING"] 

642 if not pending: 

643 return 

644 cache = inputs.cache 

645 calls = _uncached(cache, pending) 

646 workers = inputs.settings.workers 

647 answers = inputs.invoker.invoke( 

648 [_call_of(inputs, item, item.provider, 1) for item in calls], workers 

649 ) 

650 retry = [ 

651 item 

652 for item, answer in zip(calls, answers, strict=True) 

653 if not _apply_answer(item, answer, limit) 

654 ] 

655 if retry: 

656 second = inputs.invoker.invoke( 

657 [_call_of(inputs, item, inputs.fallback or item.provider, 2) for item in retry], 

658 workers, 

659 ) 

660 for item, answer in zip(retry, second, strict=True): 

661 if not _apply_answer(item, answer, limit): 661 ↛ 660line 661 didn't jump to line 660 because the condition on line 661 was always true

662 item.outcome.status = "UNKNOWN" 

663 item.outcome.findings = [] 

664 if cache is None: 

665 return 

666 for item in calls: 

667 if item.key and item.outcome.status != "UNKNOWN": 

668 cache.put(item.key, _cached_value(item.outcome)) 

669 

670 

671def _second_opinion( 

672 inputs: PanelInputs, findings: list[ReviewFinding], files: list[FileChange], limit: int 

673) -> tuple[list[ReviewFinding], dict[str, Any] | None]: 

674 config = inputs.settings.second_opinion 

675 blocking = [item for item in findings if item.blocking and item.source == "ai"] 

676 if config is None or config.mode != "blocking" or not blocking: 

677 return findings, None 

678 provider = config.provider or inputs.fallback or inputs.provider 

679 touched = sorted({item.file for item in blocking}) 

680 selected = [item for item in files if item.path in touched] 

681 locations = reportable_locations(selected) 

682 request: dict[str, Any] = { 

683 "schemaVersion": REQUEST_SCHEMA_VERSION, 

684 "kind": "review", 

685 "readOnly": True, 

686 "instructions": ( 

687 "You give a second opinion on blocking review findings. For each finding below, " 

688 "read the code it points to and keep it only if you confirm the defect; return the " 

689 "findings you confirm, unchanged, and drop the others.\n\n" + contract_text(limit) 

690 ), 

691 "workspace": str(inputs.workspace), 

692 "reviewer": {"id": SECOND_OPINION_ID, "domain": "review", "title": "Second opinion"}, 

693 "outputContract": {"maxFindings": limit}, 

694 "slice": {"files": touched, "diff": render(selected)[:MAX_SLICE_CHARS]}, 

695 "reportableLocations": locations.as_ranges(), 

696 "findings": [item.as_dict() for item in blocking], 

697 "isolation": {"readOnly": True, "tools": ["Read", "Grep", "Glob"], "mcpServers": []}, 

698 **inputs.extra, 

699 } 

700 answer = inputs.invoker.invoke( 

701 [ 

702 ReviewerCall( 

703 reviewer=SECOND_OPINION_ID, 

704 provider=provider, 

705 request=request, 

706 model=inputs.forced_model, 

707 effort=None, 

708 timeout_seconds=None, 

709 max_tokens=inputs.settings.effective_budget.for_lines( 

710 sum(item.changed_lines for item in selected) 

711 ), 

712 ) 

713 ], 

714 1, 

715 )[0] 

716 record: dict[str, Any] = {"provider": provider, "tokens": answer.tokens or 0} 

717 confirmed: set[tuple[str, str, int, str]] = set() 

718 try: 

719 if answer.status != "ANSWERED" or answer.result is None: 719 ↛ 720line 719 didn't jump to line 720 because the condition on line 719 was never true

720 raise ContractError(answer.summary) 

721 normalized = normalize( 

722 SECOND_OPINION_ID, 

723 answer.result, 

724 rules={}, 

725 locations=locations, 

726 changes={item.path: item for item in selected}, 

727 limit=len(blocking) + limit, 

728 ) 

729 confirmed = {(item.file, item.side, item.line, item.rule) for item in normalized.findings} 

730 except ContractError as error: 

731 record.update({"status": "UNKNOWN", "reason": str(error), "kept": len(blocking)}) 

732 return findings, record 

733 kept: list[ReviewFinding] = [] 

734 refuted = 0 

735 for item in findings: 

736 if item.blocking and item.source == "ai" and item.key not in confirmed: 

737 refuted += 1 

738 kept.append( 

739 replace(item, severity="suggestion", note="not confirmed by the second opinion") 

740 ) 

741 else: 

742 kept.append(item) 

743 record.update({"status": "ANSWERED", "confirmed": len(blocking) - refuted, "refuted": refuted}) 

744 return sorted(kept, key=ReviewFinding.sort_key), record 

745 

746 

747# ----- the panel -------------------------------------------------------------------------------- 

748def _global_key(inputs: PanelInputs, hashed: str) -> str: 

749 """The cache key of a whole review.""" 

750 return global_key( 

751 mode=inputs.mode, 

752 diff_hash=hashed, 

753 definitions_hash=sha256_json([definitions_hash(inputs.reviewers), inputs.catalog.digest]), 

754 runner_version=inputs.runner_version, 

755 skip=inputs.skip, 

756 forced_model=inputs.forced_model, 

757 provider=inputs.provider, 

758 fallback=inputs.fallback, 

759 options={**options_of(inputs.settings), "base": inputs.base, "head": inputs.head}, 

760 ) 

761 

762 

763def _deterministic( 

764 inputs: PanelInputs, files: list[FileChange], locations: Locations, report: PanelReport 

765) -> list[ReviewFinding]: 

766 """The findings of the harness's checks and of the repository's linters, counted in the 

767 report.""" 

768 findings, report.deterministic = deterministic_findings(inputs.catalog, files, locations) 

769 if inputs.linters is not None: 

770 linted = inputs.linters( 

771 [ 

772 rule 

773 for rule in inputs.catalog.tool_rules() 

774 if any(t != HARNESS_TOOL for t, _ in rule.tools) 

775 ], 

776 locations, 

777 ) 

778 report.deterministic["linterFindings"] = len(linted) 

779 findings.extend(linted) 

780 report.deterministic["findings"] = len(findings) 

781 return findings 

782 

783 

784def _verdict( 

785 failed: bool, reviewers: list[ReviewerOutcome], findings: list[ReviewFinding] 

786) -> Verdict: 

787 """FAIL when a consistency check failed, UNKNOWN when a reviewer never answered, else the 

788 verdict of the findings.""" 

789 if failed: 

790 return "FAIL" 

791 if any(item.status == "UNKNOWN" for item in reviewers): 

792 return "UNKNOWN" 

793 return verdict_of(findings) 

794 

795 

796def _served_from_cache(value: Mapping[str, Any]) -> PanelReport: 

797 """The report of a global cache hit: no reviewer and no second opinion was called, so the 

798 review reports 0 model calls and 0 tokens; every reviewer that answered is marked as served 

799 from the cache, and the cached review's numbers are kept under ``cachedFrom`` (#75).""" 

800 report = PanelReport.from_dict(value) 

801 report.cache = HIT 

802 usage = value.get("tokens") or {} 

803 report.cached_from = { 

804 "total": int(usage.get("total") or 0), 

805 "modelCalls": int(usage.get("modelCalls") or 0), 

806 } 

807 for outcome in report.reviewers: 

808 if outcome.status != SKIPPED: 

809 outcome.cache = HIT 

810 if report.second_opinion is not None: 

811 report.second_opinion = {**report.second_opinion, "tokens": 0, "cache": HIT} 

812 return report 

813 

814 

815def run_panel(inputs: PanelInputs) -> PanelReport: 

816 settings = inputs.settings 

817 limit = settings.finding_limit 

818 files = parse_diff(inputs.diff_text) 

819 locations = reportable_locations(files) 

820 hashed = diff_hash(inputs.diff_text) 

821 drifted = drift(inputs.reviewers, inputs.catalog) 

822 key = _global_key(inputs, hashed) if inputs.cache is not None else None 

823 cached = inputs.cache.get(key) if inputs.cache is not None and key is not None else None 

824 if cached is not None: 

825 return _served_from_cache(cached) 

826 report = PanelReport( 

827 mode=inputs.mode, 

828 base=inputs.base, 

829 head=inputs.head, 

830 diff_hash=hashed, 

831 provider=inputs.provider, 

832 fallback=inputs.fallback, 

833 verdict="PASS", 

834 blocking=False, 

835 findings=[], 

836 reviewers=[], 

837 catalog=inputs.catalog.summary(), 

838 consistency=[], 

839 deterministic={}, 

840 drift=drifted, 

841 locations=locations.count(), 

842 cache="miss" if inputs.cache is not None else "off", 

843 ) 

844 # 2. consistency checks: when one fails no model is called. 

845 if inputs.consistency is not None: 

846 report.consistency = inputs.consistency() 

847 failed = [item for item in report.consistency if item.get("status") != "PASSED"] 

848 # 3. deterministic rules. 

849 findings = _deterministic(inputs, files, locations, report) 

850 if failed: 

851 report.reviewers = [ 

852 ReviewerOutcome( 

853 reviewer.reviewer_id, 

854 reviewer.domain, 

855 SKIPPED, 

856 "a consistency check failed: no model was called", 

857 ) 

858 for reviewer in sorted(inputs.reviewers, key=lambda item: item.reviewer_id) 

859 ] 

860 else: 

861 # 4-5. reviewers. 

862 planned = _plan(inputs, files, limit) 

863 _call_reviewers(inputs, planned, limit) 

864 report.reviewers = [item.outcome for item in planned] 

865 report.reviewer_cache_hits = sum(1 for item in report.reviewers if item.cache == HIT) 

866 findings.extend(finding for outcome in report.reviewers for finding in outcome.findings) 

867 # 6. findings. 

868 merged = _dedupe(findings) 

869 merged, report.second_opinion = _second_opinion(inputs, merged, files, limit) 

870 if len(merged) > limit: 870 ↛ 871line 870 didn't jump to line 871 because the condition on line 870 was never true

871 report.deterministic["truncated"] = len(merged) - limit 

872 merged = merged[:limit] 

873 report.findings = merged 

874 # 7. verdict. 

875 report.verdict = _verdict(bool(failed), report.reviewers, merged) 

876 report.blocking = report.verdict in {"FAIL", "UNKNOWN"} and settings.mode != "warn" 

877 if inputs.cache is not None and key is not None and report.verdict != "UNKNOWN": 

878 inputs.cache.put(key, report.as_dict()) 

879 return report 

880 

881 

882__all__ = [ 

883 "REPORT_SCHEMA_VERSION", 

884 "Invoker", 

885 "PanelInputs", 

886 "PanelReport", 

887 "ReviewerAnswer", 

888 "ReviewerCall", 

889 "ReviewerOutcome", 

890 "Route", 

891 "deterministic_findings", 

892 "options_of", 

893 "run_panel", 

894]