Coverage for src/governed_harness/application/review.py: 95%

218 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""The decision brief: everything a person needs to decide on one ChangeSet digest. 

2 

3Built only from recorded data by fixed rules (no generated text): what was asked, what changed, 

4the risks with their location, what was verified on which digest and what was not, active 

5exceptions, retries and corrections, and what changed since the last decision. It reads the 

6record and never writes to it.""" 

7 

8from __future__ import annotations 

9 

10import json 

11from collections import Counter 

12from typing import Any 

13 

14from governed_harness.domain.enums import FindingSeverity, PhaseId, ResultStatus 

15from governed_harness.domain.models import ( 

16 AgentSelfReport, 

17 ChangeSet, 

18 ComponentProvenance, 

19 Execution, 

20 Finding, 

21 GateEvaluation, 

22 HumanDecision, 

23 PhaseExecution, 

24 Task, 

25 ValidationResult, 

26) 

27from governed_harness.orchestration.engine import EngineServices 

28from governed_harness.reporting.fingerprint import finding_fingerprint 

29from governed_harness.validators import TRACEABILITY_VALIDATOR_ID 

30 

31BRIEF_SCHEMA_VERSION = "1.0" 

32_MAX_DIFF_BYTES = 200_000 

33 

34 

35def location_text(finding: Finding) -> str: 

36 location = finding.location 

37 if location is None or not location.path: 

38 return "-" 

39 if location.start_line: 

40 return f"{location.path}:{location.start_line}" 

41 return location.path 

42 

43 

44def latest_validations(validations: list[ValidationResult], digest: str) -> list[ValidationResult]: 

45 """The latest attempt of each validator on ``digest`` (the gate's rule).""" 

46 latest: dict[str, ValidationResult] = {} 

47 for item in validations: 

48 if item.change_set_digest != digest: 

49 continue 

50 previous = latest.get(item.validator_id) 

51 if previous is None or item.finished_at >= previous.finished_at: 51 ↛ 47line 51 didn't jump to line 47 because the condition on line 51 was always true

52 latest[item.validator_id] = item 

53 return sorted(latest.values(), key=lambda item: item.validator_id) 

54 

55 

56def blocking_severities(services: EngineServices) -> set[FindingSeverity]: 

57 names = services.resolved.effective_policies.get("findingBlockSeverities", ["HIGH", "CRITICAL"]) 

58 return {FindingSeverity(str(name)) for name in names} 

59 

60 

61def _finding_entry(finding: Finding, blocking: set[FindingSeverity]) -> dict[str, Any]: 

62 return { 

63 "findingId": finding.finding_id, 

64 "severity": finding.severity.value, 

65 "ruleId": finding.rule_id, 

66 "validatorId": finding.validator_id, 

67 "location": location_text(finding), 

68 "message": finding.message, 

69 "blocking": finding.severity in blocking, 

70 "fingerprint": finding_fingerprint(finding), 

71 "recommendation": finding.recommendation, 

72 } 

73 

74 

75def _reason(code: str, findings: dict[str, Finding]) -> str: 

76 if code.startswith("BLOCKING_FINDING_"): 

77 finding = findings.get(code.removeprefix("BLOCKING_FINDING_")) 

78 if finding is None: 78 ↛ 79line 78 didn't jump to line 79 because the condition on line 78 was never true

79 return "a blocking finding" 

80 return ( 

81 f"{finding.severity.value} finding {finding.rule_id} at {location_text(finding)}: " 

82 f"{finding.message}" 

83 ) 

84 if code.startswith("EXCEPTION_APPLIED_"): 

85 return f"findings covered by exception {code.removeprefix('EXCEPTION_APPLIED_')}" 

86 if code == "ALL_MANDATORY_VALIDATIONS_PASSED": 

87 return "every mandatory validator passed and no finding blocks" 

88 if code == "NO_MANDATORY_VALIDATIONS": 

89 return "no mandatory validator ran: nothing establishes the change works" 

90 for status in ResultStatus: 

91 suffix = f"_{status.value}" 

92 if code.endswith(suffix): 92 ↛ 93line 92 didn't jump to line 93 because the condition on line 92 was never true

93 return f"mandatory validator {code.removesuffix(suffix)} is {status.value}" 

94 return code 

95 

96 

97def _traceability( 

98 services: EngineServices, validations: list[ValidationResult] 

99) -> dict[str, Any] | None: 

100 for validation in validations: 

101 if validation.validator_id != TRACEABILITY_VALIDATOR_ID: 

102 continue 

103 for ref in validation.evidence_refs: 

104 try: 

105 report = json.loads(services.artifacts.get(ref)) 

106 except (OSError, ValueError): 

107 continue 

108 if isinstance(report, dict) and "requirements" in report: 

109 return report 

110 return None 

111 

112 

113def _delta( 

114 services: EngineServices, 

115 decisions: list[HumanDecision], 

116 current: ChangeSet | None, 

117 current_findings: list[Finding], 

118 blocking: set[FindingSeverity], 

119) -> dict[str, Any] | None: 

120 if not decisions or current is None: 

121 return None 

122 last = decisions[-1] 

123 since = { 

124 "decisionId": last.decision_id, 

125 "decision": last.decision.value, 

126 "actorId": last.actor.actor_id, 

127 "decidedAt": last.decided_at.isoformat(), 

128 "changeSetDigest": last.change_set_digest, 

129 } 

130 if last.change_set_digest == current.digest: 

131 return {"sinceDecision": since, "unchanged": True} 

132 change_sets = services.state.list("change_set", ChangeSet, execution_id=current.execution_id) 

133 previous = next( 

134 (item for item in reversed(change_sets) if item.digest == last.change_set_digest), None 

135 ) 

136 before = {item.path: item for item in previous.files} if previous else {} 

137 after = {item.path: item for item in current.files} 

138 files = { 

139 "added": sorted(set(after) - set(before)), 

140 "removed": sorted(set(before) - set(after)), 

141 "changed": sorted( 

142 path 

143 for path in set(after) & set(before) 

144 if (after[path].after_digest, after[path].status) 

145 != (before[path].after_digest, before[path].status) 

146 ), 

147 } 

148 old_findings: list[Finding] = [] 

149 try: 

150 gate = services.state.get("gate", last.gate_evaluation_id, GateEvaluation) 

151 ids = { 

152 ref.removeprefix("record://finding/") 

153 for ref in gate.input_refs 

154 if ref.startswith("record://finding/") 

155 } 

156 old_findings = [ 

157 item 

158 for item in services.state.list("finding", Finding, execution_id=current.execution_id) 

159 if item.finding_id in ids 

160 ] 

161 except Exception: # noqa: BLE001 - a missing gate only removes the finding delta 

162 old_findings = [] 

163 old = {finding_fingerprint(item): item for item in old_findings} 

164 new = {finding_fingerprint(item): item for item in current_findings} 

165 return { 

166 "sinceDecision": since, 

167 "unchanged": False, 

168 "files": files, 

169 "findingsResolved": [ 

170 _finding_entry(old[key], blocking) for key in sorted(set(old) - set(new)) 

171 ], 

172 "findingsNew": [_finding_entry(new[key], blocking) for key in sorted(set(new) - set(old))], 

173 } 

174 

175 

176def _next(execution: Execution, awaiting: bool) -> list[str]: 

177 run_id = execution.execution_id 

178 if awaiting and execution.change_set_digest: 

179 return [ 

180 f"harness gate decide --run {run_id} (interactive on a terminal)", 

181 f"harness gate decide --run {run_id} --decision APPROVE " 

182 f"--change-set-digest {execution.change_set_digest} --rationale '...'", 

183 ] 

184 if execution.status is ResultStatus.PASSED: 

185 return [f"harness trace --run {run_id} --format markdown"] 

186 if execution.status is ResultStatus.CANCELLED: 186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true

187 return [] 

188 return [ 

189 f"harness run continue --run {run_id} (after fixing the cause)", 

190 f"harness run cancel --run {run_id}", 

191 ] 

192 

193 

194def _changed_section( 

195 services: EngineServices, digest: str | None, current: ChangeSet | None, include_diff: bool 

196) -> dict[str, Any]: 

197 """What changed: the files and totals of the current ChangeSet and, on request, its diff.""" 

198 changed: dict[str, Any] = {"changeSetDigest": digest, "files": [], "totals": None} 

199 if current is None: 

200 return changed 

201 changed["files"] = [ 

202 { 

203 "path": item.path, 

204 "status": item.status, 

205 "additions": item.additions, 

206 "deletions": item.deletions, 

207 } 

208 for item in current.files 

209 ] 

210 changed["totals"] = { 

211 "files": len(current.files), 

212 "additions": sum(item.additions for item in current.files), 

213 "deletions": sum(item.deletions for item in current.files), 

214 } 

215 changed["diffRef"] = current.diff_ref 

216 if include_diff: 

217 data = services.artifacts.get(current.diff_ref) 

218 changed["diff"] = data[:_MAX_DIFF_BYTES].decode("utf-8", "replace") 

219 changed["diffTruncated"] = len(data) > _MAX_DIFF_BYTES 

220 return changed 

221 

222 

223def _verified_section( 

224 digest: str | None, 

225 validations: list[ValidationResult], 

226 current_validations: list[ValidationResult], 

227 report: dict[str, Any] | None, 

228) -> dict[str, Any]: 

229 """The validations of the current ChangeSet and the traced requirements.""" 

230 return { 

231 "changeSetDigest": digest, 

232 "validations": [ 

233 { 

234 "validatorId": item.validator_id, 

235 "mandatory": item.mandatory, 

236 "status": item.status.value, 

237 "summary": item.summary, 

238 "attempts": sum( 

239 1 

240 for other in validations 

241 if other.validator_id == item.validator_id and other.change_set_digest == digest 

242 ), 

243 } 

244 for item in current_validations 

245 ], 

246 "requirements": [ 

247 { 

248 "requirementId": item["requirementId"], 

249 "traced": item["traced"], 

250 "tests": [test["nodeId"] for test in item.get("tests", [])], 

251 } 

252 for item in (report or {}).get("requirements", []) 

253 ], 

254 } 

255 

256 

257def _not_run_notes(current_validations: list[ValidationResult]) -> list[str]: 

258 """Validators that did not run, and optional validators that did not pass.""" 

259 notes: list[str] = [] 

260 for item in current_validations: 

261 if item.status is ResultStatus.PASSED: 

262 continue 

263 if item.status in {ResultStatus.NOT_APPLICABLE, ResultStatus.SKIPPED}: 

264 notes.append(f"{item.validator_id} did not run: {item.summary}") 

265 elif not item.mandatory: 

266 notes.append( 

267 f"optional {item.validator_id} is {item.status.value} (no effect on the gate): " 

268 f"{item.summary}" 

269 ) 

270 return notes 

271 

272 

273def _traceability_notes(report: dict[str, Any] | None, task: Task) -> list[str]: 

274 """Requirements no test names, requirements not checked, or no traceability at all.""" 

275 notes: list[str] = [] 

276 for item in (report or {}).get("requirements", []): 

277 if not item["traced"]: 

278 notes.append(f"requirement {item['requirementId']} has no test that names it") 

279 for requirement_id in (report or {}).get("skippedRequirementIds", []): 

280 notes.append(f"requirement {requirement_id} was not checked for a test") 

281 if report is None and task.requirements: 

282 notes.append( 

283 "requirement traceability did not run: no check that a test names each requirement" 

284 ) 

285 return notes 

286 

287 

288def _not_verified( 

289 current_validations: list[ValidationResult], report: dict[str, Any] | None, task: Task 

290) -> list[str]: 

291 """What the evidence of the current ChangeSet does not show.""" 

292 not_verified = _not_run_notes(current_validations) 

293 not_verified += _traceability_notes(report, task) 

294 if task.acceptance_criteria: 294 ↛ 299line 294 didn't jump to line 299 because the condition on line 294 was always true

295 not_verified.append( 

296 "acceptance criteria are not executed one by one: the evidence is the validators " 

297 "and findings above" 

298 ) 

299 if any(item.validator_id == "review.independent" for item in current_validations): 

300 not_verified.append( 

301 "INDEPENDENT_REVIEW is a deterministic rule check, not a review by another person" 

302 ) 

303 return not_verified 

304 

305 

306def _passed_after_retry( 

307 validations: list[ValidationResult], 

308 current_validations: list[ValidationResult], 

309 digest: str | None, 

310) -> list[str]: 

311 """Validators that pass on the current ChangeSet after not passing on it before.""" 

312 return sorted( 

313 item.validator_id 

314 for item in current_validations 

315 if item.status is ResultStatus.PASSED 

316 and any( 

317 other.validator_id == item.validator_id 

318 and other.change_set_digest == digest 

319 and other.status is not ResultStatus.PASSED 

320 for other in validations 

321 ) 

322 ) 

323 

324 

325def _history_section( 

326 phases: list[PhaseExecution], 

327 corrections: list[Any], 

328 events: list[Any], 

329 passed_after_retry: list[str], 

330 superseded: int, 

331) -> dict[str, Any]: 

332 phase_counts = Counter(item.phase_id for item in phases) 

333 return { 

334 "implementationAttempts": phase_counts.get(PhaseId.IMPLEMENTATION, 0), 

335 "verificationAttempts": phase_counts.get(PhaseId.VERIFICATION, 0), 

336 "automaticCorrections": sum( 

337 1 for item in corrections if item.payload.get("trigger") == "VERIFICATION_FAILED" 

338 ), 

339 "requestedChanges": sum( 

340 1 

341 for item in corrections 

342 if item.payload.get("trigger") not in {"VERIFICATION_FAILED", "REVIEW_FINDINGS"} 

343 ), 

344 "providerRetries": sum( 

345 1 for item in events if item.event_type == "agent.invocation.retried" 

346 ), 

347 "passedAfterRetry": passed_after_retry, 

348 "supersededFindings": superseded, 

349 } 

350 

351 

352def _gate_section(gate: GateEvaluation | None, by_id: dict[str, Finding]) -> dict[str, Any] | None: 

353 if not gate: 

354 return None 

355 return { 

356 "gateEvaluationId": gate.gate_evaluation_id, 

357 "status": gate.status.value, 

358 "changeSetDigest": gate.change_set_digest, 

359 "reasons": [ 

360 {"code": code, "explanation": _reason(code, by_id)} for code in gate.reason_codes 

361 ], 

362 } 

363 

364 

365def _decision_entry(item: HumanDecision) -> dict[str, Any]: 

366 return { 

367 "decision": item.decision.value, 

368 "actorId": item.actor.actor_id, 

369 "decidedAt": item.decided_at.isoformat(), 

370 "changeSetDigest": item.change_set_digest, 

371 "rationale": item.rationale, 

372 "expiresAt": item.expires_at.isoformat() if item.expires_at else None, 

373 } 

374 

375 

376def _asked_section(task: Task) -> dict[str, Any]: 

377 return { 

378 "title": task.title, 

379 "intent": task.intent, 

380 "requirements": [ 

381 {"requirementId": item.requirement_id, "text": item.text} for item in task.requirements 

382 ], 

383 "acceptanceCriteria": [ 

384 {"criterionId": item.criterion_id, "text": item.text, "priority": item.priority} 

385 for item in task.acceptance_criteria 

386 ], 

387 "constraints": list(task.constraints), 

388 **_assumptions_section(task), 

389 } 

390 

391 

392def _run_section(execution: Execution, awaiting: bool) -> dict[str, Any]: 

393 return { 

394 "executionId": execution.execution_id, 

395 "taskId": execution.task_id, 

396 "status": execution.status.value, 

397 "currentPhase": execution.current_phase.value, 

398 "awaitingDecision": awaiting, 

399 "changeSetDigest": execution.change_set_digest, 

400 "baselineRevision": execution.baseline_revision, 

401 "updatedAt": execution.updated_at.isoformat(), 

402 } 

403 

404 

405def _risk_entries( 

406 current_findings: list[Finding], blocking: set[FindingSeverity] 

407) -> list[dict[str, Any]]: 

408 """The findings of the current ChangeSet: blocking first, then by severity and location.""" 

409 return sorted( 

410 (_finding_entry(item, blocking) for item in current_findings), 

411 key=lambda item: ( 

412 not item["blocking"], 

413 -list(FindingSeverity).index(FindingSeverity(item["severity"])), 

414 item["location"], 

415 ), 

416 ) 

417 

418 

419def build_brief( 

420 services: EngineServices, 

421 execution_id: str, 

422 *, 

423 include_diff: bool = False, 

424 exceptions: list[dict[str, Any]] | None = None, 

425) -> dict[str, Any]: 

426 state = services.state 

427 execution = state.get("execution", execution_id, Execution) 

428 task = state.get("task", execution.task_id, Task) 

429 blocking = blocking_severities(services) 

430 digest = execution.change_set_digest 

431 change_sets = state.list("change_set", ChangeSet, execution_id=execution_id) 

432 current = next((item for item in reversed(change_sets) if item.digest == digest), None) 

433 validations = state.list("validation", ValidationResult, execution_id=execution_id) 

434 findings = state.list("finding", Finding, execution_id=execution_id) 

435 by_id = {item.finding_id: item for item in findings} 

436 current_validations = latest_validations(validations, digest) if digest else [] 

437 current_ids = {fid for item in current_validations for fid in item.finding_ids} 

438 current_findings = [by_id[fid] for fid in sorted(current_ids) if fid in by_id] 

439 gate = ( 

440 state.get("gate", execution.gate_evaluation_id, GateEvaluation) 

441 if execution.gate_evaluation_id 

442 else None 

443 ) 

444 decisions = sorted( 

445 state.list("decision", HumanDecision, execution_id=execution_id), 

446 key=lambda item: item.decided_at, 

447 ) 

448 phases = state.list("phase", PhaseExecution, execution_id=execution_id) 

449 events = services.events.list(execution_id) 

450 awaiting = execution.current_phase is PhaseId.DECISION and execution.status in { 

451 ResultStatus.BLOCKED, 

452 ResultStatus.PENDING, 

453 } 

454 

455 changed = _changed_section(services, digest, current, include_diff) 

456 report = _traceability(services, current_validations) 

457 verified = _verified_section(digest, validations, current_validations, report) 

458 not_verified = _not_verified(current_validations, report, task) 

459 

460 corrections = [item for item in events if item.event_type == "correction.authorized"] 

461 history = _history_section( 

462 phases, 

463 corrections, 

464 events, 

465 _passed_after_retry(validations, current_validations, digest), 

466 len(findings) - len(current_findings), 

467 ) 

468 

469 risks = _risk_entries(current_findings, blocking) 

470 review_corrections = sum( 

471 1 for item in corrections if item.payload.get("trigger") == "REVIEW_FINDINGS" 

472 ) 

473 if review_corrections: 473 ↛ 474line 473 didn't jump to line 474 because the condition on line 473 was never true

474 history["reviewCorrections"] = review_corrections 

475 if any(item.validator_id == "review.agent" for item in current_validations): 475 ↛ 476line 475 didn't jump to line 476 because the condition on line 475 was never true

476 not_verified.append( 

477 "a second agent reviewed the ChangeSet (review.agentReview); it is not a person" 

478 ) 

479 not_verified.extend(_assumption_notes(task)) 

480 agent_results = _agent_results_section(services, execution, current_findings) 

481 brief: dict[str, Any] = { 

482 "schemaVersion": BRIEF_SCHEMA_VERSION, 

483 "run": _run_section(execution, awaiting), 

484 "asked": _asked_section(task), 

485 "changed": changed, 

486 "gate": _gate_section(gate, by_id), 

487 "risks": risks, 

488 "verified": verified, 

489 "notVerified": not_verified, 

490 "exceptions": exceptions or [], 

491 "history": history, 

492 "delta": _delta(services, decisions, current, current_findings, blocking), 

493 "decisions": [_decision_entry(item) for item in decisions], 

494 "next": _next(execution, awaiting), 

495 **_provenance_sections(services, execution_id, digest), 

496 } 

497 if agent_results: 

498 brief["riskFactors"] = agent_results 

499 # Since #55: certification, preflight, deferred items, checklist, contract, interruptions. 

500 from governed_harness.application.ladder import brief_sections, not_verified_lines 

501 

502 ladder = brief_sections(services, execution) 

503 if ladder: 

504 brief.update(ladder) 

505 brief["notVerified"] = [*brief["notVerified"], *not_verified_lines(ladder)] 

506 return brief 

507 

508 

509def _assumptions_section(task: Task) -> dict[str, Any]: 

510 """``assumptions`` of what was asked (#79): the points the agent review left open after its 

511 last round; absent when the task records none, so other briefs keep their form.""" 

512 from governed_harness.orchestration.intent_convergence import assumptions_of 

513 

514 assumptions = assumptions_of(task) 

515 return {"assumptions": assumptions} if assumptions else {} 

516 

517 

518def _assumption_notes(task: Task) -> list[str]: 

519 from governed_harness.orchestration.intent_convergence import assumptions_of 

520 

521 return [ 

522 f"assumption {item.get('assumptionId')} was not answered by a person: " 

523 f"{item.get('question')}" 

524 for item in assumptions_of(task) 

525 ] 

526 

527 

528def _agent_results_section( 

529 services: EngineServices, execution: Execution, findings: list[Finding] 

530) -> dict[str, Any] | None: 

531 """Risk factors of the ChangeSet (``verification.riskFactors``, #52): what blocks, what the 

532 decider must acknowledge (``--acknowledge-risk``) and what only informs.""" 

533 signals = [item for item in findings if item.category == "risk"] 

534 raw = services.state.get_flag(f"riskack:{execution.execution_id}:{execution.change_set_digest}") 

535 required = json.loads(raw) if raw else [] 

536 if not signals and not required: 

537 return None 

538 return { 

539 "acknowledgementRequired": required, 

540 "signals": [ 

541 { 

542 "factor": item.rule_id.removeprefix("risk."), 

543 "severity": item.severity.value, 

544 "message": item.message, 

545 "location": item.location.path if item.location else None, 

546 "line": item.location.start_line if item.location else None, 

547 } 

548 for item in signals 

549 ], 

550 } 

551 

552 

553def _provenance_sections( 

554 services: EngineServices, execution_id: str, digest: str | None 

555) -> dict[str, Any]: 

556 """``provenance`` and ``selfReports`` (``provenance`` settings, since 2.0); absent when the 

557 run recorded neither, so the brief of other runs keeps its form.""" 

558 sections: dict[str, Any] = {} 

559 records = [ 

560 item 

561 for item in services.state.list( 

562 "component_provenance", ComponentProvenance, execution_id=execution_id 

563 ) 

564 if item.change_set_digest == digest 

565 ] 

566 if records: 

567 latest = records[-1] 

568 out_of_band = [item.path for item in latest.files if item.source == "OUT_OF_BAND"] 

569 sections["provenance"] = { 

570 "changeSetDigest": latest.change_set_digest, 

571 "agentFiles": len(latest.files) - len(out_of_band), 

572 "outOfBandFiles": len(out_of_band), 

573 "outOfBandPaths": out_of_band, 

574 "files": [item.model_dump(mode="json", by_alias=True) for item in latest.files], 

575 } 

576 reports = services.state.list("agent_self_report", AgentSelfReport, execution_id=execution_id) 

577 if reports: 

578 sections["selfReports"] = [ 

579 { 

580 "invocationId": item.invocation_id, 

581 "quality": item.quality.value, 

582 "assumptions": list(item.assumptions), 

583 "alternativesDiscarded": list(item.alternatives_discarded), 

584 "lowConfidenceAreas": [ 

585 entry.model_dump(mode="json", by_alias=True) 

586 for entry in item.low_confidence_areas 

587 ], 

588 "unrequestedChanges": [ 

589 entry.model_dump(mode="json", by_alias=True) 

590 for entry in item.unrequested_changes 

591 ], 

592 "problems": list(item.problems), 

593 "contrast": item.contrast.model_dump(mode="json", by_alias=True) 

594 if item.contrast 

595 else None, 

596 } 

597 for item in reports[-3:] 

598 ] 

599 return sections