Coverage for src/governed_harness/application/review.py: 95%
218 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""The decision brief: everything a person needs to decide on one ChangeSet digest.
3Built only from recorded data by fixed rules (no generated text): what was asked, what changed,
4the risks with their location, what was verified on which digest and what was not, active
5exceptions, retries and corrections, and what changed since the last decision. It reads the
6record and never writes to it."""
8from __future__ import annotations
10import json
11from collections import Counter
12from typing import Any
14from governed_harness.domain.enums import FindingSeverity, PhaseId, ResultStatus
15from governed_harness.domain.models import (
16 AgentSelfReport,
17 ChangeSet,
18 ComponentProvenance,
19 Execution,
20 Finding,
21 GateEvaluation,
22 HumanDecision,
23 PhaseExecution,
24 Task,
25 ValidationResult,
26)
27from governed_harness.orchestration.engine import EngineServices
28from governed_harness.reporting.fingerprint import finding_fingerprint
29from governed_harness.validators import TRACEABILITY_VALIDATOR_ID
31BRIEF_SCHEMA_VERSION = "1.0"
32_MAX_DIFF_BYTES = 200_000
35def location_text(finding: Finding) -> str:
36 location = finding.location
37 if location is None or not location.path:
38 return "-"
39 if location.start_line:
40 return f"{location.path}:{location.start_line}"
41 return location.path
44def latest_validations(validations: list[ValidationResult], digest: str) -> list[ValidationResult]:
45 """The latest attempt of each validator on ``digest`` (the gate's rule)."""
46 latest: dict[str, ValidationResult] = {}
47 for item in validations:
48 if item.change_set_digest != digest:
49 continue
50 previous = latest.get(item.validator_id)
51 if previous is None or item.finished_at >= previous.finished_at: 51 ↛ 47line 51 didn't jump to line 47 because the condition on line 51 was always true
52 latest[item.validator_id] = item
53 return sorted(latest.values(), key=lambda item: item.validator_id)
56def blocking_severities(services: EngineServices) -> set[FindingSeverity]:
57 names = services.resolved.effective_policies.get("findingBlockSeverities", ["HIGH", "CRITICAL"])
58 return {FindingSeverity(str(name)) for name in names}
61def _finding_entry(finding: Finding, blocking: set[FindingSeverity]) -> dict[str, Any]:
62 return {
63 "findingId": finding.finding_id,
64 "severity": finding.severity.value,
65 "ruleId": finding.rule_id,
66 "validatorId": finding.validator_id,
67 "location": location_text(finding),
68 "message": finding.message,
69 "blocking": finding.severity in blocking,
70 "fingerprint": finding_fingerprint(finding),
71 "recommendation": finding.recommendation,
72 }
75def _reason(code: str, findings: dict[str, Finding]) -> str:
76 if code.startswith("BLOCKING_FINDING_"):
77 finding = findings.get(code.removeprefix("BLOCKING_FINDING_"))
78 if finding is None: 78 ↛ 79line 78 didn't jump to line 79 because the condition on line 78 was never true
79 return "a blocking finding"
80 return (
81 f"{finding.severity.value} finding {finding.rule_id} at {location_text(finding)}: "
82 f"{finding.message}"
83 )
84 if code.startswith("EXCEPTION_APPLIED_"):
85 return f"findings covered by exception {code.removeprefix('EXCEPTION_APPLIED_')}"
86 if code == "ALL_MANDATORY_VALIDATIONS_PASSED":
87 return "every mandatory validator passed and no finding blocks"
88 if code == "NO_MANDATORY_VALIDATIONS":
89 return "no mandatory validator ran: nothing establishes the change works"
90 for status in ResultStatus:
91 suffix = f"_{status.value}"
92 if code.endswith(suffix): 92 ↛ 93line 92 didn't jump to line 93 because the condition on line 92 was never true
93 return f"mandatory validator {code.removesuffix(suffix)} is {status.value}"
94 return code
97def _traceability(
98 services: EngineServices, validations: list[ValidationResult]
99) -> dict[str, Any] | None:
100 for validation in validations:
101 if validation.validator_id != TRACEABILITY_VALIDATOR_ID:
102 continue
103 for ref in validation.evidence_refs:
104 try:
105 report = json.loads(services.artifacts.get(ref))
106 except (OSError, ValueError):
107 continue
108 if isinstance(report, dict) and "requirements" in report:
109 return report
110 return None
113def _delta(
114 services: EngineServices,
115 decisions: list[HumanDecision],
116 current: ChangeSet | None,
117 current_findings: list[Finding],
118 blocking: set[FindingSeverity],
119) -> dict[str, Any] | None:
120 if not decisions or current is None:
121 return None
122 last = decisions[-1]
123 since = {
124 "decisionId": last.decision_id,
125 "decision": last.decision.value,
126 "actorId": last.actor.actor_id,
127 "decidedAt": last.decided_at.isoformat(),
128 "changeSetDigest": last.change_set_digest,
129 }
130 if last.change_set_digest == current.digest:
131 return {"sinceDecision": since, "unchanged": True}
132 change_sets = services.state.list("change_set", ChangeSet, execution_id=current.execution_id)
133 previous = next(
134 (item for item in reversed(change_sets) if item.digest == last.change_set_digest), None
135 )
136 before = {item.path: item for item in previous.files} if previous else {}
137 after = {item.path: item for item in current.files}
138 files = {
139 "added": sorted(set(after) - set(before)),
140 "removed": sorted(set(before) - set(after)),
141 "changed": sorted(
142 path
143 for path in set(after) & set(before)
144 if (after[path].after_digest, after[path].status)
145 != (before[path].after_digest, before[path].status)
146 ),
147 }
148 old_findings: list[Finding] = []
149 try:
150 gate = services.state.get("gate", last.gate_evaluation_id, GateEvaluation)
151 ids = {
152 ref.removeprefix("record://finding/")
153 for ref in gate.input_refs
154 if ref.startswith("record://finding/")
155 }
156 old_findings = [
157 item
158 for item in services.state.list("finding", Finding, execution_id=current.execution_id)
159 if item.finding_id in ids
160 ]
161 except Exception: # noqa: BLE001 - a missing gate only removes the finding delta
162 old_findings = []
163 old = {finding_fingerprint(item): item for item in old_findings}
164 new = {finding_fingerprint(item): item for item in current_findings}
165 return {
166 "sinceDecision": since,
167 "unchanged": False,
168 "files": files,
169 "findingsResolved": [
170 _finding_entry(old[key], blocking) for key in sorted(set(old) - set(new))
171 ],
172 "findingsNew": [_finding_entry(new[key], blocking) for key in sorted(set(new) - set(old))],
173 }
176def _next(execution: Execution, awaiting: bool) -> list[str]:
177 run_id = execution.execution_id
178 if awaiting and execution.change_set_digest:
179 return [
180 f"harness gate decide --run {run_id} (interactive on a terminal)",
181 f"harness gate decide --run {run_id} --decision APPROVE "
182 f"--change-set-digest {execution.change_set_digest} --rationale '...'",
183 ]
184 if execution.status is ResultStatus.PASSED:
185 return [f"harness trace --run {run_id} --format markdown"]
186 if execution.status is ResultStatus.CANCELLED: 186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true
187 return []
188 return [
189 f"harness run continue --run {run_id} (after fixing the cause)",
190 f"harness run cancel --run {run_id}",
191 ]
194def _changed_section(
195 services: EngineServices, digest: str | None, current: ChangeSet | None, include_diff: bool
196) -> dict[str, Any]:
197 """What changed: the files and totals of the current ChangeSet and, on request, its diff."""
198 changed: dict[str, Any] = {"changeSetDigest": digest, "files": [], "totals": None}
199 if current is None:
200 return changed
201 changed["files"] = [
202 {
203 "path": item.path,
204 "status": item.status,
205 "additions": item.additions,
206 "deletions": item.deletions,
207 }
208 for item in current.files
209 ]
210 changed["totals"] = {
211 "files": len(current.files),
212 "additions": sum(item.additions for item in current.files),
213 "deletions": sum(item.deletions for item in current.files),
214 }
215 changed["diffRef"] = current.diff_ref
216 if include_diff:
217 data = services.artifacts.get(current.diff_ref)
218 changed["diff"] = data[:_MAX_DIFF_BYTES].decode("utf-8", "replace")
219 changed["diffTruncated"] = len(data) > _MAX_DIFF_BYTES
220 return changed
223def _verified_section(
224 digest: str | None,
225 validations: list[ValidationResult],
226 current_validations: list[ValidationResult],
227 report: dict[str, Any] | None,
228) -> dict[str, Any]:
229 """The validations of the current ChangeSet and the traced requirements."""
230 return {
231 "changeSetDigest": digest,
232 "validations": [
233 {
234 "validatorId": item.validator_id,
235 "mandatory": item.mandatory,
236 "status": item.status.value,
237 "summary": item.summary,
238 "attempts": sum(
239 1
240 for other in validations
241 if other.validator_id == item.validator_id and other.change_set_digest == digest
242 ),
243 }
244 for item in current_validations
245 ],
246 "requirements": [
247 {
248 "requirementId": item["requirementId"],
249 "traced": item["traced"],
250 "tests": [test["nodeId"] for test in item.get("tests", [])],
251 }
252 for item in (report or {}).get("requirements", [])
253 ],
254 }
257def _not_run_notes(current_validations: list[ValidationResult]) -> list[str]:
258 """Validators that did not run, and optional validators that did not pass."""
259 notes: list[str] = []
260 for item in current_validations:
261 if item.status is ResultStatus.PASSED:
262 continue
263 if item.status in {ResultStatus.NOT_APPLICABLE, ResultStatus.SKIPPED}:
264 notes.append(f"{item.validator_id} did not run: {item.summary}")
265 elif not item.mandatory:
266 notes.append(
267 f"optional {item.validator_id} is {item.status.value} (no effect on the gate): "
268 f"{item.summary}"
269 )
270 return notes
273def _traceability_notes(report: dict[str, Any] | None, task: Task) -> list[str]:
274 """Requirements no test names, requirements not checked, or no traceability at all."""
275 notes: list[str] = []
276 for item in (report or {}).get("requirements", []):
277 if not item["traced"]:
278 notes.append(f"requirement {item['requirementId']} has no test that names it")
279 for requirement_id in (report or {}).get("skippedRequirementIds", []):
280 notes.append(f"requirement {requirement_id} was not checked for a test")
281 if report is None and task.requirements:
282 notes.append(
283 "requirement traceability did not run: no check that a test names each requirement"
284 )
285 return notes
288def _not_verified(
289 current_validations: list[ValidationResult], report: dict[str, Any] | None, task: Task
290) -> list[str]:
291 """What the evidence of the current ChangeSet does not show."""
292 not_verified = _not_run_notes(current_validations)
293 not_verified += _traceability_notes(report, task)
294 if task.acceptance_criteria: 294 ↛ 299line 294 didn't jump to line 299 because the condition on line 294 was always true
295 not_verified.append(
296 "acceptance criteria are not executed one by one: the evidence is the validators "
297 "and findings above"
298 )
299 if any(item.validator_id == "review.independent" for item in current_validations):
300 not_verified.append(
301 "INDEPENDENT_REVIEW is a deterministic rule check, not a review by another person"
302 )
303 return not_verified
306def _passed_after_retry(
307 validations: list[ValidationResult],
308 current_validations: list[ValidationResult],
309 digest: str | None,
310) -> list[str]:
311 """Validators that pass on the current ChangeSet after not passing on it before."""
312 return sorted(
313 item.validator_id
314 for item in current_validations
315 if item.status is ResultStatus.PASSED
316 and any(
317 other.validator_id == item.validator_id
318 and other.change_set_digest == digest
319 and other.status is not ResultStatus.PASSED
320 for other in validations
321 )
322 )
325def _history_section(
326 phases: list[PhaseExecution],
327 corrections: list[Any],
328 events: list[Any],
329 passed_after_retry: list[str],
330 superseded: int,
331) -> dict[str, Any]:
332 phase_counts = Counter(item.phase_id for item in phases)
333 return {
334 "implementationAttempts": phase_counts.get(PhaseId.IMPLEMENTATION, 0),
335 "verificationAttempts": phase_counts.get(PhaseId.VERIFICATION, 0),
336 "automaticCorrections": sum(
337 1 for item in corrections if item.payload.get("trigger") == "VERIFICATION_FAILED"
338 ),
339 "requestedChanges": sum(
340 1
341 for item in corrections
342 if item.payload.get("trigger") not in {"VERIFICATION_FAILED", "REVIEW_FINDINGS"}
343 ),
344 "providerRetries": sum(
345 1 for item in events if item.event_type == "agent.invocation.retried"
346 ),
347 "passedAfterRetry": passed_after_retry,
348 "supersededFindings": superseded,
349 }
352def _gate_section(gate: GateEvaluation | None, by_id: dict[str, Finding]) -> dict[str, Any] | None:
353 if not gate:
354 return None
355 return {
356 "gateEvaluationId": gate.gate_evaluation_id,
357 "status": gate.status.value,
358 "changeSetDigest": gate.change_set_digest,
359 "reasons": [
360 {"code": code, "explanation": _reason(code, by_id)} for code in gate.reason_codes
361 ],
362 }
365def _decision_entry(item: HumanDecision) -> dict[str, Any]:
366 return {
367 "decision": item.decision.value,
368 "actorId": item.actor.actor_id,
369 "decidedAt": item.decided_at.isoformat(),
370 "changeSetDigest": item.change_set_digest,
371 "rationale": item.rationale,
372 "expiresAt": item.expires_at.isoformat() if item.expires_at else None,
373 }
376def _asked_section(task: Task) -> dict[str, Any]:
377 return {
378 "title": task.title,
379 "intent": task.intent,
380 "requirements": [
381 {"requirementId": item.requirement_id, "text": item.text} for item in task.requirements
382 ],
383 "acceptanceCriteria": [
384 {"criterionId": item.criterion_id, "text": item.text, "priority": item.priority}
385 for item in task.acceptance_criteria
386 ],
387 "constraints": list(task.constraints),
388 **_assumptions_section(task),
389 }
392def _run_section(execution: Execution, awaiting: bool) -> dict[str, Any]:
393 return {
394 "executionId": execution.execution_id,
395 "taskId": execution.task_id,
396 "status": execution.status.value,
397 "currentPhase": execution.current_phase.value,
398 "awaitingDecision": awaiting,
399 "changeSetDigest": execution.change_set_digest,
400 "baselineRevision": execution.baseline_revision,
401 "updatedAt": execution.updated_at.isoformat(),
402 }
405def _risk_entries(
406 current_findings: list[Finding], blocking: set[FindingSeverity]
407) -> list[dict[str, Any]]:
408 """The findings of the current ChangeSet: blocking first, then by severity and location."""
409 return sorted(
410 (_finding_entry(item, blocking) for item in current_findings),
411 key=lambda item: (
412 not item["blocking"],
413 -list(FindingSeverity).index(FindingSeverity(item["severity"])),
414 item["location"],
415 ),
416 )
419def build_brief(
420 services: EngineServices,
421 execution_id: str,
422 *,
423 include_diff: bool = False,
424 exceptions: list[dict[str, Any]] | None = None,
425) -> dict[str, Any]:
426 state = services.state
427 execution = state.get("execution", execution_id, Execution)
428 task = state.get("task", execution.task_id, Task)
429 blocking = blocking_severities(services)
430 digest = execution.change_set_digest
431 change_sets = state.list("change_set", ChangeSet, execution_id=execution_id)
432 current = next((item for item in reversed(change_sets) if item.digest == digest), None)
433 validations = state.list("validation", ValidationResult, execution_id=execution_id)
434 findings = state.list("finding", Finding, execution_id=execution_id)
435 by_id = {item.finding_id: item for item in findings}
436 current_validations = latest_validations(validations, digest) if digest else []
437 current_ids = {fid for item in current_validations for fid in item.finding_ids}
438 current_findings = [by_id[fid] for fid in sorted(current_ids) if fid in by_id]
439 gate = (
440 state.get("gate", execution.gate_evaluation_id, GateEvaluation)
441 if execution.gate_evaluation_id
442 else None
443 )
444 decisions = sorted(
445 state.list("decision", HumanDecision, execution_id=execution_id),
446 key=lambda item: item.decided_at,
447 )
448 phases = state.list("phase", PhaseExecution, execution_id=execution_id)
449 events = services.events.list(execution_id)
450 awaiting = execution.current_phase is PhaseId.DECISION and execution.status in {
451 ResultStatus.BLOCKED,
452 ResultStatus.PENDING,
453 }
455 changed = _changed_section(services, digest, current, include_diff)
456 report = _traceability(services, current_validations)
457 verified = _verified_section(digest, validations, current_validations, report)
458 not_verified = _not_verified(current_validations, report, task)
460 corrections = [item for item in events if item.event_type == "correction.authorized"]
461 history = _history_section(
462 phases,
463 corrections,
464 events,
465 _passed_after_retry(validations, current_validations, digest),
466 len(findings) - len(current_findings),
467 )
469 risks = _risk_entries(current_findings, blocking)
470 review_corrections = sum(
471 1 for item in corrections if item.payload.get("trigger") == "REVIEW_FINDINGS"
472 )
473 if review_corrections: 473 ↛ 474line 473 didn't jump to line 474 because the condition on line 473 was never true
474 history["reviewCorrections"] = review_corrections
475 if any(item.validator_id == "review.agent" for item in current_validations): 475 ↛ 476line 475 didn't jump to line 476 because the condition on line 475 was never true
476 not_verified.append(
477 "a second agent reviewed the ChangeSet (review.agentReview); it is not a person"
478 )
479 not_verified.extend(_assumption_notes(task))
480 agent_results = _agent_results_section(services, execution, current_findings)
481 brief: dict[str, Any] = {
482 "schemaVersion": BRIEF_SCHEMA_VERSION,
483 "run": _run_section(execution, awaiting),
484 "asked": _asked_section(task),
485 "changed": changed,
486 "gate": _gate_section(gate, by_id),
487 "risks": risks,
488 "verified": verified,
489 "notVerified": not_verified,
490 "exceptions": exceptions or [],
491 "history": history,
492 "delta": _delta(services, decisions, current, current_findings, blocking),
493 "decisions": [_decision_entry(item) for item in decisions],
494 "next": _next(execution, awaiting),
495 **_provenance_sections(services, execution_id, digest),
496 }
497 if agent_results:
498 brief["riskFactors"] = agent_results
499 # Since #55: certification, preflight, deferred items, checklist, contract, interruptions.
500 from governed_harness.application.ladder import brief_sections, not_verified_lines
502 ladder = brief_sections(services, execution)
503 if ladder:
504 brief.update(ladder)
505 brief["notVerified"] = [*brief["notVerified"], *not_verified_lines(ladder)]
506 return brief
509def _assumptions_section(task: Task) -> dict[str, Any]:
510 """``assumptions`` of what was asked (#79): the points the agent review left open after its
511 last round; absent when the task records none, so other briefs keep their form."""
512 from governed_harness.orchestration.intent_convergence import assumptions_of
514 assumptions = assumptions_of(task)
515 return {"assumptions": assumptions} if assumptions else {}
518def _assumption_notes(task: Task) -> list[str]:
519 from governed_harness.orchestration.intent_convergence import assumptions_of
521 return [
522 f"assumption {item.get('assumptionId')} was not answered by a person: "
523 f"{item.get('question')}"
524 for item in assumptions_of(task)
525 ]
528def _agent_results_section(
529 services: EngineServices, execution: Execution, findings: list[Finding]
530) -> dict[str, Any] | None:
531 """Risk factors of the ChangeSet (``verification.riskFactors``, #52): what blocks, what the
532 decider must acknowledge (``--acknowledge-risk``) and what only informs."""
533 signals = [item for item in findings if item.category == "risk"]
534 raw = services.state.get_flag(f"riskack:{execution.execution_id}:{execution.change_set_digest}")
535 required = json.loads(raw) if raw else []
536 if not signals and not required:
537 return None
538 return {
539 "acknowledgementRequired": required,
540 "signals": [
541 {
542 "factor": item.rule_id.removeprefix("risk."),
543 "severity": item.severity.value,
544 "message": item.message,
545 "location": item.location.path if item.location else None,
546 "line": item.location.start_line if item.location else None,
547 }
548 for item in signals
549 ],
550 }
553def _provenance_sections(
554 services: EngineServices, execution_id: str, digest: str | None
555) -> dict[str, Any]:
556 """``provenance`` and ``selfReports`` (``provenance`` settings, since 2.0); absent when the
557 run recorded neither, so the brief of other runs keeps its form."""
558 sections: dict[str, Any] = {}
559 records = [
560 item
561 for item in services.state.list(
562 "component_provenance", ComponentProvenance, execution_id=execution_id
563 )
564 if item.change_set_digest == digest
565 ]
566 if records:
567 latest = records[-1]
568 out_of_band = [item.path for item in latest.files if item.source == "OUT_OF_BAND"]
569 sections["provenance"] = {
570 "changeSetDigest": latest.change_set_digest,
571 "agentFiles": len(latest.files) - len(out_of_band),
572 "outOfBandFiles": len(out_of_band),
573 "outOfBandPaths": out_of_band,
574 "files": [item.model_dump(mode="json", by_alias=True) for item in latest.files],
575 }
576 reports = services.state.list("agent_self_report", AgentSelfReport, execution_id=execution_id)
577 if reports:
578 sections["selfReports"] = [
579 {
580 "invocationId": item.invocation_id,
581 "quality": item.quality.value,
582 "assumptions": list(item.assumptions),
583 "alternativesDiscarded": list(item.alternatives_discarded),
584 "lowConfidenceAreas": [
585 entry.model_dump(mode="json", by_alias=True)
586 for entry in item.low_confidence_areas
587 ],
588 "unrequestedChanges": [
589 entry.model_dump(mode="json", by_alias=True)
590 for entry in item.unrequested_changes
591 ],
592 "problems": list(item.problems),
593 "contrast": item.contrast.model_dump(mode="json", by_alias=True)
594 if item.contrast
595 else None,
596 }
597 for item in reports[-3:]
598 ]
599 return sections