Coverage for src/governed_harness/retrospective/causes.py: 95%
96 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Causes of a run's stops and corrections, by reason code (``retrospective.causal``).
3The 1.0.0 retrospective counted every non-passed validation, including optional validators that
4had no effect on the gate, and so attributed cycles to the wrong cause. Here a cause is only
5what stopped VERIFICATION (a mandatory validator that did not pass), what failed a delivery gate
6(a blocking finding), a decision that sent the run back or ended it, an exception, a transient
7provider failure or an outcome recorded after the run. Subjects are validators, rules and
8providers, never people (no per-person metric)."""
10from __future__ import annotations
12from collections import defaultdict
13from dataclasses import dataclass, field
15from governed_harness.domain.enums import DecisionKind, FindingSeverity, PhaseId, ResultStatus
16from governed_harness.domain.models import (
17 ExceptionRecord,
18 Execution,
19 Finding,
20 GateEvaluation,
21 HumanDecision,
22 OutcomeRecord,
23 RetrospectiveCause,
24 ValidationResult,
25)
26from governed_harness.events import StoredEvent
27from governed_harness.storage import SQLiteStateStore
29STOPPING = {
30 ResultStatus.FAILED,
31 ResultStatus.BLOCKED,
32 ResultStatus.ERROR,
33 ResultStatus.TIMED_OUT,
34 ResultStatus.INCONCLUSIVE,
35}
38@dataclass
39class CauseAnalysis:
40 causes: list[RetrospectiveCause] = field(default_factory=list)
41 notes: list[str] = field(default_factory=list)
44def _stopping_validations(
45 validations: list[ValidationResult], attempt_of: dict[str, int]
46) -> tuple[dict[tuple[str, str], list[int]], dict[tuple[str, str], list[str]], set[str]]:
47 """Mandatory validators that stopped VERIFICATION, by (validator, status), with their
48 attempts and evidence; and the optional validators that did not pass."""
49 mandatory: dict[tuple[str, str], list[int]] = defaultdict(list)
50 refs: dict[tuple[str, str], list[str]] = defaultdict(list)
51 optional_failures: set[str] = set()
52 for item in validations:
53 if item.status not in STOPPING:
54 continue
55 if not item.mandatory:
56 optional_failures.add(item.validator_id)
57 continue
58 key = (item.validator_id, item.status.value)
59 attempt = attempt_of[item.change_set_digest]
60 if attempt not in mandatory[key]: 60 ↛ 62line 60 didn't jump to line 62 because the condition on line 60 was always true
61 mandatory[key].append(attempt)
62 refs[key].append(f"record://validation/{item.validation_result_id}")
63 return mandatory, refs, optional_failures
66def _add_validator_causes(
67 analysis: CauseAnalysis,
68 state: SQLiteStateStore,
69 events: list[StoredEvent],
70 run_id: str,
71) -> None:
72 validations = sorted(
73 state.list("validation", ValidationResult, execution_id=run_id),
74 key=lambda item: item.finished_at,
75 )
76 # VERIFICATION attempts: each new ChangeSet digest verified is one attempt.
77 attempt_of: dict[str, int] = {}
78 for item in validations:
79 attempt_of.setdefault(item.change_set_digest, len(attempt_of) + 1)
81 mandatory, refs, optional_failures = _stopping_validations(validations, attempt_of)
82 corrected = any(event.event_type == "correction.authorized" for event in events)
83 for (validator, status), attempts in sorted(mandatory.items()):
84 last = max(attempt_of.values()) if attempt_of else 0
85 effect = (
86 "stopped VERIFICATION and was corrected in a later attempt"
87 if corrected and max(attempts) < last
88 else "stopped VERIFICATION"
89 )
90 analysis.causes.append(
91 RetrospectiveCause(
92 reason_code=f"MANDATORY_VALIDATOR_{status}",
93 subject=validator,
94 phase=PhaseId.VERIFICATION,
95 effect=effect,
96 occurrences=len(attempts),
97 attempts=tuple(sorted(attempts)),
98 evidence_refs=tuple(refs[(validator, status)][:10]),
99 )
100 )
101 gate_optional = optional_failures - {key[0] for key in mandatory}
102 if gate_optional:
103 analysis.notes.append(
104 "Optional validators that did not pass had no effect on the gate and are not counted "
105 "as causes: " + ", ".join(sorted(gate_optional)) + "."
106 )
109def _blocking_finding(
110 code: str, findings: dict[str, Finding], blocking: set[FindingSeverity]
111) -> Finding | None:
112 """The finding a ``BLOCKING_FINDING_<id>`` reason code names, when its severity blocks."""
113 if not code.startswith("BLOCKING_FINDING_"):
114 return None
115 finding = findings.get(code.removeprefix("BLOCKING_FINDING_"))
116 if finding is None or finding.severity not in blocking: 116 ↛ 117line 116 didn't jump to line 117 because the condition on line 116 was never true
117 return None
118 return finding
121def _blocking_rules(
122 gates: list[GateEvaluation], findings: dict[str, Finding], blocking: set[FindingSeverity]
123) -> tuple[dict[str, list[int]], dict[str, list[str]]]:
124 """The gates (by index) each rule failed with a blocking finding, and the findings."""
125 rule_gates: dict[str, list[int]] = defaultdict(list)
126 rule_refs: dict[str, list[str]] = defaultdict(list)
127 for index, gate in enumerate(gates, start=1):
128 for code in gate.reason_codes:
129 finding = _blocking_finding(code, findings, blocking)
130 if finding is None:
131 continue
132 if index not in rule_gates[finding.rule_id]: 132 ↛ 134line 132 didn't jump to line 134 because the condition on line 132 was always true
133 rule_gates[finding.rule_id].append(index)
134 rule_refs[finding.rule_id].append(f"record://finding/{finding.finding_id}")
135 return rule_gates, rule_refs
138def _finding_causes(
139 state: SQLiteStateStore, run_id: str, blocking: set[FindingSeverity]
140) -> list[RetrospectiveCause]:
141 findings = {
142 item.finding_id: item for item in state.list("finding", Finding, execution_id=run_id)
143 }
144 gates = sorted(
145 state.list("gate", GateEvaluation, execution_id=run_id), key=lambda item: item.evaluated_at
146 )
147 rule_gates, rule_refs = _blocking_rules(gates, findings, blocking)
148 return [
149 RetrospectiveCause(
150 reason_code="BLOCKING_FINDING",
151 subject=rule,
152 phase=PhaseId.DECISION,
153 effect="failed the delivery gate",
154 occurrences=len(indexes),
155 attempts=tuple(indexes),
156 evidence_refs=tuple(rule_refs[rule][:10]),
157 )
158 for rule, indexes in sorted(rule_gates.items())
159 ]
162def _decision_causes(state: SQLiteStateStore, run_id: str) -> list[RetrospectiveCause]:
163 decisions = sorted(
164 state.list("decision", HumanDecision, execution_id=run_id), key=lambda item: item.decided_at
165 )
166 by_kind: dict[DecisionKind, list[int]] = defaultdict(list)
167 for index, decision in enumerate(decisions, start=1):
168 by_kind[decision.decision].append(index)
169 effects = {
170 DecisionKind.REQUEST_CHANGES: ("CHANGES_REQUESTED", "sent the run back to IMPLEMENTATION"),
171 DecisionKind.REJECT: ("REJECTED", "ended the run"),
172 DecisionKind.APPROVE_EXCEPTION: ("EXCEPTION_APPROVED", "closed the run over a failed gate"),
173 }
174 return [
175 RetrospectiveCause(
176 reason_code=code,
177 subject="human-decision",
178 phase=PhaseId.DECISION,
179 effect=effect,
180 occurrences=len(by_kind[kind]),
181 attempts=tuple(by_kind[kind]),
182 evidence_refs=tuple(
183 f"record://decision/{item.decision_id}"
184 for item in decisions
185 if item.decision is kind
186 ),
187 )
188 for kind, (code, effect) in effects.items()
189 if by_kind.get(kind)
190 ]
193def _exception_causes(state: SQLiteStateStore, run_id: str) -> list[RetrospectiveCause]:
194 return [
195 RetrospectiveCause(
196 reason_code="EXCEPTION_GRANTED",
197 subject=scope.rule_id,
198 phase=PhaseId.DECISION,
199 effect=f"excepted until {record.expires_at.isoformat()}",
200 occurrences=1,
201 evidence_refs=(f"record://exception/{record.exception_id}",),
202 )
203 for record in state.list("exception", ExceptionRecord, execution_id=run_id)
204 for scope in record.scope
205 ]
208def _run_causes(
209 state: SQLiteStateStore, events: list[StoredEvent], execution: Execution
210) -> list[RetrospectiveCause]:
211 """Transient provider failures, a cancellation and outcomes recorded after the run."""
212 run_id = execution.execution_id
213 causes: list[RetrospectiveCause] = []
214 retries = [event for event in events if event.event_type == "agent.invocation.retried"]
215 if retries:
216 provider = state.get_flag(f"provider:{run_id}") or "provider"
217 causes.append(
218 RetrospectiveCause(
219 reason_code="PROVIDER_TRANSIENT_FAILURE",
220 subject=provider,
221 phase=PhaseId.IMPLEMENTATION,
222 effect="repeated the agent call",
223 occurrences=len(retries),
224 )
225 )
226 if execution.status is ResultStatus.CANCELLED:
227 causes.append(
228 RetrospectiveCause(
229 reason_code="RUN_CANCELLED",
230 subject=execution.current_phase.value,
231 phase=execution.current_phase,
232 effect="ended the run before closure",
233 occurrences=1,
234 )
235 )
236 for outcome in state.list("outcome", OutcomeRecord, execution_id=run_id): 236 ↛ 237line 236 didn't jump to line 237 because the loop on line 236 never started
237 causes.append(
238 RetrospectiveCause(
239 reason_code=f"POST_RUN_{outcome.kind}",
240 subject=outcome.reference or outcome.kind.lower(),
241 phase=PhaseId.CLOSURE,
242 effect="recorded after the run",
243 occurrences=1,
244 evidence_refs=(f"record://outcome/{outcome.outcome_id}",),
245 )
246 )
247 return causes
250def analyze(
251 state: SQLiteStateStore,
252 events: list[StoredEvent],
253 execution: Execution,
254 blocking: set[FindingSeverity],
255) -> CauseAnalysis:
256 run_id = execution.execution_id
257 analysis = CauseAnalysis()
258 _add_validator_causes(analysis, state, events, run_id)
259 analysis.causes.extend(_finding_causes(state, run_id, blocking))
260 analysis.causes.extend(_decision_causes(state, run_id))
261 analysis.causes.extend(_exception_causes(state, run_id))
262 analysis.causes.extend(_run_causes(state, events, execution))
263 return analysis