Coverage for src/governed_harness/retrospective/causes.py: 95%

96 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Causes of a run's stops and corrections, by reason code (``retrospective.causal``). 

2 

3The 1.0.0 retrospective counted every non-passed validation, including optional validators that 

4had no effect on the gate, and so attributed cycles to the wrong cause. Here a cause is only 

5what stopped VERIFICATION (a mandatory validator that did not pass), what failed a delivery gate 

6(a blocking finding), a decision that sent the run back or ended it, an exception, a transient 

7provider failure or an outcome recorded after the run. Subjects are validators, rules and 

8providers, never people (no per-person metric).""" 

9 

10from __future__ import annotations 

11 

12from collections import defaultdict 

13from dataclasses import dataclass, field 

14 

15from governed_harness.domain.enums import DecisionKind, FindingSeverity, PhaseId, ResultStatus 

16from governed_harness.domain.models import ( 

17 ExceptionRecord, 

18 Execution, 

19 Finding, 

20 GateEvaluation, 

21 HumanDecision, 

22 OutcomeRecord, 

23 RetrospectiveCause, 

24 ValidationResult, 

25) 

26from governed_harness.events import StoredEvent 

27from governed_harness.storage import SQLiteStateStore 

28 

29STOPPING = { 

30 ResultStatus.FAILED, 

31 ResultStatus.BLOCKED, 

32 ResultStatus.ERROR, 

33 ResultStatus.TIMED_OUT, 

34 ResultStatus.INCONCLUSIVE, 

35} 

36 

37 

38@dataclass 

39class CauseAnalysis: 

40 causes: list[RetrospectiveCause] = field(default_factory=list) 

41 notes: list[str] = field(default_factory=list) 

42 

43 

44def _stopping_validations( 

45 validations: list[ValidationResult], attempt_of: dict[str, int] 

46) -> tuple[dict[tuple[str, str], list[int]], dict[tuple[str, str], list[str]], set[str]]: 

47 """Mandatory validators that stopped VERIFICATION, by (validator, status), with their 

48 attempts and evidence; and the optional validators that did not pass.""" 

49 mandatory: dict[tuple[str, str], list[int]] = defaultdict(list) 

50 refs: dict[tuple[str, str], list[str]] = defaultdict(list) 

51 optional_failures: set[str] = set() 

52 for item in validations: 

53 if item.status not in STOPPING: 

54 continue 

55 if not item.mandatory: 

56 optional_failures.add(item.validator_id) 

57 continue 

58 key = (item.validator_id, item.status.value) 

59 attempt = attempt_of[item.change_set_digest] 

60 if attempt not in mandatory[key]: 60 ↛ 62line 60 didn't jump to line 62 because the condition on line 60 was always true

61 mandatory[key].append(attempt) 

62 refs[key].append(f"record://validation/{item.validation_result_id}") 

63 return mandatory, refs, optional_failures 

64 

65 

66def _add_validator_causes( 

67 analysis: CauseAnalysis, 

68 state: SQLiteStateStore, 

69 events: list[StoredEvent], 

70 run_id: str, 

71) -> None: 

72 validations = sorted( 

73 state.list("validation", ValidationResult, execution_id=run_id), 

74 key=lambda item: item.finished_at, 

75 ) 

76 # VERIFICATION attempts: each new ChangeSet digest verified is one attempt. 

77 attempt_of: dict[str, int] = {} 

78 for item in validations: 

79 attempt_of.setdefault(item.change_set_digest, len(attempt_of) + 1) 

80 

81 mandatory, refs, optional_failures = _stopping_validations(validations, attempt_of) 

82 corrected = any(event.event_type == "correction.authorized" for event in events) 

83 for (validator, status), attempts in sorted(mandatory.items()): 

84 last = max(attempt_of.values()) if attempt_of else 0 

85 effect = ( 

86 "stopped VERIFICATION and was corrected in a later attempt" 

87 if corrected and max(attempts) < last 

88 else "stopped VERIFICATION" 

89 ) 

90 analysis.causes.append( 

91 RetrospectiveCause( 

92 reason_code=f"MANDATORY_VALIDATOR_{status}", 

93 subject=validator, 

94 phase=PhaseId.VERIFICATION, 

95 effect=effect, 

96 occurrences=len(attempts), 

97 attempts=tuple(sorted(attempts)), 

98 evidence_refs=tuple(refs[(validator, status)][:10]), 

99 ) 

100 ) 

101 gate_optional = optional_failures - {key[0] for key in mandatory} 

102 if gate_optional: 

103 analysis.notes.append( 

104 "Optional validators that did not pass had no effect on the gate and are not counted " 

105 "as causes: " + ", ".join(sorted(gate_optional)) + "." 

106 ) 

107 

108 

109def _blocking_finding( 

110 code: str, findings: dict[str, Finding], blocking: set[FindingSeverity] 

111) -> Finding | None: 

112 """The finding a ``BLOCKING_FINDING_<id>`` reason code names, when its severity blocks.""" 

113 if not code.startswith("BLOCKING_FINDING_"): 

114 return None 

115 finding = findings.get(code.removeprefix("BLOCKING_FINDING_")) 

116 if finding is None or finding.severity not in blocking: 116 ↛ 117line 116 didn't jump to line 117 because the condition on line 116 was never true

117 return None 

118 return finding 

119 

120 

121def _blocking_rules( 

122 gates: list[GateEvaluation], findings: dict[str, Finding], blocking: set[FindingSeverity] 

123) -> tuple[dict[str, list[int]], dict[str, list[str]]]: 

124 """The gates (by index) each rule failed with a blocking finding, and the findings.""" 

125 rule_gates: dict[str, list[int]] = defaultdict(list) 

126 rule_refs: dict[str, list[str]] = defaultdict(list) 

127 for index, gate in enumerate(gates, start=1): 

128 for code in gate.reason_codes: 

129 finding = _blocking_finding(code, findings, blocking) 

130 if finding is None: 

131 continue 

132 if index not in rule_gates[finding.rule_id]: 132 ↛ 134line 132 didn't jump to line 134 because the condition on line 132 was always true

133 rule_gates[finding.rule_id].append(index) 

134 rule_refs[finding.rule_id].append(f"record://finding/{finding.finding_id}") 

135 return rule_gates, rule_refs 

136 

137 

138def _finding_causes( 

139 state: SQLiteStateStore, run_id: str, blocking: set[FindingSeverity] 

140) -> list[RetrospectiveCause]: 

141 findings = { 

142 item.finding_id: item for item in state.list("finding", Finding, execution_id=run_id) 

143 } 

144 gates = sorted( 

145 state.list("gate", GateEvaluation, execution_id=run_id), key=lambda item: item.evaluated_at 

146 ) 

147 rule_gates, rule_refs = _blocking_rules(gates, findings, blocking) 

148 return [ 

149 RetrospectiveCause( 

150 reason_code="BLOCKING_FINDING", 

151 subject=rule, 

152 phase=PhaseId.DECISION, 

153 effect="failed the delivery gate", 

154 occurrences=len(indexes), 

155 attempts=tuple(indexes), 

156 evidence_refs=tuple(rule_refs[rule][:10]), 

157 ) 

158 for rule, indexes in sorted(rule_gates.items()) 

159 ] 

160 

161 

162def _decision_causes(state: SQLiteStateStore, run_id: str) -> list[RetrospectiveCause]: 

163 decisions = sorted( 

164 state.list("decision", HumanDecision, execution_id=run_id), key=lambda item: item.decided_at 

165 ) 

166 by_kind: dict[DecisionKind, list[int]] = defaultdict(list) 

167 for index, decision in enumerate(decisions, start=1): 

168 by_kind[decision.decision].append(index) 

169 effects = { 

170 DecisionKind.REQUEST_CHANGES: ("CHANGES_REQUESTED", "sent the run back to IMPLEMENTATION"), 

171 DecisionKind.REJECT: ("REJECTED", "ended the run"), 

172 DecisionKind.APPROVE_EXCEPTION: ("EXCEPTION_APPROVED", "closed the run over a failed gate"), 

173 } 

174 return [ 

175 RetrospectiveCause( 

176 reason_code=code, 

177 subject="human-decision", 

178 phase=PhaseId.DECISION, 

179 effect=effect, 

180 occurrences=len(by_kind[kind]), 

181 attempts=tuple(by_kind[kind]), 

182 evidence_refs=tuple( 

183 f"record://decision/{item.decision_id}" 

184 for item in decisions 

185 if item.decision is kind 

186 ), 

187 ) 

188 for kind, (code, effect) in effects.items() 

189 if by_kind.get(kind) 

190 ] 

191 

192 

193def _exception_causes(state: SQLiteStateStore, run_id: str) -> list[RetrospectiveCause]: 

194 return [ 

195 RetrospectiveCause( 

196 reason_code="EXCEPTION_GRANTED", 

197 subject=scope.rule_id, 

198 phase=PhaseId.DECISION, 

199 effect=f"excepted until {record.expires_at.isoformat()}", 

200 occurrences=1, 

201 evidence_refs=(f"record://exception/{record.exception_id}",), 

202 ) 

203 for record in state.list("exception", ExceptionRecord, execution_id=run_id) 

204 for scope in record.scope 

205 ] 

206 

207 

208def _run_causes( 

209 state: SQLiteStateStore, events: list[StoredEvent], execution: Execution 

210) -> list[RetrospectiveCause]: 

211 """Transient provider failures, a cancellation and outcomes recorded after the run.""" 

212 run_id = execution.execution_id 

213 causes: list[RetrospectiveCause] = [] 

214 retries = [event for event in events if event.event_type == "agent.invocation.retried"] 

215 if retries: 

216 provider = state.get_flag(f"provider:{run_id}") or "provider" 

217 causes.append( 

218 RetrospectiveCause( 

219 reason_code="PROVIDER_TRANSIENT_FAILURE", 

220 subject=provider, 

221 phase=PhaseId.IMPLEMENTATION, 

222 effect="repeated the agent call", 

223 occurrences=len(retries), 

224 ) 

225 ) 

226 if execution.status is ResultStatus.CANCELLED: 

227 causes.append( 

228 RetrospectiveCause( 

229 reason_code="RUN_CANCELLED", 

230 subject=execution.current_phase.value, 

231 phase=execution.current_phase, 

232 effect="ended the run before closure", 

233 occurrences=1, 

234 ) 

235 ) 

236 for outcome in state.list("outcome", OutcomeRecord, execution_id=run_id): 236 ↛ 237line 236 didn't jump to line 237 because the loop on line 236 never started

237 causes.append( 

238 RetrospectiveCause( 

239 reason_code=f"POST_RUN_{outcome.kind}", 

240 subject=outcome.reference or outcome.kind.lower(), 

241 phase=PhaseId.CLOSURE, 

242 effect="recorded after the run", 

243 occurrences=1, 

244 evidence_refs=(f"record://outcome/{outcome.outcome_id}",), 

245 ) 

246 ) 

247 return causes 

248 

249 

250def analyze( 

251 state: SQLiteStateStore, 

252 events: list[StoredEvent], 

253 execution: Execution, 

254 blocking: set[FindingSeverity], 

255) -> CauseAnalysis: 

256 run_id = execution.execution_id 

257 analysis = CauseAnalysis() 

258 _add_validator_causes(analysis, state, events, run_id) 

259 analysis.causes.extend(_finding_causes(state, run_id, blocking)) 

260 analysis.causes.extend(_decision_causes(state, run_id)) 

261 analysis.causes.extend(_exception_causes(state, run_id)) 

262 analysis.causes.extend(_run_causes(state, events, execution)) 

263 return analysis