Coverage for src/governed_harness/application/health.py: 87%

125 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Rule health across runs and outcomes recorded after a run. 

2 

3Rule health reads every run of the project and says, for each rule and validator, how often it 

4fired, how often it blocked a gate, how often a person granted an exception over it, how often 

5it fired on a ChangeSet that was later corrected, how often it fired in a rejected run and how 

6often a run where it fired was later linked to an incident or revert. A rule that blocks often 

7and is often excepted is a candidate false positive; one that is quiet in runs that later had 

8incidents is a candidate gap. Nothing is changed: a person decides. No figure is per person.""" 

9 

10from __future__ import annotations 

11 

12from collections import defaultdict 

13from datetime import datetime, timedelta 

14from typing import Any 

15 

16from governed_harness import __version__ 

17from governed_harness.domain.actors import NON_HUMAN_ACTOR_PREFIXES 

18from governed_harness.domain.enums import ActorType, DecisionKind, ResultStatus 

19from governed_harness.domain.errors import ConfigurationError, PolicyViolationError 

20from governed_harness.domain.ids import new_id 

21from governed_harness.domain.models import ( 

22 Actor, 

23 ChangeSet, 

24 ExceptionRecord, 

25 Execution, 

26 Finding, 

27 GateEvaluation, 

28 HumanDecision, 

29 OutcomeKind, 

30 OutcomeRecord, 

31 Provenance, 

32 ValidationResult, 

33 utc_now, 

34) 

35from governed_harness.gates.exceptions import exception_ids 

36from governed_harness.orchestration.engine import EngineServices 

37 

38OUTCOME_KINDS: tuple[OutcomeKind, ...] = ("INCIDENT", "REVERT", "HOTFIX", "REGRESSION", "OTHER") 

39 

40 

41def record_outcome( 

42 services: EngineServices, 

43 *, 

44 execution_id: str, 

45 kind: str, 

46 summary: str, 

47 reference: str | None, 

48 observed_at: str | None, 

49 actor_id: str, 

50) -> OutcomeRecord: 

51 if actor_id.startswith(NON_HUMAN_ACTOR_PREFIXES): 

52 raise PolicyViolationError("only a person can link an outcome to a run") 

53 upper = kind.upper() 

54 if upper not in OUTCOME_KINDS: 

55 raise ConfigurationError(f"outcome kind must be one of {', '.join(OUTCOME_KINDS)}: {kind}") 

56 if not summary.strip(): 56 ↛ 57line 56 didn't jump to line 57 because the condition on line 56 was never true

57 raise ConfigurationError("an outcome needs a summary") 

58 when = utc_now() 

59 if observed_at: 59 ↛ 67line 59 didn't jump to line 67 because the condition on line 59 was always true

60 try: 

61 when = datetime.fromisoformat(observed_at) 

62 except ValueError as error: 

63 raise ConfigurationError( 

64 f"--observed-at is not an ISO 8601 instant: {observed_at}" 

65 ) from error 

66 when = when if when.tzinfo else when.replace(tzinfo=utc_now().tzinfo) 

67 execution = services.state.get("execution", execution_id, Execution) 

68 actor = Actor(actor_type=ActorType.HUMAN, actor_id=actor_id) 

69 record = OutcomeRecord( 

70 outcome_id=new_id("outcome"), 

71 project_id=execution.project_id, 

72 execution_id=execution.execution_id, 

73 kind=upper, 

74 summary=summary.strip(), 

75 reference=(reference or "").strip() or None, 

76 change_set_digest=execution.change_set_digest, 

77 observed_at=when, 

78 actor=actor, 

79 provenance=Provenance( 

80 actor=actor, 

81 core_version=__version__, 

82 configuration_digest=execution.configuration_digest, 

83 policy_digest=execution.policy_digest, 

84 source_refs=(f"record://execution/{execution.execution_id}",), 

85 ), 

86 ) 

87 services.state.put( 

88 "outcome", 

89 record.outcome_id, 

90 record, 

91 execution_id=execution.execution_id, 

92 project_id=execution.project_id, 

93 ) 

94 services.artifacts.put_json( 

95 record.model_dump(mode="json", by_alias=True), 

96 metadata={"kind": "outcome", "executionId": execution.execution_id}, 

97 ) 

98 return record 

99 

100 

101def list_outcomes(services: EngineServices, execution_id: str | None = None) -> list[OutcomeRecord]: 

102 records = services.state.list( 

103 "outcome", 

104 OutcomeRecord, 

105 execution_id=execution_id, 

106 project_id=services.resolved.project.project_id, 

107 ) 

108 return sorted(records, key=lambda item: item.observed_at) 

109 

110 

111def _blank() -> dict[str, Any]: 

112 return { 

113 "fired": 0, 

114 "runs": set(), 

115 "blockedGates": 0, 

116 "excepted": 0, 

117 "exceptionApplications": 0, 

118 "correctedRuns": set(), 

119 "rejectedRuns": set(), 

120 "outcomeRuns": set(), 

121 } 

122 

123 

124def _blank_validator() -> dict[str, Any]: 

125 return {"runs": 0, "mandatory": False, "notPassed": 0, "results": 0} 

126 

127 

128def _tally_validations( 

129 services: EngineServices, run_id: str, validators: dict[str, dict[str, Any]] 

130) -> dict[str, str]: 

131 """Count the validation results of one run; the ChangeSet digest each finding came from.""" 

132 digest_of: dict[str, str] = {} 

133 seen_validators: set[str] = set() 

134 for validation in services.state.list("validation", ValidationResult, execution_id=run_id): 

135 for finding_id in validation.finding_ids: 

136 digest_of[finding_id] = validation.change_set_digest 

137 entry = validators[validation.validator_id] 

138 entry["results"] += 1 

139 entry["mandatory"] = entry["mandatory"] or validation.mandatory 

140 if validation.status is not ResultStatus.PASSED: 

141 entry["notPassed"] += 1 

142 seen_validators.add(validation.validator_id) 

143 for validator_id in seen_validators: 

144 validators[validator_id]["runs"] += 1 

145 return digest_of 

146 

147 

148def _tally_findings( 

149 run_id: str, 

150 findings: dict[str, Finding], 

151 rules: dict[str, dict[str, Any]], 

152 signals: tuple[bool, bool, bool, str | None, dict[str, str]], 

153) -> None: 

154 """Count the findings of one run by rule: ``signals`` is (corrected, rejected, has an 

155 outcome, final ChangeSet digest, digest by finding).""" 

156 corrected, rejected, has_outcome, final, digest_of = signals 

157 for finding in findings.values(): 

158 entry = rules[finding.rule_id] 

159 entry["fired"] += 1 

160 entry["runs"].add(run_id) 

161 if corrected and digest_of.get(finding.finding_id) not in {None, final}: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true

162 entry["correctedRuns"].add(run_id) 

163 if rejected and digest_of.get(finding.finding_id) == final: 163 ↛ 164line 163 didn't jump to line 164 because the condition on line 163 was never true

164 entry["rejectedRuns"].add(run_id) 

165 if has_outcome: 165 ↛ 157line 165 didn't jump to line 157 because the condition on line 165 was always true

166 entry["outcomeRuns"].add(run_id) 

167 

168 

169def _tally_gates( 

170 services: EngineServices, 

171 run_id: str, 

172 findings: dict[str, Finding], 

173 rules: dict[str, dict[str, Any]], 

174 applications: dict[str, int], 

175) -> None: 

176 """Count the gates each rule blocked in one run and the exceptions they applied.""" 

177 for gate in services.state.list("gate", GateEvaluation, execution_id=run_id): 

178 for code in gate.reason_codes: 

179 blocked = ( 

180 findings.get(code.removeprefix("BLOCKING_FINDING_")) 

181 if code.startswith("BLOCKING_FINDING_") 

182 else None 

183 ) 

184 if blocked is not None: 184 ↛ 178line 184 didn't jump to line 178 because the condition on line 184 was always true

185 rules[blocked.rule_id]["blockedGates"] += 1 

186 for exception_id in exception_ids(gate.reason_codes): 186 ↛ 187line 186 didn't jump to line 187 because the loop on line 186 never started

187 applications[exception_id] += 1 

188 

189 

190def _tally_run( 

191 services: EngineServices, 

192 execution: Execution, 

193 outcome_runs: set[str], 

194 tallies: tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]], dict[str, int]], 

195) -> bool: 

196 """Count one run into ``tallies`` (rules, validators, exception applications); whether a 

197 correction was authorized in it.""" 

198 rules, validators, applications = tallies 

199 run_id = execution.execution_id 

200 events = services.events.list(run_id) 

201 corrected = any(event.event_type == "correction.authorized" for event in events) 

202 decisions = services.state.list("decision", HumanDecision, execution_id=run_id) 

203 rejected = any(item.decision is DecisionKind.REJECT for item in decisions) 

204 final = execution.change_set_digest 

205 findings = { 

206 item.finding_id: item 

207 for item in services.state.list("finding", Finding, execution_id=run_id) 

208 } 

209 digest_of = _tally_validations(services, run_id, validators) 

210 signals = (corrected, rejected, run_id in outcome_runs, final, digest_of) 

211 _tally_findings(run_id, findings, rules, signals) 

212 _tally_gates(services, run_id, findings, rules, applications) 

213 return corrected 

214 

215 

216def _tally_exceptions( 

217 services: EngineServices, 

218 project_id: str, 

219 run_ids: set[str], 

220 rules: dict[str, dict[str, Any]], 

221 applications: dict[str, int], 

222) -> None: 

223 """Count the exceptions recorded for each rule in the selected runs.""" 

224 for record in services.state.list("exception", ExceptionRecord, project_id=project_id): 

225 if record.execution_id not in run_ids: 225 ↛ 226line 225 didn't jump to line 226 because the condition on line 225 was never true

226 continue 

227 for rule_id in {scope.rule_id for scope in record.scope}: 

228 rules[rule_id]["excepted"] += 1 

229 rules[rule_id]["exceptionApplications"] += applications[record.exception_id] 

230 

231 

232def _rule_row(rule_id: str, entry: dict[str, Any]) -> dict[str, Any]: 

233 return { 

234 "ruleId": rule_id, 

235 "fired": entry["fired"], 

236 "runs": len(entry["runs"]), 

237 "blockedGates": entry["blockedGates"], 

238 "excepted": entry["excepted"], 

239 "exceptionApplications": entry["exceptionApplications"], 

240 "correctedRuns": len(entry["correctedRuns"]), 

241 "rejectedRuns": len(entry["rejectedRuns"]), 

242 "runsWithLaterOutcomes": len(entry["outcomeRuns"]), 

243 "signal": _signal(entry), 

244 } 

245 

246 

247def rule_health(services: EngineServices, since_days: int | None = None) -> dict[str, Any]: 

248 project_id = services.resolved.project.project_id 

249 cutoff = utc_now() - timedelta(days=since_days) if since_days is not None else None 

250 runs = [ 

251 item 

252 for item in services.state.list("execution", Execution, project_id=project_id) 

253 if cutoff is None or item.created_at >= cutoff 

254 ] 

255 run_ids = {item.execution_id for item in runs} 

256 rules: dict[str, dict[str, Any]] = defaultdict(_blank) 

257 validators: dict[str, dict[str, Any]] = defaultdict(_blank_validator) 

258 outcomes = [ 

259 item 

260 for item in services.state.list("outcome", OutcomeRecord, project_id=project_id) 

261 if item.execution_id in run_ids 

262 ] 

263 outcome_runs = {item.execution_id for item in outcomes} 

264 corrected_runs: set[str] = set() 

265 applications: dict[str, int] = defaultdict(int) 

266 for execution in runs: 

267 if _tally_run(services, execution, outcome_runs, (rules, validators, applications)): 267 ↛ 268line 267 didn't jump to line 268 because the condition on line 267 was never true

268 corrected_runs.add(execution.execution_id) 

269 _tally_exceptions(services, project_id, run_ids, rules, applications) 

270 rows = [_rule_row(rule_id, entry) for rule_id, entry in rules.items()] 

271 rows.sort(key=lambda item: (-item["fired"], item["ruleId"])) 

272 return { 

273 "projectId": project_id, 

274 "since": cutoff.isoformat() if cutoff else None, 

275 "runs": len(runs), 

276 "correctedRuns": len(corrected_runs), 

277 "outcomes": { 

278 kind: sum(1 for item in outcomes if item.kind == kind) for kind in OUTCOME_KINDS 

279 }, 

280 "rules": rows, 

281 "validators": [ 

282 { 

283 "validatorId": validator_id, 

284 "mandatory": entry["mandatory"], 

285 "runs": entry["runs"], 

286 "results": entry["results"], 

287 "notPassed": entry["notPassed"], 

288 } 

289 for validator_id, entry in sorted(validators.items()) 

290 ], 

291 "changeSets": sum( 

292 len(services.state.list("change_set", ChangeSet, execution_id=item.execution_id)) 

293 for item in runs 

294 ), 

295 } 

296 

297 

298def _signal(entry: dict[str, Any]) -> str | None: 

299 """A fixed-rule hint, not a verdict: what a person may want to look at.""" 

300 if entry["blockedGates"] and entry["excepted"] >= max(1, entry["blockedGates"] // 2): 

301 return "often excepted when it blocks: check its precision" 

302 if entry["outcomeRuns"]: 302 ↛ 304line 302 didn't jump to line 304 because the condition on line 302 was always true

303 return "fired in runs later linked to an outcome" 

304 if entry["correctedRuns"]: 

305 return "led to corrections" 

306 return None