Coverage for src/governed_harness/application/health.py: 87%
125 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Rule health across runs and outcomes recorded after a run.
3Rule health reads every run of the project and says, for each rule and validator, how often it
4fired, how often it blocked a gate, how often a person granted an exception over it, how often
5it fired on a ChangeSet that was later corrected, how often it fired in a rejected run and how
6often a run where it fired was later linked to an incident or revert. A rule that blocks often
7and is often excepted is a candidate false positive; one that is quiet in runs that later had
8incidents is a candidate gap. Nothing is changed: a person decides. No figure is per person."""
10from __future__ import annotations
12from collections import defaultdict
13from datetime import datetime, timedelta
14from typing import Any
16from governed_harness import __version__
17from governed_harness.domain.actors import NON_HUMAN_ACTOR_PREFIXES
18from governed_harness.domain.enums import ActorType, DecisionKind, ResultStatus
19from governed_harness.domain.errors import ConfigurationError, PolicyViolationError
20from governed_harness.domain.ids import new_id
21from governed_harness.domain.models import (
22 Actor,
23 ChangeSet,
24 ExceptionRecord,
25 Execution,
26 Finding,
27 GateEvaluation,
28 HumanDecision,
29 OutcomeKind,
30 OutcomeRecord,
31 Provenance,
32 ValidationResult,
33 utc_now,
34)
35from governed_harness.gates.exceptions import exception_ids
36from governed_harness.orchestration.engine import EngineServices
38OUTCOME_KINDS: tuple[OutcomeKind, ...] = ("INCIDENT", "REVERT", "HOTFIX", "REGRESSION", "OTHER")
41def record_outcome(
42 services: EngineServices,
43 *,
44 execution_id: str,
45 kind: str,
46 summary: str,
47 reference: str | None,
48 observed_at: str | None,
49 actor_id: str,
50) -> OutcomeRecord:
51 if actor_id.startswith(NON_HUMAN_ACTOR_PREFIXES):
52 raise PolicyViolationError("only a person can link an outcome to a run")
53 upper = kind.upper()
54 if upper not in OUTCOME_KINDS:
55 raise ConfigurationError(f"outcome kind must be one of {', '.join(OUTCOME_KINDS)}: {kind}")
56 if not summary.strip(): 56 ↛ 57line 56 didn't jump to line 57 because the condition on line 56 was never true
57 raise ConfigurationError("an outcome needs a summary")
58 when = utc_now()
59 if observed_at: 59 ↛ 67line 59 didn't jump to line 67 because the condition on line 59 was always true
60 try:
61 when = datetime.fromisoformat(observed_at)
62 except ValueError as error:
63 raise ConfigurationError(
64 f"--observed-at is not an ISO 8601 instant: {observed_at}"
65 ) from error
66 when = when if when.tzinfo else when.replace(tzinfo=utc_now().tzinfo)
67 execution = services.state.get("execution", execution_id, Execution)
68 actor = Actor(actor_type=ActorType.HUMAN, actor_id=actor_id)
69 record = OutcomeRecord(
70 outcome_id=new_id("outcome"),
71 project_id=execution.project_id,
72 execution_id=execution.execution_id,
73 kind=upper,
74 summary=summary.strip(),
75 reference=(reference or "").strip() or None,
76 change_set_digest=execution.change_set_digest,
77 observed_at=when,
78 actor=actor,
79 provenance=Provenance(
80 actor=actor,
81 core_version=__version__,
82 configuration_digest=execution.configuration_digest,
83 policy_digest=execution.policy_digest,
84 source_refs=(f"record://execution/{execution.execution_id}",),
85 ),
86 )
87 services.state.put(
88 "outcome",
89 record.outcome_id,
90 record,
91 execution_id=execution.execution_id,
92 project_id=execution.project_id,
93 )
94 services.artifacts.put_json(
95 record.model_dump(mode="json", by_alias=True),
96 metadata={"kind": "outcome", "executionId": execution.execution_id},
97 )
98 return record
101def list_outcomes(services: EngineServices, execution_id: str | None = None) -> list[OutcomeRecord]:
102 records = services.state.list(
103 "outcome",
104 OutcomeRecord,
105 execution_id=execution_id,
106 project_id=services.resolved.project.project_id,
107 )
108 return sorted(records, key=lambda item: item.observed_at)
111def _blank() -> dict[str, Any]:
112 return {
113 "fired": 0,
114 "runs": set(),
115 "blockedGates": 0,
116 "excepted": 0,
117 "exceptionApplications": 0,
118 "correctedRuns": set(),
119 "rejectedRuns": set(),
120 "outcomeRuns": set(),
121 }
124def _blank_validator() -> dict[str, Any]:
125 return {"runs": 0, "mandatory": False, "notPassed": 0, "results": 0}
128def _tally_validations(
129 services: EngineServices, run_id: str, validators: dict[str, dict[str, Any]]
130) -> dict[str, str]:
131 """Count the validation results of one run; the ChangeSet digest each finding came from."""
132 digest_of: dict[str, str] = {}
133 seen_validators: set[str] = set()
134 for validation in services.state.list("validation", ValidationResult, execution_id=run_id):
135 for finding_id in validation.finding_ids:
136 digest_of[finding_id] = validation.change_set_digest
137 entry = validators[validation.validator_id]
138 entry["results"] += 1
139 entry["mandatory"] = entry["mandatory"] or validation.mandatory
140 if validation.status is not ResultStatus.PASSED:
141 entry["notPassed"] += 1
142 seen_validators.add(validation.validator_id)
143 for validator_id in seen_validators:
144 validators[validator_id]["runs"] += 1
145 return digest_of
148def _tally_findings(
149 run_id: str,
150 findings: dict[str, Finding],
151 rules: dict[str, dict[str, Any]],
152 signals: tuple[bool, bool, bool, str | None, dict[str, str]],
153) -> None:
154 """Count the findings of one run by rule: ``signals`` is (corrected, rejected, has an
155 outcome, final ChangeSet digest, digest by finding)."""
156 corrected, rejected, has_outcome, final, digest_of = signals
157 for finding in findings.values():
158 entry = rules[finding.rule_id]
159 entry["fired"] += 1
160 entry["runs"].add(run_id)
161 if corrected and digest_of.get(finding.finding_id) not in {None, final}: 161 ↛ 162line 161 didn't jump to line 162 because the condition on line 161 was never true
162 entry["correctedRuns"].add(run_id)
163 if rejected and digest_of.get(finding.finding_id) == final: 163 ↛ 164line 163 didn't jump to line 164 because the condition on line 163 was never true
164 entry["rejectedRuns"].add(run_id)
165 if has_outcome: 165 ↛ 157line 165 didn't jump to line 157 because the condition on line 165 was always true
166 entry["outcomeRuns"].add(run_id)
169def _tally_gates(
170 services: EngineServices,
171 run_id: str,
172 findings: dict[str, Finding],
173 rules: dict[str, dict[str, Any]],
174 applications: dict[str, int],
175) -> None:
176 """Count the gates each rule blocked in one run and the exceptions they applied."""
177 for gate in services.state.list("gate", GateEvaluation, execution_id=run_id):
178 for code in gate.reason_codes:
179 blocked = (
180 findings.get(code.removeprefix("BLOCKING_FINDING_"))
181 if code.startswith("BLOCKING_FINDING_")
182 else None
183 )
184 if blocked is not None: 184 ↛ 178line 184 didn't jump to line 178 because the condition on line 184 was always true
185 rules[blocked.rule_id]["blockedGates"] += 1
186 for exception_id in exception_ids(gate.reason_codes): 186 ↛ 187line 186 didn't jump to line 187 because the loop on line 186 never started
187 applications[exception_id] += 1
190def _tally_run(
191 services: EngineServices,
192 execution: Execution,
193 outcome_runs: set[str],
194 tallies: tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]], dict[str, int]],
195) -> bool:
196 """Count one run into ``tallies`` (rules, validators, exception applications); whether a
197 correction was authorized in it."""
198 rules, validators, applications = tallies
199 run_id = execution.execution_id
200 events = services.events.list(run_id)
201 corrected = any(event.event_type == "correction.authorized" for event in events)
202 decisions = services.state.list("decision", HumanDecision, execution_id=run_id)
203 rejected = any(item.decision is DecisionKind.REJECT for item in decisions)
204 final = execution.change_set_digest
205 findings = {
206 item.finding_id: item
207 for item in services.state.list("finding", Finding, execution_id=run_id)
208 }
209 digest_of = _tally_validations(services, run_id, validators)
210 signals = (corrected, rejected, run_id in outcome_runs, final, digest_of)
211 _tally_findings(run_id, findings, rules, signals)
212 _tally_gates(services, run_id, findings, rules, applications)
213 return corrected
216def _tally_exceptions(
217 services: EngineServices,
218 project_id: str,
219 run_ids: set[str],
220 rules: dict[str, dict[str, Any]],
221 applications: dict[str, int],
222) -> None:
223 """Count the exceptions recorded for each rule in the selected runs."""
224 for record in services.state.list("exception", ExceptionRecord, project_id=project_id):
225 if record.execution_id not in run_ids: 225 ↛ 226line 225 didn't jump to line 226 because the condition on line 225 was never true
226 continue
227 for rule_id in {scope.rule_id for scope in record.scope}:
228 rules[rule_id]["excepted"] += 1
229 rules[rule_id]["exceptionApplications"] += applications[record.exception_id]
232def _rule_row(rule_id: str, entry: dict[str, Any]) -> dict[str, Any]:
233 return {
234 "ruleId": rule_id,
235 "fired": entry["fired"],
236 "runs": len(entry["runs"]),
237 "blockedGates": entry["blockedGates"],
238 "excepted": entry["excepted"],
239 "exceptionApplications": entry["exceptionApplications"],
240 "correctedRuns": len(entry["correctedRuns"]),
241 "rejectedRuns": len(entry["rejectedRuns"]),
242 "runsWithLaterOutcomes": len(entry["outcomeRuns"]),
243 "signal": _signal(entry),
244 }
247def rule_health(services: EngineServices, since_days: int | None = None) -> dict[str, Any]:
248 project_id = services.resolved.project.project_id
249 cutoff = utc_now() - timedelta(days=since_days) if since_days is not None else None
250 runs = [
251 item
252 for item in services.state.list("execution", Execution, project_id=project_id)
253 if cutoff is None or item.created_at >= cutoff
254 ]
255 run_ids = {item.execution_id for item in runs}
256 rules: dict[str, dict[str, Any]] = defaultdict(_blank)
257 validators: dict[str, dict[str, Any]] = defaultdict(_blank_validator)
258 outcomes = [
259 item
260 for item in services.state.list("outcome", OutcomeRecord, project_id=project_id)
261 if item.execution_id in run_ids
262 ]
263 outcome_runs = {item.execution_id for item in outcomes}
264 corrected_runs: set[str] = set()
265 applications: dict[str, int] = defaultdict(int)
266 for execution in runs:
267 if _tally_run(services, execution, outcome_runs, (rules, validators, applications)): 267 ↛ 268line 267 didn't jump to line 268 because the condition on line 267 was never true
268 corrected_runs.add(execution.execution_id)
269 _tally_exceptions(services, project_id, run_ids, rules, applications)
270 rows = [_rule_row(rule_id, entry) for rule_id, entry in rules.items()]
271 rows.sort(key=lambda item: (-item["fired"], item["ruleId"]))
272 return {
273 "projectId": project_id,
274 "since": cutoff.isoformat() if cutoff else None,
275 "runs": len(runs),
276 "correctedRuns": len(corrected_runs),
277 "outcomes": {
278 kind: sum(1 for item in outcomes if item.kind == kind) for kind in OUTCOME_KINDS
279 },
280 "rules": rows,
281 "validators": [
282 {
283 "validatorId": validator_id,
284 "mandatory": entry["mandatory"],
285 "runs": entry["runs"],
286 "results": entry["results"],
287 "notPassed": entry["notPassed"],
288 }
289 for validator_id, entry in sorted(validators.items())
290 ],
291 "changeSets": sum(
292 len(services.state.list("change_set", ChangeSet, execution_id=item.execution_id))
293 for item in runs
294 ),
295 }
298def _signal(entry: dict[str, Any]) -> str | None:
299 """A fixed-rule hint, not a verdict: what a person may want to look at."""
300 if entry["blockedGates"] and entry["excepted"] >= max(1, entry["blockedGates"] // 2):
301 return "often excepted when it blocks: check its precision"
302 if entry["outcomeRuns"]: 302 ↛ 304line 302 didn't jump to line 304 because the condition on line 302 was always true
303 return "fired in runs later linked to an outcome"
304 if entry["correctedRuns"]:
305 return "led to corrections"
306 return None