Coverage for src/governed_harness/orchestration/lessons.py: 96%
95 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Lessons from recurring findings across steps (#43).
3In a multi-step plan the agent repeated mistakes an earlier step had already corrected (literal
4secrets in tests after a correction), paying again for the same correction. Under
5``memory.learnFromFindings: auto``:
7* when a run closes, the findings that caused a correction (rule, validator, the paths and a
8 short lesson) are proposed as project memory records (``lesson:<validator>:<rule>``) with
9 provenance (the run, the findings, the correction events). A lesson seen in
10 ``memory.recurrenceRuns`` runs or more (default 2) is approved automatically only under
11 ``memory.autoApproveRecurring``; otherwise it waits for a person (``harness memory
12 approve``);
13* before IMPLEMENTATION, the active (approved, unexpired) lessons relevant to the task (a rule
14 of a validator or check the project runs, or a path the task touches) are added to the
15 provider request as ``lessons`` and listed in the context manifest; each use is an
16 event."""
18from __future__ import annotations
20import fnmatch
21from typing import TYPE_CHECKING, Any
23from governed_harness.domain.enums import ActorType, FindingSeverity, MemoryLevel, ResultStatus
24from governed_harness.domain.ids import new_id
25from governed_harness.domain.models import (
26 Actor,
27 Execution,
28 Finding,
29 MemoryRecord,
30 PhaseExecution,
31 Task,
32 ValidationResult,
33 utc_now,
34)
35from governed_harness.memory import MemoryStore
36from governed_harness.memory.store import is_effective
38if TYPE_CHECKING:
39 from governed_harness.orchestration.hosts import ResultsHost
41LESSON_PREFIX = "lesson:"
42DEFAULT_RECURRENCE_RUNS = 2
43DEFAULT_MAX_LESSONS = 10
44_SEVERE = {FindingSeverity.MEDIUM, FindingSeverity.HIGH, FindingSeverity.CRITICAL}
45_BLOCKING = {FindingSeverity.HIGH, FindingSeverity.CRITICAL}
46LESSON_ACTOR = Actor(actor_type=ActorType.HARNESS, actor_id="harness.lessons", version="1")
49def lesson_text(finding: Finding) -> str:
50 """A short, reusable statement of what to avoid, from the finding's own words."""
51 advice = f" {finding.recommendation}" if finding.recommendation else ""
52 return f"Avoid {finding.rule_id}: {finding.message[:240]}.{advice[:240]}".strip()
55class Lessons:
56 def __init__(self, results: ResultsHost) -> None:
57 self.results = results
59 @property
60 def config(self) -> Any:
61 return self.results.project.memory
63 @property
64 def enabled(self) -> bool:
65 return bool(self.config and self.config.enabled)
67 # ----- proposals -----------------------------------------------------------------------
68 def corrected_findings(self, execution: Execution) -> list[Finding]:
69 """Findings of the ChangeSets a correction was authorized for (MEDIUM or worse)."""
70 state = self.results.s.state
71 digests: set[str] = set()
72 for event in self.results.s.events.list(execution.execution_id):
73 if event.event_type != "correction.authorized":
74 continue
75 digest = event.payload.get("changeSetDigest") or event.payload.get(
76 "approvedChangeSetDigest"
77 )
78 if isinstance(digest, str): 78 ↛ 72line 78 didn't jump to line 72 because the condition on line 78 was always true
79 digests.add(digest)
80 if not digests:
81 return []
82 validations = [
83 item
84 for item in state.list(
85 "validation", ValidationResult, execution_id=execution.execution_id
86 )
87 if item.change_set_digest in digests
88 ]
89 # What caused the correction: the findings of mandatory validations that did not pass,
90 # and blocking findings of any validation. Optional validators' findings did not.
91 failing = {
92 finding_id
93 for item in validations
94 if item.mandatory and item.status is not ResultStatus.PASSED
95 for finding_id in item.finding_ids
96 }
97 ids = {finding_id for item in validations for finding_id in item.finding_ids}
98 return [
99 item
100 for item in state.list("finding", Finding, execution_id=execution.execution_id)
101 if item.finding_id in ids
102 and item.introduced is not False
103 and (
104 (item.finding_id in failing and item.severity in _SEVERE)
105 or item.severity in _BLOCKING
106 )
107 ]
109 def propose(self, execution: Execution) -> list[MemoryRecord]:
110 """Propose (or recur) one lesson per (validator, rule) that caused a correction."""
111 if not self.enabled:
112 return []
113 findings = self.corrected_findings(execution)
114 groups: dict[tuple[str, str], list[Finding]] = {}
115 for item in findings:
116 groups.setdefault((item.validator_id, item.rule_id), []).append(item)
117 store = MemoryStore(self.results.s.state)
118 existing = {
119 record.key: record
120 for record in sorted(
121 store.list_project(execution.project_id), key=lambda item: item.created_at
122 )
123 if record.key.startswith(LESSON_PREFIX)
124 }
125 recurrence = self.config.recurrence_runs or DEFAULT_RECURRENCE_RUNS
126 proposed: list[MemoryRecord] = []
127 for (validator_id, rule_id), items in sorted(groups.items()):
128 key = f"{LESSON_PREFIX}{validator_id}:{rule_id}"
129 record = self._recur(
130 execution, store, validator_id, rule_id, items, existing.get(key), recurrence
131 )
132 if record is not None: 132 ↛ 127line 132 didn't jump to line 127 because the condition on line 132 was always true
133 proposed.append(record)
134 return proposed
136 def _recur(
137 self,
138 execution: Execution,
139 store: MemoryStore,
140 validator_id: str,
141 rule_id: str,
142 items: list[Finding],
143 previous: MemoryRecord | None,
144 recurrence: int,
145 ) -> MemoryRecord | None:
146 """Propose the lesson of one (validator, rule), or record its recurrence; ``None`` when
147 this run already counts for it."""
148 key = f"{LESSON_PREFIX}{validator_id}:{rule_id}"
149 runs = list(previous.value.get("runs", [])) if previous else []
150 if execution.execution_id in runs: 150 ↛ 151line 150 didn't jump to line 151 because the condition on line 150 was never true
151 return None
152 runs.append(execution.execution_id)
153 approved = bool(
154 (previous is not None and previous.approved)
155 or (self.config.auto_approve_recurring and len(runs) >= recurrence)
156 )
157 record = self._lesson_record(
158 execution, key, validator_id, rule_id, items, previous, runs, recurrence, approved
159 )
160 store.put(record)
161 self.results.s.events.append(
162 execution.execution_id,
163 "lesson.proposed",
164 {
165 "memoryId": record.memory_id,
166 "key": key,
167 "runs": len(runs),
168 "approved": approved,
169 "autoApproved": approved and not (previous and previous.approved),
170 "findingIds": [item.finding_id for item in items[:10]],
171 },
172 actor=LESSON_ACTOR,
173 )
174 return record
176 def _lesson_record(
177 self,
178 execution: Execution,
179 key: str,
180 validator_id: str,
181 rule_id: str,
182 items: list[Finding],
183 previous: MemoryRecord | None,
184 runs: list[str],
185 recurrence: int,
186 approved: bool,
187 ) -> MemoryRecord:
188 """The memory record of a lesson: its text, the paths and runs where it applied."""
189 paths = sorted(
190 {
191 *(previous.value.get("paths", []) if previous else []),
192 *(item.location.path for item in items if item.location and item.location.path),
193 }
194 )[:20]
195 return MemoryRecord(
196 memory_id=new_id("mem"),
197 project_id=execution.project_id,
198 level=MemoryLevel.PROJECT,
199 key=key,
200 value={
201 "kind": "lesson",
202 "validatorId": validator_id,
203 "ruleId": rule_id,
204 "lesson": lesson_text(items[0]),
205 "paths": paths,
206 "runs": runs,
207 "occurrences": int(previous.value.get("occurrences", 0) if previous else 0)
208 + len(items),
209 "recurring": len(runs) >= recurrence,
210 },
211 provenance=self.results.engine._provenance(execution).model_copy(
212 update={
213 "actor": LESSON_ACTOR,
214 "source_refs": tuple(
215 f"record://finding/{item.finding_id}" for item in items[:10]
216 ),
217 }
218 ),
219 execution_id=execution.execution_id,
220 supersedes=previous.memory_id if previous else None,
221 approved=approved,
222 )
224 # ----- use -------------------------------------------------------------------------------
225 def active(self, execution: Execution, task: Task, paths: list[str]) -> list[MemoryRecord]:
226 """Approved, unexpired lessons relevant to the task: a rule of a validator the project
227 runs, or a path the task touches."""
228 if not self.enabled:
229 return []
230 store = MemoryStore(self.results.s.state)
231 now = utc_now()
232 records = [
233 record
234 for record in store.list_project(execution.project_id)
235 if record.key.startswith(LESSON_PREFIX)
236 and is_effective(record)
237 and not (record.valid_until and record.valid_until <= now)
238 ]
239 superseded = {record.supersedes for record in records if record.supersedes}
240 records = [record for record in records if record.memory_id not in superseded]
241 validators = {item.validator_id for item in self.results.s.resolved.effective_validators}
242 relevant: list[MemoryRecord] = []
243 for record in sorted(records, key=lambda item: item.key):
244 validator = str(record.value.get("validatorId", ""))
245 lesson_paths = [str(item) for item in record.value.get("paths", [])]
246 touches = any(
247 fnmatch.fnmatchcase(path, pattern) or path == pattern
248 for path in paths
249 for pattern in lesson_paths
250 )
251 if validator in validators or validator.startswith(("harness.", "review.")) or touches: 251 ↛ 243line 251 didn't jump to line 243 because the condition on line 251 was always true
252 relevant.append(record)
253 return relevant[: self.config.max_lessons or DEFAULT_MAX_LESSONS]
255 def for_request(
256 self, execution: Execution, phase: PhaseExecution, task: Task, paths: list[str]
257 ) -> list[dict[str, Any]]:
258 lessons = self.active(execution, task, paths)
259 if not lessons:
260 return []
261 self.results.s.events.append(
262 execution.execution_id,
263 "lesson.applied",
264 {
265 "memoryIds": [item.memory_id for item in lessons],
266 "keys": [item.key for item in lessons],
267 },
268 phase_execution_id=phase.phase_execution_id,
269 )
270 return [
271 {
272 "memoryId": item.memory_id,
273 "rule": item.value.get("ruleId"),
274 "validator": item.value.get("validatorId"),
275 "lesson": item.value.get("lesson"),
276 "paths": item.value.get("paths", []),
277 }
278 for item in lessons
279 ]