Coverage for src/governed_harness/orchestration/lessons.py: 96%

95 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Lessons from recurring findings across steps (#43). 

2 

3In a multi-step plan the agent repeated mistakes an earlier step had already corrected (literal 

4secrets in tests after a correction), paying again for the same correction. Under 

5``memory.learnFromFindings: auto``: 

6 

7* when a run closes, the findings that caused a correction (rule, validator, the paths and a 

8 short lesson) are proposed as project memory records (``lesson:<validator>:<rule>``) with 

9 provenance (the run, the findings, the correction events). A lesson seen in 

10 ``memory.recurrenceRuns`` runs or more (default 2) is approved automatically only under 

11 ``memory.autoApproveRecurring``; otherwise it waits for a person (``harness memory 

12 approve``); 

13* before IMPLEMENTATION, the active (approved, unexpired) lessons relevant to the task (a rule 

14 of a validator or check the project runs, or a path the task touches) are added to the 

15 provider request as ``lessons`` and listed in the context manifest; each use is an 

16 event.""" 

17 

18from __future__ import annotations 

19 

20import fnmatch 

21from typing import TYPE_CHECKING, Any 

22 

23from governed_harness.domain.enums import ActorType, FindingSeverity, MemoryLevel, ResultStatus 

24from governed_harness.domain.ids import new_id 

25from governed_harness.domain.models import ( 

26 Actor, 

27 Execution, 

28 Finding, 

29 MemoryRecord, 

30 PhaseExecution, 

31 Task, 

32 ValidationResult, 

33 utc_now, 

34) 

35from governed_harness.memory import MemoryStore 

36from governed_harness.memory.store import is_effective 

37 

38if TYPE_CHECKING: 

39 from governed_harness.orchestration.hosts import ResultsHost 

40 

41LESSON_PREFIX = "lesson:" 

42DEFAULT_RECURRENCE_RUNS = 2 

43DEFAULT_MAX_LESSONS = 10 

44_SEVERE = {FindingSeverity.MEDIUM, FindingSeverity.HIGH, FindingSeverity.CRITICAL} 

45_BLOCKING = {FindingSeverity.HIGH, FindingSeverity.CRITICAL} 

46LESSON_ACTOR = Actor(actor_type=ActorType.HARNESS, actor_id="harness.lessons", version="1") 

47 

48 

49def lesson_text(finding: Finding) -> str: 

50 """A short, reusable statement of what to avoid, from the finding's own words.""" 

51 advice = f" {finding.recommendation}" if finding.recommendation else "" 

52 return f"Avoid {finding.rule_id}: {finding.message[:240]}.{advice[:240]}".strip() 

53 

54 

55class Lessons: 

56 def __init__(self, results: ResultsHost) -> None: 

57 self.results = results 

58 

59 @property 

60 def config(self) -> Any: 

61 return self.results.project.memory 

62 

63 @property 

64 def enabled(self) -> bool: 

65 return bool(self.config and self.config.enabled) 

66 

67 # ----- proposals ----------------------------------------------------------------------- 

68 def corrected_findings(self, execution: Execution) -> list[Finding]: 

69 """Findings of the ChangeSets a correction was authorized for (MEDIUM or worse).""" 

70 state = self.results.s.state 

71 digests: set[str] = set() 

72 for event in self.results.s.events.list(execution.execution_id): 

73 if event.event_type != "correction.authorized": 

74 continue 

75 digest = event.payload.get("changeSetDigest") or event.payload.get( 

76 "approvedChangeSetDigest" 

77 ) 

78 if isinstance(digest, str): 78 ↛ 72line 78 didn't jump to line 72 because the condition on line 78 was always true

79 digests.add(digest) 

80 if not digests: 

81 return [] 

82 validations = [ 

83 item 

84 for item in state.list( 

85 "validation", ValidationResult, execution_id=execution.execution_id 

86 ) 

87 if item.change_set_digest in digests 

88 ] 

89 # What caused the correction: the findings of mandatory validations that did not pass, 

90 # and blocking findings of any validation. Optional validators' findings did not. 

91 failing = { 

92 finding_id 

93 for item in validations 

94 if item.mandatory and item.status is not ResultStatus.PASSED 

95 for finding_id in item.finding_ids 

96 } 

97 ids = {finding_id for item in validations for finding_id in item.finding_ids} 

98 return [ 

99 item 

100 for item in state.list("finding", Finding, execution_id=execution.execution_id) 

101 if item.finding_id in ids 

102 and item.introduced is not False 

103 and ( 

104 (item.finding_id in failing and item.severity in _SEVERE) 

105 or item.severity in _BLOCKING 

106 ) 

107 ] 

108 

109 def propose(self, execution: Execution) -> list[MemoryRecord]: 

110 """Propose (or recur) one lesson per (validator, rule) that caused a correction.""" 

111 if not self.enabled: 

112 return [] 

113 findings = self.corrected_findings(execution) 

114 groups: dict[tuple[str, str], list[Finding]] = {} 

115 for item in findings: 

116 groups.setdefault((item.validator_id, item.rule_id), []).append(item) 

117 store = MemoryStore(self.results.s.state) 

118 existing = { 

119 record.key: record 

120 for record in sorted( 

121 store.list_project(execution.project_id), key=lambda item: item.created_at 

122 ) 

123 if record.key.startswith(LESSON_PREFIX) 

124 } 

125 recurrence = self.config.recurrence_runs or DEFAULT_RECURRENCE_RUNS 

126 proposed: list[MemoryRecord] = [] 

127 for (validator_id, rule_id), items in sorted(groups.items()): 

128 key = f"{LESSON_PREFIX}{validator_id}:{rule_id}" 

129 record = self._recur( 

130 execution, store, validator_id, rule_id, items, existing.get(key), recurrence 

131 ) 

132 if record is not None: 132 ↛ 127line 132 didn't jump to line 127 because the condition on line 132 was always true

133 proposed.append(record) 

134 return proposed 

135 

136 def _recur( 

137 self, 

138 execution: Execution, 

139 store: MemoryStore, 

140 validator_id: str, 

141 rule_id: str, 

142 items: list[Finding], 

143 previous: MemoryRecord | None, 

144 recurrence: int, 

145 ) -> MemoryRecord | None: 

146 """Propose the lesson of one (validator, rule), or record its recurrence; ``None`` when 

147 this run already counts for it.""" 

148 key = f"{LESSON_PREFIX}{validator_id}:{rule_id}" 

149 runs = list(previous.value.get("runs", [])) if previous else [] 

150 if execution.execution_id in runs: 150 ↛ 151line 150 didn't jump to line 151 because the condition on line 150 was never true

151 return None 

152 runs.append(execution.execution_id) 

153 approved = bool( 

154 (previous is not None and previous.approved) 

155 or (self.config.auto_approve_recurring and len(runs) >= recurrence) 

156 ) 

157 record = self._lesson_record( 

158 execution, key, validator_id, rule_id, items, previous, runs, recurrence, approved 

159 ) 

160 store.put(record) 

161 self.results.s.events.append( 

162 execution.execution_id, 

163 "lesson.proposed", 

164 { 

165 "memoryId": record.memory_id, 

166 "key": key, 

167 "runs": len(runs), 

168 "approved": approved, 

169 "autoApproved": approved and not (previous and previous.approved), 

170 "findingIds": [item.finding_id for item in items[:10]], 

171 }, 

172 actor=LESSON_ACTOR, 

173 ) 

174 return record 

175 

176 def _lesson_record( 

177 self, 

178 execution: Execution, 

179 key: str, 

180 validator_id: str, 

181 rule_id: str, 

182 items: list[Finding], 

183 previous: MemoryRecord | None, 

184 runs: list[str], 

185 recurrence: int, 

186 approved: bool, 

187 ) -> MemoryRecord: 

188 """The memory record of a lesson: its text, the paths and runs where it applied.""" 

189 paths = sorted( 

190 { 

191 *(previous.value.get("paths", []) if previous else []), 

192 *(item.location.path for item in items if item.location and item.location.path), 

193 } 

194 )[:20] 

195 return MemoryRecord( 

196 memory_id=new_id("mem"), 

197 project_id=execution.project_id, 

198 level=MemoryLevel.PROJECT, 

199 key=key, 

200 value={ 

201 "kind": "lesson", 

202 "validatorId": validator_id, 

203 "ruleId": rule_id, 

204 "lesson": lesson_text(items[0]), 

205 "paths": paths, 

206 "runs": runs, 

207 "occurrences": int(previous.value.get("occurrences", 0) if previous else 0) 

208 + len(items), 

209 "recurring": len(runs) >= recurrence, 

210 }, 

211 provenance=self.results.engine._provenance(execution).model_copy( 

212 update={ 

213 "actor": LESSON_ACTOR, 

214 "source_refs": tuple( 

215 f"record://finding/{item.finding_id}" for item in items[:10] 

216 ), 

217 } 

218 ), 

219 execution_id=execution.execution_id, 

220 supersedes=previous.memory_id if previous else None, 

221 approved=approved, 

222 ) 

223 

224 # ----- use ------------------------------------------------------------------------------- 

225 def active(self, execution: Execution, task: Task, paths: list[str]) -> list[MemoryRecord]: 

226 """Approved, unexpired lessons relevant to the task: a rule of a validator the project 

227 runs, or a path the task touches.""" 

228 if not self.enabled: 

229 return [] 

230 store = MemoryStore(self.results.s.state) 

231 now = utc_now() 

232 records = [ 

233 record 

234 for record in store.list_project(execution.project_id) 

235 if record.key.startswith(LESSON_PREFIX) 

236 and is_effective(record) 

237 and not (record.valid_until and record.valid_until <= now) 

238 ] 

239 superseded = {record.supersedes for record in records if record.supersedes} 

240 records = [record for record in records if record.memory_id not in superseded] 

241 validators = {item.validator_id for item in self.results.s.resolved.effective_validators} 

242 relevant: list[MemoryRecord] = [] 

243 for record in sorted(records, key=lambda item: item.key): 

244 validator = str(record.value.get("validatorId", "")) 

245 lesson_paths = [str(item) for item in record.value.get("paths", [])] 

246 touches = any( 

247 fnmatch.fnmatchcase(path, pattern) or path == pattern 

248 for path in paths 

249 for pattern in lesson_paths 

250 ) 

251 if validator in validators or validator.startswith(("harness.", "review.")) or touches: 251 ↛ 243line 251 didn't jump to line 243 because the condition on line 251 was always true

252 relevant.append(record) 

253 return relevant[: self.config.max_lessons or DEFAULT_MAX_LESSONS] 

254 

255 def for_request( 

256 self, execution: Execution, phase: PhaseExecution, task: Task, paths: list[str] 

257 ) -> list[dict[str, Any]]: 

258 lessons = self.active(execution, task, paths) 

259 if not lessons: 

260 return [] 

261 self.results.s.events.append( 

262 execution.execution_id, 

263 "lesson.applied", 

264 { 

265 "memoryIds": [item.memory_id for item in lessons], 

266 "keys": [item.key for item in lessons], 

267 }, 

268 phase_execution_id=phase.phase_execution_id, 

269 ) 

270 return [ 

271 { 

272 "memoryId": item.memory_id, 

273 "rule": item.value.get("ruleId"), 

274 "validator": item.value.get("validatorId"), 

275 "lesson": item.value.get("lesson"), 

276 "paths": item.value.get("paths", []), 

277 } 

278 for item in lessons 

279 ]