Coverage for src/governed_harness/orchestration/corrections.py: 90%

97 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Reproduce-first corrections and empty corrections (#52, N5; extends #36). 

2 

3In the evaluation half of the correction rounds a small model concluded that the code already 

4complied and changed nothing, and a reported defect was never fixed because the agent's own tests 

5passed. Under ``runtime.reproduceFirst``: 

6 

7* a correction attempt (an IMPLEMENTATION after a failed VERIFICATION or a REQUEST_CHANGES) 

8 that changes nothing gets an ``agent.empty-correction`` finding, with the agent's summary, so 

9 the reviewer sees the claim instead of a silent stop; 

10* a correction after REQUEST_CHANGES must add or change a test that fails on the code as it was 

11 before the correction and passes after it (Python projects, pytest). A correction without such 

12 a test gets a HIGH ``correction.not-reproduced`` finding, which blocks the gate. 

13 

14The findings belong to the ChangeSet the correction produced: VERIFICATION records them in the 

15``harness.reproduce-first`` validation.""" 

16 

17from __future__ import annotations 

18 

19import json 

20from datetime import UTC, datetime 

21from typing import TYPE_CHECKING, Any 

22 

23from governed_harness.agents import AgentExecutionResult 

24from governed_harness.capabilities import grants_from_rules 

25from governed_harness.checks import is_test_path 

26from governed_harness.domain.enums import ( 

27 ActorType, 

28 FindingSeverity, 

29 PhaseId, 

30 ResultStatus, 

31 ValidationKind, 

32) 

33from governed_harness.domain.models import Actor, ChangeSet, Execution, Finding, PhaseExecution 

34from governed_harness.orchestration.workspace_ops import ( 

35 changes_since, 

36 materialized, 

37 snapshot_contents, 

38) 

39from governed_harness.runtime import CancellationToken, SafeProcessRunner, WorkspaceSnapshotter 

40from governed_harness.runtime.process_runner import CommandSpec 

41from governed_harness.runtime.workspace import WorkspaceSnapshot 

42from governed_harness.validators import ValidatorOutput 

43 

44if TYPE_CHECKING: 

45 from governed_harness.orchestration.hosts import ResultsHost 

46 

47REPRODUCE_ID = "harness.reproduce-first" 

48EMPTY_RULE = "agent.empty-correction" 

49NOT_REPRODUCED_RULE = "correction.not-reproduced" 

50 

51 

52class Corrections: 

53 def __init__(self, results: ResultsHost) -> None: 

54 self.results = results 

55 self._started: dict[str, WorkspaceSnapshot] = {} 

56 """The snapshot each correction attempt started from, by its stored reference, while 

57 this process runs the attempt: a stored snapshot whose texts were redacted (#178) 

58 cannot give the content before the correction back.""" 

59 

60 @property 

61 def enabled(self) -> bool: 

62 return bool(self.results.project.runtime.reproduce_first) 

63 

64 def _trigger(self, execution: Execution) -> str | None: 

65 """The trigger of the correction this IMPLEMENTATION attempt answers, if any.""" 

66 events = self.results.s.events.list(execution.execution_id) 

67 for event in reversed(events): 67 ↛ 79line 67 didn't jump to line 79 because the loop on line 67 didn't complete

68 if event.event_type == "phase.started" and event.payload.get("phaseId") == ( 

69 PhaseId.IMPLEMENTATION 

70 ): 

71 # Skip the start of the current attempt, recorded just before this call. 

72 continue 

73 if event.event_type == "correction.authorized": 

74 return str(event.payload.get("trigger") or "CHANGES_REQUESTED") 

75 if event.event_type == "phase.completed" and event.payload.get("phaseId") == ( 75 ↛ 78line 75 didn't jump to line 78 because the condition on line 75 was never true

76 PhaseId.IMPLEMENTATION 

77 ): 

78 return None 

79 return None 

80 

81 def start(self, execution: Execution, phase: PhaseExecution) -> None: 

82 if not self.enabled or phase.attempt < 2: 

83 return 

84 trigger = self._trigger(execution) 

85 if trigger is None: 85 ↛ 86line 85 didn't jump to line 86 because the condition on line 85 was never true

86 return 

87 snapshot = WorkspaceSnapshotter(self.results.s.paths.workspace).snapshot() 

88 ref = self.results.s.artifacts.put_json( 

89 self.results.engine._snapshot_to_dict(snapshot), 

90 metadata={"kind": "correction-start", "executionId": execution.execution_id}, 

91 ) 

92 self._started = {ref.uri: snapshot} 

93 self.results.set_flag_json( 

94 f"correctionstart:{execution.execution_id}", 

95 { 

96 "phaseExecutionId": phase.phase_execution_id, 

97 "snapshotRef": ref.uri, 

98 "trigger": trigger, 

99 }, 

100 ) 

101 self.results.s.state.set_flag(f"reproduce:{execution.execution_id}", "") 

102 

103 def end( 

104 self, execution: Execution, phase: PhaseExecution, result: AgentExecutionResult 

105 ) -> None: 

106 """Compare the workspace with the start of the correction attempt.""" 

107 if not self.enabled: 

108 return 

109 start = self.results.flag_json(f"correctionstart:{execution.execution_id}") 

110 if not isinstance(start, dict) or start.get("phaseExecutionId") != phase.phase_execution_id: 

111 return 

112 engine = self.results.engine 

113 # Read back from the store, a redacted text is never given back (snapshot_contents). 

114 before = self._started.pop(start["snapshotRef"], None) or engine._snapshot_from_dict( 

115 json.loads(self.results.s.artifacts.get(start["snapshotRef"])) 

116 ) 

117 workspace = self.results.s.paths.workspace 

118 diff = changes_since(workspace, before) 

119 findings: list[Finding] = [] 

120 if not diff.changes: 

121 findings.append( 

122 self.results.record_finding( 

123 execution, 

124 validator_id=REPRODUCE_ID, 

125 rule_id=EMPTY_RULE, 

126 category="agent-claim", 

127 severity=FindingSeverity.MEDIUM, 

128 message=( 

129 f"The correction after {start['trigger']} changed nothing; the agent " 

130 f"said: {result.summary[:500]!r}" 

131 ), 

132 evidence_refs=((result.output_ref,) if result.output_ref else ()), 

133 recommendation=( 

134 "Read the agent's explanation: decide again, or request the change " 

135 "with a verifiable condition (review.structuredChanges)." 

136 ), 

137 introduced=True, 

138 ) 

139 ) 

140 elif start["trigger"] == "CHANGES_REQUESTED": 140 ↛ 142line 140 didn't jump to line 142 because the condition on line 140 was always true

141 findings.extend(self._reproduced(execution, before, diff.changes)) 

142 self.results.set_flag_json( 

143 f"reproduce:{execution.execution_id}", 

144 { 

145 "phaseExecutionId": phase.phase_execution_id, 

146 "trigger": start["trigger"], 

147 "changedPaths": [item.path for item in diff.changes], 

148 "findingIds": [item.finding_id for item in findings], 

149 }, 

150 ) 

151 

152 def _reproduced(self, execution: Execution, before: Any, changes: Any) -> list[Finding]: 

153 results = self.results 

154 tests = [ 

155 item.path for item in changes if is_test_path(item.path) and item.status != "DELETED" 

156 ] 

157 sources = [item.path for item in changes if not is_test_path(item.path)] 

158 python = any(item.technology == "python" for item in results.s.resolved.profiles) 

159 if not python: 159 ↛ 160line 159 didn't jump to line 160 because the condition on line 159 was never true

160 return [ 

161 results.record_finding( 

162 execution, 

163 validator_id=REPRODUCE_ID, 

164 rule_id="correction.reproduce-unavailable", 

165 category="corrections", 

166 severity=FindingSeverity.INFO, 

167 message="The reproduce-first check runs pytest; this project is not Python", 

168 ) 

169 ] 

170 python_tests = [path for path in tests if path.endswith(".py")] 

171 if not python_tests or not sources: 

172 return [ 

173 results.record_finding( 

174 execution, 

175 validator_id=REPRODUCE_ID, 

176 rule_id=NOT_REPRODUCED_RULE, 

177 category="corrections", 

178 severity=FindingSeverity.HIGH, 

179 message=( 

180 "The correction after REQUEST_CHANGES adds no test that reproduces the " 

181 "reported problem" 

182 if not python_tests 

183 else "The correction changes only tests; nothing was corrected" 

184 ), 

185 recommendation=( 

186 "Add a test that fails on the code before the correction and passes " 

187 "after it, then correct the code." 

188 ), 

189 introduced=True, 

190 ) 

191 ] 

192 engine = results.engine 

193 workspace = results.s.paths.workspace 

194 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{REPRODUCE_ID}", version="1") 

195 grants = grants_from_rules( 

196 execution.execution_id, actor, results.s.resolved.effective_capabilities 

197 ) 

198 argv = ("python", "-m", "pytest", "-q", *python_tests) 

199 record: dict[str, Any] = {"tests": python_tests, "revertedSources": sources} 

200 with materialized( 

201 workspace, results.s.paths.harness_dir / "tmp", snapshot_contents(before), sources 

202 ) as copy: 

203 if copy is None: 203 ↛ 204line 203 didn't jump to line 204 because the condition on line 203 was never true

204 return [] 

205 outcome = SafeProcessRunner(copy).run( 

206 CommandSpec( 

207 argv=argv, 

208 cwd=copy, 

209 timeout_seconds=float(engine._bounded_timeout(900)), 

210 max_output_bytes=results.project.runtime.max_output_bytes, 

211 ), 

212 actor=actor, 

213 grants=grants, 

214 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)), 

215 ) 

216 record["before"] = { 

217 "status": outcome.status.value, 

218 "exitCode": outcome.exit_code, 

219 "stdoutRef": results.s.artifacts.put(outcome.stdout, media_type="text/plain").uri, 

220 } 

221 ref = results.record_json( 

222 execution, 

223 PhaseId.IMPLEMENTATION, 

224 record, 

225 kind="reproduce-first", 

226 summary=( 

227 f"Reproduce-first: the correction's tests {outcome.status.value} on the code " 

228 "before the correction" 

229 ), 

230 ) 

231 if outcome.status is ResultStatus.FAILED: 231 ↛ 233line 231 didn't jump to line 233 because the condition on line 231 was always true

232 return [] 

233 return [ 

234 results.record_finding( 

235 execution, 

236 validator_id=REPRODUCE_ID, 

237 rule_id=NOT_REPRODUCED_RULE, 

238 category="corrections", 

239 severity=FindingSeverity.HIGH, 

240 message=( 

241 "The tests the correction added or changed " 

242 f"({', '.join(python_tests[:5])}) do not fail on the code before the " 

243 f"correction ({outcome.status.value}): they do not reproduce the problem" 

244 ), 

245 evidence_refs=(ref,), 

246 recommendation="Write a test that fails without the correction.", 

247 introduced=True, 

248 ) 

249 ] 

250 

251 def validation(self, execution: Execution, change_set: ChangeSet) -> ValidatorOutput | None: 

252 """The findings of the latest correction attempt as a validation of the ChangeSet.""" 

253 if not self.enabled: 

254 return None 

255 value = self.results.flag_json(f"reproduce:{execution.execution_id}") 

256 if not isinstance(value, dict): 

257 return None 

258 findings = tuple( 

259 self.results.s.state.get("finding", item, Finding) for item in value["findingIds"] 

260 ) 

261 blocking = [ 

262 item 

263 for item in findings 

264 if item.severity in {FindingSeverity.HIGH, FindingSeverity.CRITICAL} 

265 ] 

266 result = self.results.record_validation( 

267 execution, 

268 validator_id=REPRODUCE_ID, 

269 digest=change_set.digest, 

270 status=ResultStatus.FAILED if blocking else ResultStatus.PASSED, 

271 kind=ValidationKind.VALIDATION_FAILURE if blocking else ValidationKind.SUCCESS, 

272 mandatory=True, 

273 summary=( 

274 f"Correction after {value['trigger']}: {len(findings)} finding(s)" 

275 if findings 

276 else f"Correction after {value['trigger']} reproduced and changed the code" 

277 ), 

278 findings=findings, 

279 evidence_refs=(change_set.diff_ref,), 

280 started_at=datetime.now(UTC), 

281 ) 

282 return ValidatorOutput(result, ())