Coverage for src/governed_harness/orchestration/corrections.py: 90%
97 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Reproduce-first corrections and empty corrections (#52, N5; extends #36).
3In the evaluation half of the correction rounds a small model concluded that the code already
4complied and changed nothing, and a reported defect was never fixed because the agent's own tests
5passed. Under ``runtime.reproduceFirst``:
7* a correction attempt (an IMPLEMENTATION after a failed VERIFICATION or a REQUEST_CHANGES)
8 that changes nothing gets an ``agent.empty-correction`` finding, with the agent's summary, so
9 the reviewer sees the claim instead of a silent stop;
10* a correction after REQUEST_CHANGES must add or change a test that fails on the code as it was
11 before the correction and passes after it (Python projects, pytest). A correction without such
12 a test gets a HIGH ``correction.not-reproduced`` finding, which blocks the gate.
14The findings belong to the ChangeSet the correction produced: VERIFICATION records them in the
15``harness.reproduce-first`` validation."""
17from __future__ import annotations
19import json
20from datetime import UTC, datetime
21from typing import TYPE_CHECKING, Any
23from governed_harness.agents import AgentExecutionResult
24from governed_harness.capabilities import grants_from_rules
25from governed_harness.checks import is_test_path
26from governed_harness.domain.enums import (
27 ActorType,
28 FindingSeverity,
29 PhaseId,
30 ResultStatus,
31 ValidationKind,
32)
33from governed_harness.domain.models import Actor, ChangeSet, Execution, Finding, PhaseExecution
34from governed_harness.orchestration.workspace_ops import (
35 changes_since,
36 materialized,
37 snapshot_contents,
38)
39from governed_harness.runtime import CancellationToken, SafeProcessRunner, WorkspaceSnapshotter
40from governed_harness.runtime.process_runner import CommandSpec
41from governed_harness.runtime.workspace import WorkspaceSnapshot
42from governed_harness.validators import ValidatorOutput
44if TYPE_CHECKING:
45 from governed_harness.orchestration.hosts import ResultsHost
47REPRODUCE_ID = "harness.reproduce-first"
48EMPTY_RULE = "agent.empty-correction"
49NOT_REPRODUCED_RULE = "correction.not-reproduced"
52class Corrections:
53 def __init__(self, results: ResultsHost) -> None:
54 self.results = results
55 self._started: dict[str, WorkspaceSnapshot] = {}
56 """The snapshot each correction attempt started from, by its stored reference, while
57 this process runs the attempt: a stored snapshot whose texts were redacted (#178)
58 cannot give the content before the correction back."""
60 @property
61 def enabled(self) -> bool:
62 return bool(self.results.project.runtime.reproduce_first)
64 def _trigger(self, execution: Execution) -> str | None:
65 """The trigger of the correction this IMPLEMENTATION attempt answers, if any."""
66 events = self.results.s.events.list(execution.execution_id)
67 for event in reversed(events): 67 ↛ 79line 67 didn't jump to line 79 because the loop on line 67 didn't complete
68 if event.event_type == "phase.started" and event.payload.get("phaseId") == (
69 PhaseId.IMPLEMENTATION
70 ):
71 # Skip the start of the current attempt, recorded just before this call.
72 continue
73 if event.event_type == "correction.authorized":
74 return str(event.payload.get("trigger") or "CHANGES_REQUESTED")
75 if event.event_type == "phase.completed" and event.payload.get("phaseId") == ( 75 ↛ 78line 75 didn't jump to line 78 because the condition on line 75 was never true
76 PhaseId.IMPLEMENTATION
77 ):
78 return None
79 return None
81 def start(self, execution: Execution, phase: PhaseExecution) -> None:
82 if not self.enabled or phase.attempt < 2:
83 return
84 trigger = self._trigger(execution)
85 if trigger is None: 85 ↛ 86line 85 didn't jump to line 86 because the condition on line 85 was never true
86 return
87 snapshot = WorkspaceSnapshotter(self.results.s.paths.workspace).snapshot()
88 ref = self.results.s.artifacts.put_json(
89 self.results.engine._snapshot_to_dict(snapshot),
90 metadata={"kind": "correction-start", "executionId": execution.execution_id},
91 )
92 self._started = {ref.uri: snapshot}
93 self.results.set_flag_json(
94 f"correctionstart:{execution.execution_id}",
95 {
96 "phaseExecutionId": phase.phase_execution_id,
97 "snapshotRef": ref.uri,
98 "trigger": trigger,
99 },
100 )
101 self.results.s.state.set_flag(f"reproduce:{execution.execution_id}", "")
103 def end(
104 self, execution: Execution, phase: PhaseExecution, result: AgentExecutionResult
105 ) -> None:
106 """Compare the workspace with the start of the correction attempt."""
107 if not self.enabled:
108 return
109 start = self.results.flag_json(f"correctionstart:{execution.execution_id}")
110 if not isinstance(start, dict) or start.get("phaseExecutionId") != phase.phase_execution_id:
111 return
112 engine = self.results.engine
113 # Read back from the store, a redacted text is never given back (snapshot_contents).
114 before = self._started.pop(start["snapshotRef"], None) or engine._snapshot_from_dict(
115 json.loads(self.results.s.artifacts.get(start["snapshotRef"]))
116 )
117 workspace = self.results.s.paths.workspace
118 diff = changes_since(workspace, before)
119 findings: list[Finding] = []
120 if not diff.changes:
121 findings.append(
122 self.results.record_finding(
123 execution,
124 validator_id=REPRODUCE_ID,
125 rule_id=EMPTY_RULE,
126 category="agent-claim",
127 severity=FindingSeverity.MEDIUM,
128 message=(
129 f"The correction after {start['trigger']} changed nothing; the agent "
130 f"said: {result.summary[:500]!r}"
131 ),
132 evidence_refs=((result.output_ref,) if result.output_ref else ()),
133 recommendation=(
134 "Read the agent's explanation: decide again, or request the change "
135 "with a verifiable condition (review.structuredChanges)."
136 ),
137 introduced=True,
138 )
139 )
140 elif start["trigger"] == "CHANGES_REQUESTED": 140 ↛ 142line 140 didn't jump to line 142 because the condition on line 140 was always true
141 findings.extend(self._reproduced(execution, before, diff.changes))
142 self.results.set_flag_json(
143 f"reproduce:{execution.execution_id}",
144 {
145 "phaseExecutionId": phase.phase_execution_id,
146 "trigger": start["trigger"],
147 "changedPaths": [item.path for item in diff.changes],
148 "findingIds": [item.finding_id for item in findings],
149 },
150 )
152 def _reproduced(self, execution: Execution, before: Any, changes: Any) -> list[Finding]:
153 results = self.results
154 tests = [
155 item.path for item in changes if is_test_path(item.path) and item.status != "DELETED"
156 ]
157 sources = [item.path for item in changes if not is_test_path(item.path)]
158 python = any(item.technology == "python" for item in results.s.resolved.profiles)
159 if not python: 159 ↛ 160line 159 didn't jump to line 160 because the condition on line 159 was never true
160 return [
161 results.record_finding(
162 execution,
163 validator_id=REPRODUCE_ID,
164 rule_id="correction.reproduce-unavailable",
165 category="corrections",
166 severity=FindingSeverity.INFO,
167 message="The reproduce-first check runs pytest; this project is not Python",
168 )
169 ]
170 python_tests = [path for path in tests if path.endswith(".py")]
171 if not python_tests or not sources:
172 return [
173 results.record_finding(
174 execution,
175 validator_id=REPRODUCE_ID,
176 rule_id=NOT_REPRODUCED_RULE,
177 category="corrections",
178 severity=FindingSeverity.HIGH,
179 message=(
180 "The correction after REQUEST_CHANGES adds no test that reproduces the "
181 "reported problem"
182 if not python_tests
183 else "The correction changes only tests; nothing was corrected"
184 ),
185 recommendation=(
186 "Add a test that fails on the code before the correction and passes "
187 "after it, then correct the code."
188 ),
189 introduced=True,
190 )
191 ]
192 engine = results.engine
193 workspace = results.s.paths.workspace
194 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{REPRODUCE_ID}", version="1")
195 grants = grants_from_rules(
196 execution.execution_id, actor, results.s.resolved.effective_capabilities
197 )
198 argv = ("python", "-m", "pytest", "-q", *python_tests)
199 record: dict[str, Any] = {"tests": python_tests, "revertedSources": sources}
200 with materialized(
201 workspace, results.s.paths.harness_dir / "tmp", snapshot_contents(before), sources
202 ) as copy:
203 if copy is None: 203 ↛ 204line 203 didn't jump to line 204 because the condition on line 203 was never true
204 return []
205 outcome = SafeProcessRunner(copy).run(
206 CommandSpec(
207 argv=argv,
208 cwd=copy,
209 timeout_seconds=float(engine._bounded_timeout(900)),
210 max_output_bytes=results.project.runtime.max_output_bytes,
211 ),
212 actor=actor,
213 grants=grants,
214 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)),
215 )
216 record["before"] = {
217 "status": outcome.status.value,
218 "exitCode": outcome.exit_code,
219 "stdoutRef": results.s.artifacts.put(outcome.stdout, media_type="text/plain").uri,
220 }
221 ref = results.record_json(
222 execution,
223 PhaseId.IMPLEMENTATION,
224 record,
225 kind="reproduce-first",
226 summary=(
227 f"Reproduce-first: the correction's tests {outcome.status.value} on the code "
228 "before the correction"
229 ),
230 )
231 if outcome.status is ResultStatus.FAILED: 231 ↛ 233line 231 didn't jump to line 233 because the condition on line 231 was always true
232 return []
233 return [
234 results.record_finding(
235 execution,
236 validator_id=REPRODUCE_ID,
237 rule_id=NOT_REPRODUCED_RULE,
238 category="corrections",
239 severity=FindingSeverity.HIGH,
240 message=(
241 "The tests the correction added or changed "
242 f"({', '.join(python_tests[:5])}) do not fail on the code before the "
243 f"correction ({outcome.status.value}): they do not reproduce the problem"
244 ),
245 evidence_refs=(ref,),
246 recommendation="Write a test that fails without the correction.",
247 introduced=True,
248 )
249 ]
251 def validation(self, execution: Execution, change_set: ChangeSet) -> ValidatorOutput | None:
252 """The findings of the latest correction attempt as a validation of the ChangeSet."""
253 if not self.enabled:
254 return None
255 value = self.results.flag_json(f"reproduce:{execution.execution_id}")
256 if not isinstance(value, dict):
257 return None
258 findings = tuple(
259 self.results.s.state.get("finding", item, Finding) for item in value["findingIds"]
260 )
261 blocking = [
262 item
263 for item in findings
264 if item.severity in {FindingSeverity.HIGH, FindingSeverity.CRITICAL}
265 ]
266 result = self.results.record_validation(
267 execution,
268 validator_id=REPRODUCE_ID,
269 digest=change_set.digest,
270 status=ResultStatus.FAILED if blocking else ResultStatus.PASSED,
271 kind=ValidationKind.VALIDATION_FAILURE if blocking else ValidationKind.SUCCESS,
272 mandatory=True,
273 summary=(
274 f"Correction after {value['trigger']}: {len(findings)} finding(s)"
275 if findings
276 else f"Correction after {value['trigger']} reproduced and changed the code"
277 ),
278 findings=findings,
279 evidence_refs=(change_set.diff_ref,),
280 started_at=datetime.now(UTC),
281 )
282 return ValidatorOutput(result, ())