Coverage for src/governed_harness/orchestration/acceptance_dispute.py: 91%

134 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Disputed frozen acceptance tests (``verification.acceptanceTests.disputes: person``, #94). 

2 

3The frozen acceptance tests are written by an agent call and approved by a person from their 

4digest; they can be wrong. In the wave 9 pilot a correct change failed VERIFICATION three times 

5against a test that expected ``b"Hellon"`` for ``b"Hello"`` and against inputs the clarified task 

6says must be rejected, and the only outcome the run had was to fail. 

7 

8Under ``disputes: person`` the implement request tells the agent not to change a frozen test it 

9believes is wrong but to list it, with the reason, in its self-report 

10(``disputedAcceptanceTests``). When VERIFICATION then fails **only** on the frozen acceptance 

11tests (no frozen file was changed, and any other mandatory validator that failed, such as the 

12project's own test command, locates all its failures in the frozen files) and the self-report 

13of the implementation that produced the ChangeSet carries a dispute, the run waits for a person 

14(``BLOCKED``, an ``acceptance-dispute`` entry in the inbox) instead of failing: 

15 

16* ``UPHOLD``: the tests stand. VERIFICATION runs again, fails as before and the run goes on to 

17 the correction cycles or stops, as it would have without the dispute; the next implement 

18 request says the person upheld the tests. 

19* ``REVISE``: the frozen files are removed and the run returns to SPECIFICATION, where the 

20 acceptance call writes the tests again knowing what was disputed, a person approves them from 

21 their digest and the harness freezes them (the earlier ones stay recorded with their digests). 

22 The plan and the implementation stand: the run goes from SPECIFICATION straight back to 

23 VERIFICATION of the same change. 

24 

25The decision is a person's, bound to the digest of the dispute (the ChangeSet, the approved tests 

26and what the agent disputed). A self-report is the agent's word (``REPORTED``): it opens the 

27wait, it never changes a test.""" 

28 

29from __future__ import annotations 

30 

31from enum import StrEnum 

32from typing import TYPE_CHECKING, Any 

33 

34from governed_harness.domain.enums import ActorType, PhaseId, ResultStatus 

35from governed_harness.domain.errors import NotFoundError, PolicyViolationError 

36from governed_harness.domain.models import ( 

37 Actor, 

38 AgentSelfReport, 

39 ChangeSet, 

40 Execution, 

41 Finding, 

42 ValidationResult, 

43 utc_now, 

44) 

45from governed_harness.evidence import sha256_json 

46from governed_harness.orchestration.acceptance import ACCEPTANCE_ID, AcceptanceTests 

47from governed_harness.orchestration.engine_types import PhaseOutcome 

48 

49if TYPE_CHECKING: 

50 from governed_harness.orchestration.hosts import ResultsHost 

51 

52DISPUTE_FLAG = "acceptdispute" 

53RESUME_FLAG = "acceptresume" 

54DISPUTE_KIND = "acceptance-dispute" 

55DISPUTE_NOTE = ( 

56 "If you believe one of these tests is wrong (it contradicts the task or asserts a wrong " 

57 "value), do not change it and do not write code to satisfy it: list it in your self-report " 

58 'under disputedAcceptanceTests as [{"test": "<file or test id>", "reason": "<why>"}]. A ' 

59 "person decides whether the test stands or is written again." 

60) 

61UPHELD_NOTE = ( 

62 "A person reviewed your dispute of these tests and upheld them: they stand, make them pass." 

63) 

64 

65 

66class DisputeDecision(StrEnum): 

67 UPHOLD = "UPHOLD" 

68 REVISE = "REVISE" 

69 

70 

71def decide_command(execution_id: str, digest: str) -> str: 

72 return ( 

73 f"harness acceptance dispute --run {execution_id} --decision <UPHOLD|REVISE> " 

74 f"--digest {digest}" 

75 ) 

76 

77 

78class AcceptanceDisputes: 

79 def __init__(self, results: ResultsHost, acceptance: AcceptanceTests) -> None: 

80 self.results = results 

81 self.acceptance = acceptance 

82 

83 @property 

84 def enabled(self) -> bool: 

85 config = self.acceptance.config 

86 return bool(config and config.enabled and config.disputes_enabled) 

87 

88 def _key(self, execution_id: str) -> str: 

89 return f"{DISPUTE_FLAG}:{execution_id}" 

90 

91 def state(self, execution: Execution) -> dict[str, Any] | None: 

92 value = self.results.flag_json(self._key(execution.execution_id)) 

93 return value if isinstance(value, dict) else None 

94 

95 # ----- the implement request ------------------------------------------------------------- 

96 def request_extra(self, execution: Execution) -> dict[str, Any]: 

97 """What the ``acceptanceTests`` block of the implement request adds: how to dispute a 

98 test and, after a person upheld a dispute, that the tests stand.""" 

99 if not self.enabled: 

100 return {} 

101 extra: dict[str, Any] = {"dispute": DISPUTE_NOTE} 

102 state = self.state(execution) 

103 if state is not None and state.get("status") == "UPHELD": 

104 extra["upheld"] = { 

105 "tests": state["disputed"], 

106 "rationale": state.get("rationale"), 

107 "note": UPHELD_NOTE, 

108 } 

109 return extra 

110 

111 # ----- VERIFICATION ---------------------------------------------------------------------- 

112 def disputable(self, execution: Execution) -> frozenset[str]: 

113 """The frozen acceptance files a dispute can cover; empty without ``disputes: person``. 

114 The fast lane leaves them out of the affected tests it runs first (#161): a failure 

115 there would stop the attempt before ``harness.acceptance-tests`` runs, and a dispute 

116 needs that result. They still run, in that validator and in the full suite.""" 

117 if not self.enabled: 

118 return frozenset() 

119 state = self.acceptance.state(execution) or {} 

120 if state.get("status") != "APPROVED": 

121 return frozenset() 

122 return frozenset(state.get("frozen") or {}) 

123 

124 def waiting(self, execution: Execution, change_set: ChangeSet) -> PhaseOutcome | None: 

125 """The wait of an open dispute of this ChangeSet, without running the validators 

126 again.""" 

127 state = self.state(execution) if self.enabled else None 

128 if state is None or state.get("status") != "OPEN": 

129 return None 

130 if state.get("changeSetDigest") != change_set.digest: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true

131 return None 

132 return self._blocked(execution, state) 

133 

134 def _blocked(self, execution: Execution, state: dict[str, Any]) -> PhaseOutcome: 

135 return PhaseOutcome( 

136 ResultStatus.BLOCKED, 

137 f"The agent disputes {len(state['disputed'])} frozen acceptance test(s) and only " 

138 "they fail; a person decides: " 

139 + decide_command(execution.execution_id, state["digest"]), 

140 (state["ref"],), 

141 ) 

142 

143 def open( 

144 self, 

145 execution: Execution, 

146 change_set: ChangeSet, 

147 failing: list[ValidationResult], 

148 ) -> PhaseOutcome | None: 

149 """Open a dispute when the frozen acceptance tests are the only mandatory validation 

150 that did not pass and the agent disputed them; ``None`` otherwise (the run fails as it 

151 did before).""" 

152 if not self.enabled or not self._only_acceptance(execution, failing): 

153 return None 

154 previous = self.state(execution) 

155 if previous is not None and previous.get("changeSetDigest") == change_set.digest: 

156 return None # a person already decided the dispute of this ChangeSet 

157 report = self._report(execution, change_set) 

158 if report is None: 

159 return None 

160 acceptance = self.acceptance.state(execution) or {} 

161 disputed = [ 

162 {"test": item.test, "reason": item.reason} for item in report.disputed_acceptance_tests 

163 ] 

164 record: dict[str, Any] = { 

165 "status": "OPEN", 

166 "changeSetDigest": change_set.digest, 

167 "acceptanceDigest": acceptance.get("digest"), 

168 "frozen": acceptance.get("frozen") or {}, 

169 "disputed": disputed, 

170 "reportId": report.report_id, 

171 "invocationId": report.invocation_id, 

172 "validation": failing[0].summary, 

173 } 

174 record["digest"] = sha256_json( 

175 {key: record[key] for key in ("changeSetDigest", "acceptanceDigest", "disputed")} 

176 ) 

177 ref = self.results.record_json( 

178 execution, 

179 PhaseId.VERIFICATION, 

180 record, 

181 kind=DISPUTE_KIND, 

182 summary=f"Acceptance tests disputed by the agent: {len(disputed)}", 

183 ) 

184 state = {**record, "ref": ref} 

185 self.results.set_flag_json(self._key(execution.execution_id), state) 

186 self.results.s.events.append( 

187 execution.execution_id, 

188 "acceptance.dispute.opened", 

189 { 

190 "digest": record["digest"], 

191 "changeSetDigest": change_set.digest, 

192 "disputed": [item["test"] for item in disputed], 

193 "evidenceRef": ref, 

194 }, 

195 ) 

196 return self._blocked(execution, state) 

197 

198 def _only_acceptance(self, execution: Execution, failing: list[ValidationResult]) -> bool: 

199 """Whether the failed frozen tests are all that failed, with every frozen file as it 

200 was approved (a changed frozen file is the agent's fault, not a dispute). Another 

201 mandatory validator that failed counts as the same failure only when every failure it 

202 locates is in a frozen file: the project's own test command also collects them.""" 

203 frozen = (self.acceptance.state(execution) or {}).get("frozen") or {} 

204 if not frozen or self.acceptance.untouched(execution) != set(frozen): 204 ↛ 205line 204 didn't jump to line 205 because the condition on line 204 was never true

205 return False 

206 if any(item.status is not ResultStatus.FAILED for item in failing): 206 ↛ 207line 206 didn't jump to line 207 because the condition on line 206 was never true

207 return False 

208 others = [item for item in failing if item.validator_id != ACCEPTANCE_ID] 

209 if len(others) == len(failing): 209 ↛ 210line 209 didn't jump to line 210 because the condition on line 209 was never true

210 return False 

211 return all(self._explained(item, frozen) for item in others) 

212 

213 def _explained(self, validation: ValidationResult, frozen: dict[str, str]) -> bool: 

214 """Whether a failed validation locates its failures, all of them in frozen files 

215 (needs parsed output, ``verification.outputParsers``).""" 

216 located: list[str] = [] 

217 for finding_id in validation.finding_ids: 

218 try: 

219 finding = self.results.s.state.get("finding", finding_id, Finding) 

220 except NotFoundError: 

221 return False 

222 if finding.location is not None and finding.location.path: 

223 located.append(finding.location.path) 

224 return bool(located) and all(path in frozen for path in located) 

225 

226 def _report(self, execution: Execution, change_set: ChangeSet) -> AgentSelfReport | None: 

227 """The self-report of the implementation that produced this ChangeSet, when it 

228 disputes a test.""" 

229 reports = [ 

230 item 

231 for item in self.results.s.state.list( 

232 "agent_self_report", AgentSelfReport, execution_id=execution.execution_id 

233 ) 

234 if item.contrast is not None and item.contrast.change_set_digest == change_set.digest 

235 ] 

236 if not reports: 236 ↛ 237line 236 didn't jump to line 237 because the condition on line 236 was never true

237 return None 

238 latest = max(reports, key=lambda item: item.created_at) 

239 return latest if latest.disputed_acceptance_tests else None 

240 

241 # ----- the decision (a person) ----------------------------------------------------------- 

242 def decide( 

243 self, 

244 execution: Execution, 

245 *, 

246 decision: DisputeDecision, 

247 digest: str, 

248 actor: Actor, 

249 rationale: str, 

250 ) -> dict[str, Any]: 

251 if actor.actor_type is not ActorType.HUMAN: 251 ↛ 252line 251 didn't jump to line 252 because the condition on line 251 was never true

252 raise PolicyViolationError("only a person decides a dispute of acceptance tests") 

253 state = self.state(execution) 

254 if state is None or state.get("status") != "OPEN": 

255 raise NotFoundError( 

256 f"run {execution.execution_id} has no open dispute of acceptance tests" 

257 ) 

258 if state["digest"] != digest: 

259 raise PolicyViolationError("the digest does not match the dispute") 

260 if not rationale.strip(): 260 ↛ 261line 260 didn't jump to line 261 because the condition on line 260 was never true

261 raise PolicyViolationError("a rationale is required") 

262 revise = decision is DisputeDecision.REVISE 

263 decided = { 

264 **state, 

265 "status": "REVISED" if revise else "UPHELD", 

266 "decidedBy": actor.actor_id, 

267 "rationale": rationale, 

268 "decidedAt": utc_now().isoformat(), 

269 } 

270 self.results.set_flag_json(self._key(execution.execution_id), decided) 

271 payload: dict[str, Any] = {"decision": decision.value, "digest": digest} 

272 if revise: 

273 payload["invalidatedPhases"] = self._revise(execution, decided) 

274 self.results.s.events.append( 

275 execution.execution_id, "acceptance.dispute.decided", payload, actor=actor 

276 ) 

277 return decided 

278 

279 def _revise(self, execution: Execution, decided: dict[str, Any]) -> list[str]: 

280 """Drop the frozen tests and send the run back to SPECIFICATION.""" 

281 self.acceptance.revise( 

282 execution, 

283 { 

284 "digest": decided["digest"], 

285 "disputed": decided["disputed"], 

286 "rationale": decided["rationale"], 

287 "decidedBy": decided["decidedBy"], 

288 }, 

289 ) 

290 engine = self.results.engine 

291 transition = engine.state_machine.authorize_acceptance_revision(PhaseId.VERIFICATION) 

292 latest = engine.get_execution(execution.execution_id) 

293 engine._save_execution( 

294 latest.model_copy( 

295 update={ 

296 "status": ResultStatus.PENDING, 

297 "current_phase": transition.target, 

298 "gate_evaluation_id": None, 

299 "human_decision_id": None, 

300 "terminal_reason": None, 

301 "updated_at": utc_now(), 

302 } 

303 ) 

304 ) 

305 self.results.s.state.set_flag(f"{RESUME_FLAG}:{execution.execution_id}", "1") 

306 return [phase.value for phase in transition.invalidated] 

307 

308 # ----- after the revision ---------------------------------------------------------------- 

309 def resume_target(self, execution_id: str) -> PhaseId | None: 

310 """VERIFICATION, once, for the SPECIFICATION that passed after a revision: the plan 

311 and the implementation stand, so the run does not plan or implement again.""" 

312 key = f"{RESUME_FLAG}:{execution_id}" 

313 if not self.results.s.state.get_flag(key): 

314 return None 

315 self.results.s.state.set_flag(key, "") 

316 self.results.s.events.append( 

317 execution_id, 

318 "acceptance.revision.resumed", 

319 {"from": PhaseId.SPECIFICATION.value, "to": PhaseId.VERIFICATION.value}, 

320 ) 

321 return PhaseId.VERIFICATION 

322 

323 

324__all__ = [ 

325 "DISPUTE_FLAG", 

326 "DISPUTE_KIND", 

327 "AcceptanceDisputes", 

328 "DisputeDecision", 

329 "decide_command", 

330]