Coverage for src/governed_harness/orchestration/acceptance.py: 89%

328 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Independent, frozen acceptance tests (#52, N4). 

2 

3The gate checked the agent's work with the agent's own tests: approved steps of a small model 

4covered some sections at 31-40 %, and a single systematic defect produced hundreds of errors 

5while the agent's fifty tests passed. Under ``verification.acceptanceTests.mode: agent``, in 

6SPECIFICATION a separate call (kind ``acceptance``, read-only, the provider and model of 

7``acceptanceTests.author``) writes acceptance tests from the criteria and requirements only and 

8returns them as files. The harness does not write them yet: a person approves them, bound to 

9their digest (``harness acceptance decide``). On approval the harness writes them under 

10``acceptanceTests.directory``, records their digests (frozen) and runs them on the workspace as it 

11is before the change: a test that already passes there does not test the change (a MEDIUM 

12finding). Every later VERIFICATION checks that the frozen files are unchanged (a modified or 

13deleted file is a HIGH finding) and that they pass; the implement request names them as frozen. 

14A proposal without tests is recorded and the run continues. 

15 

16Since #56, under ``testing.strategy: bdd`` the same call writes the criteria as Gherkin feature 

17files (under ``testing.featuresDirectory``, default ``features``) instead of pytest files: a 

18person approves the scenarios, the harness freezes them and runs the BDD runner of the project 

19(``testing.bddCommand`` or the standards pack's runner) before the change (it must fail: the 

20steps are not defined yet) and in every VERIFICATION; the implement request asks for the step 

21definitions. 

22 

23Since 2.0 (#94), under ``acceptanceTests.protect`` the frozen files are write-protected for the 

24implementing agent: they are among the paths the agent sandbox denies writes to, and a frozen 

25file an agent invocation changed or deleted anyway is put back as approved, with a HIGH 

26``acceptance.write-undone`` finding. Under ``acceptanceTests.disputes`` a person may send a run 

27back to SPECIFICATION to have wrong tests written again (``revise``; see 

28``acceptance_dispute``): the earlier frozen files are removed and stay recorded with their 

29digests under ``previous``, the acceptance call receives what was disputed, and the new tests 

30run once on a copy of the baseline (the workspace already holds the change).""" 

31 

32from __future__ import annotations 

33 

34from pathlib import Path, PurePosixPath 

35from typing import TYPE_CHECKING, Any 

36 

37from governed_harness.capabilities import grants_from_rules 

38from governed_harness.capabilities.authorizer import contained_path 

39from governed_harness.domain.enums import ( 

40 ActorType, 

41 DecisionKind, 

42 FindingSeverity, 

43 PhaseId, 

44 ResultStatus, 

45 ValidationKind, 

46) 

47from governed_harness.domain.errors import NotFoundError, PolicyViolationError 

48from governed_harness.domain.models import ( 

49 Actor, 

50 ChangeSet, 

51 Execution, 

52 Finding, 

53 PhaseExecution, 

54 Task, 

55) 

56from governed_harness.evidence import sha256_json 

57from governed_harness.evidence.hashing import sha256_bytes 

58from governed_harness.orchestration.engine_types import PhaseOutcome 

59from governed_harness.orchestration.workspace_ops import materialized 

60from governed_harness.runtime import CancellationToken, SafeProcessRunner 

61from governed_harness.runtime.process_runner import CommandSpec 

62from governed_harness.validators import CommandValidator, ValidatorOutput 

63 

64if TYPE_CHECKING: 

65 from governed_harness.orchestration.hosts import ResultsHost 

66 

67ACCEPTANCE_ID = "harness.acceptance-tests" 

68WRITE_UNDONE_RULE = "acceptance.write-undone" 

69REVISION_NOTE = ( 

70 "A person agreed that the acceptance tests approved before were wrong, for the reasons " 

71 "listed. Write the acceptance tests again from the criteria and requirements; do not repeat " 

72 "the disputed assertions." 

73) 

74MAX_FILES = 20 

75MAX_FILE_BYTES = 200_000 

76 

77 

78def _check_feature(index: int, path: str, content: Any, directory: str) -> None: 

79 """A feature file lives under ``directory`` and has a Feature and a Scenario.""" 

80 if ( 80 ↛ 85line 80 didn't jump to line 85 because the condition on line 80 was never true

81 not path.startswith(directory + "/") 

82 or ".." in PurePosixPath(path).parts 

83 or not path.endswith(".feature") 

84 ): 

85 raise ValueError(f"feature file {index} must be {directory}/<name>.feature, got {path!r}") 

86 if not isinstance(content, str) or "Feature:" not in content or ("Scenario" not in content): 86 ↛ 87line 86 didn't jump to line 87 because the condition on line 86 was never true

87 raise ValueError(f"feature file {path} needs a Feature and a Scenario") 

88 

89 

90def _check_test_path(index: int, path: str, directory: str) -> None: 

91 """A pytest file lives under ``directory`` and is named ``test_<name>.py``.""" 

92 if ( 92 ↛ 98line 92 didn't jump to line 98 because the condition on line 92 was never true

93 not path.startswith(directory + "/") 

94 or ".." in PurePosixPath(path).parts 

95 or not PurePosixPath(path).name.startswith("test_") 

96 or not path.endswith(".py") 

97 ): 

98 raise ValueError(f"test file {index} must be {directory}/test_<name>.py, got {path!r}") 

99 

100 

101def _test_file(index: int, entry: Any, directory: str, gherkin: bool) -> dict[str, str]: 

102 """One file of the answer, checked: its place, its name and its content.""" 

103 if not isinstance(entry, dict): 103 ↛ 104line 103 didn't jump to line 104 because the condition on line 103 was never true

104 raise ValueError(f"test file {index} is not an object") 

105 path = str(entry.get("path") or "") 

106 content = entry.get("content") 

107 if gherkin: 

108 _check_feature(index, path, content, directory) 

109 else: 

110 _check_test_path(index, path, directory) 

111 if not isinstance(content, str) or not content.strip(): 111 ↛ 112line 111 didn't jump to line 112 because the condition on line 111 was never true

112 raise ValueError(f"test file {path} has no content") 

113 if len(content.encode("utf-8")) > MAX_FILE_BYTES: 113 ↛ 114line 113 didn't jump to line 114 because the condition on line 113 was never true

114 raise ValueError(f"test file {path} is larger than {MAX_FILE_BYTES} bytes") 

115 return {"path": path, "content": content} 

116 

117 

118def validate_tests( 

119 result: dict[str, Any], directory: str, *, gherkin: bool = False 

120) -> list[dict[str, str]]: 

121 raw = result.get("tests") 

122 if not isinstance(raw, list): 

123 raise ValueError("the acceptance result needs a 'tests' list") 

124 if len(raw) > MAX_FILES: 124 ↛ 125line 124 didn't jump to line 125 because the condition on line 124 was never true

125 raise ValueError(f"at most {MAX_FILES} acceptance test files are allowed") 

126 files: list[dict[str, str]] = [] 

127 seen: set[str] = set() 

128 for index, entry in enumerate(raw, start=1): 

129 item = _test_file(index, entry, directory, gherkin) 

130 if item["path"] in seen: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true

131 raise ValueError(f"test file {item['path']} appears twice") 

132 seen.add(item["path"]) 

133 files.append(item) 

134 return files 

135 

136 

137def _free_path(workspace: Path, path: str, execution_id: str) -> str: 

138 """``path`` when no file is there; otherwise the same name with the run's suffix (and a 

139 counter if that is taken too), in the same directory and with the same extension, so the 

140 runner still collects it (``test_<name>_<run>.py``, ``<name>_<run>.feature``).""" 

141 if not (workspace / path).exists(): 

142 return path 

143 pure = PurePosixPath(path) 

144 suffix = execution_id.rsplit("_", 1)[-1][-8:] 

145 candidate = str(pure.with_name(f"{pure.stem}_{suffix}{pure.suffix}")) 

146 number = 2 

147 while (workspace / candidate).exists(): 147 ↛ 148line 147 didn't jump to line 148 because the condition on line 147 was never true

148 candidate = str(pure.with_name(f"{pure.stem}_{suffix}_{number}{pure.suffix}")) 

149 number += 1 

150 return candidate 

151 

152 

153class AcceptanceTests: 

154 def __init__(self, results: ResultsHost) -> None: 

155 self.results = results 

156 

157 @property 

158 def config(self) -> Any: 

159 verification = self.results.project.verification 

160 return verification.acceptance_tests if verification else None 

161 

162 @property 

163 def bdd(self) -> bool: 

164 """``testing.strategy: bdd`` (#56): the acceptance tests are Gherkin scenarios.""" 

165 engineering = self.results.engineering 

166 if not engineering.configured: 

167 return False 

168 return engineering.strategy(self.results.project.project_id).strategy == "bdd" 

169 

170 @property 

171 def enabled(self) -> bool: 

172 return bool(self.config and self.config.enabled) or self.bdd 

173 

174 def _command(self, paths: list[str]) -> tuple[str, ...]: 

175 if self.bdd: 

176 return self.results.engineering.bdd_command() 

177 return ("python", "-m", "pytest", "-q", *paths) 

178 

179 def _key(self, execution: Execution) -> str: 

180 return f"acceptance:{execution.execution_id}" 

181 

182 def state(self, execution: Execution) -> dict[str, Any] | None: 

183 value = self.results.flag_json(self._key(execution)) 

184 return value if isinstance(value, dict) else None 

185 

186 # ----- SPECIFICATION ------------------------------------------------------------------- 

187 def propose( 

188 self, execution: Execution, phase: PhaseExecution, task: Task 

189 ) -> PhaseOutcome | None: 

190 if not self.enabled: 

191 return None 

192 state = self.state(execution) 

193 carried = _carried(state) 

194 if state is not None and not carried: 

195 if state["status"] == "PROPOSED": 195 ↛ 196line 195 didn't jump to line 196 because the condition on line 195 was never true

196 return PhaseOutcome( 

197 ResultStatus.BLOCKED, 

198 f"{len(state['tests'])} acceptance test file(s) wait for a person: harness " 

199 f"acceptance decide --run {execution.execution_id} --decision APPROVE " 

200 f"--digest {state['digest']}", 

201 (state["ref"],), 

202 ) 

203 return None 

204 gherkin = self.bdd 

205 directory = self.results.engineering.features_directory() if gherkin else self.config.path 

206 outcome = self.results.call_agent( 

207 execution, 

208 phase, 

209 "acceptance", 

210 { 

211 "directory": directory, 

212 **({"format": "gherkin"} if gherkin else {}), 

213 **_revision_request(carried), 

214 }, 

215 task=task, 

216 instruction_values={ 

217 "directory": directory, 

218 "format": "gherkin" if gherkin else "pytest", 

219 }, 

220 validate=lambda result: validate_tests(result, directory, gherkin=gherkin), 

221 ) 

222 if outcome.status is not ResultStatus.PASSED or outcome.result is None: 

223 return PhaseOutcome( 

224 ResultStatus.BLOCKED, 

225 f"The acceptance-test call did not answer ({outcome.status}): {outcome.summary}", 

226 outcome.evidence_refs, 

227 ) 

228 try: 

229 tests = validate_tests(outcome.result, directory, gherkin=gherkin) 

230 except ValueError as error: 

231 self.results.record_finding( 

232 execution, 

233 validator_id=ACCEPTANCE_ID, 

234 rule_id="acceptance.malformed", 

235 category="acceptance-tests", 

236 severity=FindingSeverity.HIGH, 

237 message=f"The proposed acceptance tests are not valid: {error}", 

238 evidence_refs=outcome.evidence_refs, 

239 ) 

240 return PhaseOutcome( 

241 ResultStatus.BLOCKED, 

242 f"The proposed acceptance tests are not valid: {error}", 

243 outcome.evidence_refs, 

244 ) 

245 digest = sha256_json({"tests": tests}) 

246 status = "PROPOSED" if tests else "EMPTY" 

247 record = { 

248 "status": status, 

249 "digest": digest, 

250 "directory": directory, 

251 "format": "gherkin" if gherkin else "pytest", 

252 "tests": tests, 

253 "invocationId": outcome.invocation_id, 

254 **carried, 

255 } 

256 ref = self.results.record_json( 

257 execution, 

258 PhaseId.SPECIFICATION, 

259 record, 

260 kind="acceptance-tests-proposal", 

261 summary=f"Proposed acceptance tests: {len(tests)} file(s)", 

262 supports=tuple(item.criterion_id for item in task.acceptance_criteria), 

263 ) 

264 self.results.set_flag_json(self._key(execution), {**record, "ref": ref}) 

265 self.results.s.events.append( 

266 execution.execution_id, 

267 "acceptance.tests.proposed", 

268 {"digest": digest, "files": [item["path"] for item in tests], "evidenceRef": ref}, 

269 phase_execution_id=phase.phase_execution_id, 

270 ) 

271 if not tests: 

272 return None 

273 return PhaseOutcome( 

274 ResultStatus.BLOCKED, 

275 f"{len(tests)} acceptance test file(s) wait for a person: harness acceptance decide " 

276 f"--run {execution.execution_id} --decision APPROVE --digest {digest}", 

277 (ref,), 

278 ) 

279 

280 def decide( 

281 self, 

282 execution: Execution, 

283 *, 

284 decision: DecisionKind, 

285 digest: str, 

286 actor: Actor, 

287 rationale: str, 

288 ) -> dict[str, Any]: 

289 """Approve (write and freeze) or reject the proposed acceptance tests.""" 

290 if actor.actor_type is not ActorType.HUMAN: 290 ↛ 291line 290 didn't jump to line 291 because the condition on line 290 was never true

291 raise PolicyViolationError("only a person can decide acceptance tests") 

292 if decision not in {DecisionKind.APPROVE, DecisionKind.REJECT}: 292 ↛ 293line 292 didn't jump to line 293 because the condition on line 292 was never true

293 raise PolicyViolationError("acceptance tests are approved or rejected") 

294 state = self.state(execution) 

295 if not state or state.get("status") != "PROPOSED": 295 ↛ 296line 295 didn't jump to line 296 because the condition on line 295 was never true

296 raise NotFoundError(f"run {execution.execution_id} has no proposed acceptance tests") 

297 if state["digest"] != digest: 297 ↛ 298line 297 didn't jump to line 298 because the condition on line 297 was never true

298 raise PolicyViolationError("the digest does not match the proposed acceptance tests") 

299 frozen: dict[str, str] = {} 

300 renamed: dict[str, str] = {} 

301 if decision is DecisionKind.APPROVE: 301 ↛ 303line 301 didn't jump to line 303 because the condition on line 301 was always true

302 frozen, renamed = self._write(execution, state["tests"]) 

303 status = "APPROVED" if decision is DecisionKind.APPROVE else "REJECTED" 

304 state = { 

305 **state, 

306 "status": status, 

307 "frozen": frozen, 

308 "decidedBy": actor.actor_id, 

309 "rationale": rationale, 

310 } 

311 payload: dict[str, Any] = {"decision": decision.value, "digest": digest, "frozen": frozen} 

312 if renamed: 

313 state["renamed"] = renamed 

314 payload["renamed"] = renamed 

315 self.results.set_flag_json(self._key(execution), state) 

316 self.results.s.events.append( 

317 execution.execution_id, "acceptance.tests.decided", payload, actor=actor 

318 ) 

319 if frozen: 319 ↛ 322line 319 didn't jump to line 322 because the condition on line 319 was always true

320 self._fail_before(execution, list(frozen)) 

321 state = self.state(execution) or state 

322 return state 

323 

324 def _write( 

325 self, execution: Execution, tests: list[dict[str, str]] 

326 ) -> tuple[dict[str, str], dict[str, str]]: 

327 """Write the approved files and return their digests (frozen) and the proposed paths 

328 written under another name. A path where a file already exists (the frozen acceptance 

329 test of an earlier run in the same workspace, or any file of the project) is never 

330 overwritten (#82): the file gets a run-unique name next to it, so no earlier run's 

331 frozen file changes and no agent is blamed for a test that disappeared.""" 

332 workspace = self.results.s.paths.workspace 

333 frozen: dict[str, str] = {} 

334 renamed: dict[str, str] = {} 

335 for item in tests: 

336 path = _free_path(workspace, item["path"], execution.execution_id) 

337 if path != item["path"]: 

338 renamed[item["path"]] = path 

339 target = contained_path(workspace, Path(path)) 

340 target.parent.mkdir(parents=True, exist_ok=True) 

341 data = item["content"].encode("utf-8") 

342 target.write_bytes(data) 

343 frozen[path] = sha256_bytes(data) 

344 return frozen, renamed 

345 

346 def revise(self, execution: Execution, dispute: dict[str, Any]) -> None: 

347 """A person agreed that the frozen tests are wrong (#94): remove the files the harness 

348 wrote and leave the run ready for a new proposal. What was approved stays recorded 

349 (digest, frozen files and their digests) under ``previous``.""" 

350 state = self.state(execution) 

351 if not state or state.get("status") != "APPROVED": 351 ↛ 352line 351 didn't jump to line 352 because the condition on line 351 was never true

352 raise NotFoundError(f"run {execution.execution_id} has no frozen acceptance tests") 

353 workspace = self.results.s.paths.workspace 

354 for path in state.get("frozen") or {}: 

355 contained_path(workspace, Path(path)).unlink(missing_ok=True) 

356 kept = ("digest", "frozen", "renamed", "decidedBy", "rationale", "ref") 

357 previous = [ 

358 *(state.get("previous") or []), 

359 {key: state[key] for key in kept if key in state}, 

360 ] 

361 self.results.set_flag_json( 

362 self._key(execution), 

363 { 

364 "status": "REVISING", 

365 "revision": len(previous), 

366 "previous": previous, 

367 "dispute": dispute, 

368 }, 

369 ) 

370 self.results.s.events.append( 

371 execution.execution_id, 

372 "acceptance.tests.revision", 

373 {"revision": len(previous), "removed": sorted(state.get("frozen") or {})}, 

374 ) 

375 

376 def _run_tests( 

377 self, execution: Execution, runner: SafeProcessRunner, root: Path, paths: list[str] 

378 ) -> Any: 

379 results = self.results 

380 engine = results.engine 

381 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{ACCEPTANCE_ID}", version="1") 

382 return runner.run( 

383 CommandSpec( 

384 argv=self._command(paths), 

385 cwd=root, 

386 timeout_seconds=900.0, 

387 max_output_bytes=results.project.runtime.max_output_bytes, 

388 ), 

389 actor=actor, 

390 grants=grants_from_rules( 

391 execution.execution_id, actor, results.s.resolved.effective_capabilities 

392 ), 

393 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)), 

394 ) 

395 

396 def _before_change(self, execution: Execution, paths: list[str]) -> Any: 

397 """The frozen tests on the code before the change: the workspace itself in the first 

398 SPECIFICATION and, for a revision (the workspace already holds the change), a copy 

399 with every other changed path as on the baseline. ``None`` when that copy cannot be 

400 built.""" 

401 results = self.results 

402 workspace = results.s.paths.workspace 

403 if not (self.state(execution) or {}).get("revision"): 

404 return self._run_tests(execution, results.engine._runner(execution), workspace, paths) 

405 contents = results.baseline_contents(execution) 

406 changes = results.baseline_changes(execution) 

407 if contents is None or changes is None: 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true

408 return None 

409 reverted = [item.path for item in changes.changes if item.path not in paths] 

410 scratch = results.s.paths.harness_dir / "tmp" 

411 with materialized(workspace, scratch, contents, reverted) as copy: 

412 if copy is None: 412 ↛ 413line 412 didn't jump to line 413 because the condition on line 412 was never true

413 return None 

414 return self._run_tests(execution, SafeProcessRunner(copy), copy, paths) 

415 

416 def _fail_before(self, execution: Execution, paths: list[str]) -> None: 

417 """The frozen tests run on the code before the change: passing there means they do not 

418 test the change.""" 

419 results = self.results 

420 process = self._before_change(execution, paths) 

421 if process is None: 421 ↛ 422line 421 didn't jump to line 422 because the condition on line 421 was never true

422 state = self.state(execution) or {} 

423 state["failBefore"] = { 

424 "status": "NOT_RUN", 

425 "reason": "the baseline of the run could not be rebuilt", 

426 } 

427 results.set_flag_json(self._key(execution), state) 

428 return 

429 ref = results.record_json( 

430 execution, 

431 PhaseId.SPECIFICATION, 

432 { 

433 "tests": paths, 

434 "status": process.status.value, 

435 "exitCode": process.exit_code, 

436 "stdoutRef": results.s.artifacts.put(process.stdout, media_type="text/plain").uri, 

437 }, 

438 kind="acceptance-fail-before", 

439 summary=f"Acceptance tests before the change: {process.status.value}", 

440 ) 

441 state = self.state(execution) or {} 

442 state["failBefore"] = {"status": process.status.value, "ref": ref} 

443 if process.status is ResultStatus.PASSED: 

444 finding = results.record_finding( 

445 execution, 

446 validator_id=ACCEPTANCE_ID, 

447 rule_id="acceptance.passes-before", 

448 category="acceptance-tests", 

449 severity=FindingSeverity.MEDIUM, 

450 message=( 

451 "The approved acceptance tests already pass before the change; they do not " 

452 "test the new behaviour" 

453 ), 

454 evidence_refs=(ref,), 

455 ) 

456 state["failBeforeFinding"] = finding.finding_id 

457 results.set_flag_json(self._key(execution), state) 

458 

459 # ----- IMPLEMENTATION and VERIFICATION -------------------------------------------------- 

460 def untouched(self, execution: Execution) -> set[str]: 

461 """The frozen files the harness wrote on approval that are still as approved: they 

462 are the harness's, not the agent's edits (#81).""" 

463 state = self.state(execution) 

464 if not state or state.get("status") != "APPROVED": 

465 return set() 

466 workspace = self.results.s.paths.workspace 

467 kept: set[str] = set() 

468 for path, digest in (state.get("frozen") or {}).items(): 

469 target = workspace / path 

470 if target.is_file() and sha256_bytes(target.read_bytes()) == digest: 470 ↛ 468line 470 didn't jump to line 468 because the condition on line 470 was always true

471 kept.add(path) 

472 return kept 

473 

474 # ----- write protection (#94) ------------------------------------------------------------ 

475 @property 

476 def protects(self) -> bool: 

477 return bool(self.config and self.config.protect) 

478 

479 def _frozen(self, execution: Execution) -> dict[str, str]: 

480 state = self.state(execution) 

481 if not state or state.get("status") != "APPROVED": 

482 return {} 

483 return dict(state.get("frozen") or {}) 

484 

485 def protected_paths(self, execution: Execution) -> tuple[Path, ...]: 

486 """The frozen files, for the agent sandbox to deny writes to (``protect``).""" 

487 if not self.protects: 

488 return () 

489 workspace = self.results.s.paths.workspace 

490 return tuple(workspace / path for path in sorted(self._frozen(execution))) 

491 

492 def restore_frozen(self, execution: Execution) -> list[str]: 

493 """After an agent invocation: put back every frozen file that is no longer as 

494 approved, and report it. Returns the restored paths.""" 

495 frozen = self._frozen(execution) if self.protects else {} 

496 if not frozen: 

497 return [] 

498 approved = _approved_contents(self.state(execution) or {}) 

499 workspace = self.results.s.paths.workspace 

500 undone: list[str] = [] 

501 for path, digest in sorted(frozen.items()): 

502 what = _put_back(workspace, path, digest, approved.get(path)) 

503 if what is not None: 

504 undone.append(f"{path} ({what})") 

505 if undone: 

506 self._report_undone(execution, undone) 

507 return [item.split(" ", 1)[0] for item in undone] 

508 

509 def _report_undone(self, execution: Execution, undone: list[str]) -> None: 

510 finding = self.results.record_finding( 

511 execution, 

512 validator_id=ACCEPTANCE_ID, 

513 rule_id=WRITE_UNDONE_RULE, 

514 category="acceptance-tests", 

515 severity=FindingSeverity.HIGH, 

516 message=( 

517 "The agent changed frozen acceptance test file(s); the harness put the approved " 

518 f"content back: {', '.join(undone)}" 

519 ), 

520 path=undone[0].split(" ", 1)[0], 

521 recommendation=( 

522 "The frozen acceptance tests are write-protected " 

523 "(verification.acceptanceTests.protect): change the code, or dispute a test in " 

524 "the self-report." 

525 ), 

526 introduced=True, 

527 ) 

528 self.results.s.events.append( 

529 execution.execution_id, 

530 "acceptance.write.undone", 

531 {"files": undone, "findingId": finding.finding_id}, 

532 ) 

533 

534 def request_extra(self, execution: Execution) -> dict[str, Any] | None: 

535 extra = self._request_extra(execution) 

536 if extra is not None and self.protects: 

537 extra["note"] += " The files are write-protected: a change to them is undone." 

538 return extra 

539 

540 def _request_extra(self, execution: Execution) -> dict[str, Any] | None: 

541 state = self.state(execution) 

542 if not state or state.get("status") != "APPROVED" or not state.get("frozen"): 

543 return None 

544 if state.get("format") == "gherkin": 

545 return { 

546 "paths": sorted(state["frozen"]), 

547 "frozen": True, 

548 "format": "gherkin", 

549 "runner": list(self._command([])), 

550 "note": "Write the step definitions and the code that make these scenarios " 

551 "pass; do not modify or delete the feature files.", 

552 } 

553 return { 

554 "paths": sorted(state["frozen"]), 

555 "frozen": True, 

556 "note": "Make these tests pass; do not modify or delete them.", 

557 } 

558 

559 def validation(self, execution: Execution, change_set: ChangeSet) -> ValidatorOutput | None: 

560 state = self.state(execution) 

561 if not self.enabled or not state or state.get("status") != "APPROVED": 

562 return None 

563 frozen: dict[str, str] = state.get("frozen") or {} 

564 if not frozen: 564 ↛ 565line 564 didn't jump to line 565 because the condition on line 564 was never true

565 return None 

566 results = self.results 

567 findings = self._frozen_findings(execution, frozen) 

568 output = self._run_frozen(execution, change_set, frozen) 

569 failed = bool(findings) or output.result.status is not ResultStatus.PASSED 

570 result = results.record_validation( 

571 execution, 

572 validator_id=ACCEPTANCE_ID, 

573 digest=change_set.digest, 

574 status=output.result.status if not findings else ResultStatus.FAILED, 

575 kind=ValidationKind.VALIDATION_FAILURE if failed else ValidationKind.SUCCESS, 

576 mandatory=True, 

577 summary=( 

578 f"frozen acceptance tests: {output.result.summary}" 

579 + (f"; {len(findings)} frozen file(s) changed" if findings else "") 

580 ), 

581 findings=(*findings, *output.findings), 

582 evidence_refs=output.result.evidence_refs, 

583 ) 

584 return ValidatorOutput(result, ()) 

585 

586 def _frozen_findings(self, execution: Execution, frozen: dict[str, str]) -> list[Finding]: 

587 """A HIGH finding for every frozen acceptance test deleted or modified.""" 

588 results = self.results 

589 workspace = results.s.paths.workspace 

590 findings: list[Finding] = [] 

591 for path, digest in sorted(frozen.items()): 

592 target = workspace / path 

593 current = sha256_bytes(target.read_bytes()) if target.is_file() else None 

594 if current != digest: 

595 findings.append( 

596 results.record_finding( 

597 execution, 

598 validator_id=ACCEPTANCE_ID, 

599 rule_id="acceptance.modified", 

600 category="acceptance-tests", 

601 severity=FindingSeverity.HIGH, 

602 message=( 

603 f"The frozen acceptance test {path} was " 

604 + ("deleted" if current is None else "modified") 

605 ), 

606 path=path, 

607 recommendation="Restore the approved acceptance test; change the code.", 

608 introduced=True, 

609 ) 

610 ) 

611 return findings 

612 

613 def _run_frozen( 

614 self, execution: Execution, change_set: ChangeSet, frozen: dict[str, str] 

615 ) -> ValidatorOutput: 

616 """Run the frozen acceptance tests and record their tool invocations and findings.""" 

617 results = self.results 

618 workspace = results.s.paths.workspace 

619 engine = results.engine 

620 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{ACCEPTANCE_ID}", version="1") 

621 from governed_harness.configuration.models import ValidatorDefinition 

622 from governed_harness.validators import ValidationContext 

623 

624 output = CommandValidator(ACCEPTANCE_ID).execute( 

625 ValidationContext( 

626 execution_id=execution.execution_id, 

627 workspace=workspace, 

628 task=engine.run_task(execution), 

629 change_set=change_set, 

630 definition=engine._bounded_definition( 

631 ValidatorDefinition( 

632 id=ACCEPTANCE_ID, 

633 command=self._command(sorted(frozen)), 

634 mandatory=True, 

635 ) 

636 ), 

637 grants=grants_from_rules( 

638 execution.execution_id, actor, results.s.resolved.effective_capabilities 

639 ), 

640 artifact_store=results.s.artifacts, 

641 process_runner=engine._runner(execution), 

642 provenance=engine._provenance(execution).model_copy(update={"actor": actor}), 

643 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)), 

644 max_output_bytes=results.project.runtime.max_output_bytes, 

645 parse_output=True, 

646 ) 

647 ) 

648 for tool in output.tool_invocations: 

649 engine._save_tool(execution, tool) 

650 for finding in output.findings: 

651 results.s.state.put( 

652 "finding", 

653 finding.finding_id, 

654 finding, 

655 execution_id=execution.execution_id, 

656 project_id=execution.project_id, 

657 ) 

658 results.s.events.append( 

659 execution.execution_id, 

660 "finding.recorded", 

661 finding.model_dump(mode="json"), 

662 actor=finding.provenance.actor, 

663 ) 

664 return output 

665 

666 

667def _carried(state: dict[str, Any] | None) -> dict[str, Any]: 

668 """What a proposal made for a revision (#94) carries over: its number, the tests approved 

669 before and the dispute; empty for a first proposal.""" 

670 if not state or state.get("status") != "REVISING": 

671 return {} 

672 return {key: state[key] for key in ("revision", "previous", "dispute") if key in state} 

673 

674 

675def _revision_request(carried: dict[str, Any]) -> dict[str, Any]: 

676 """The ``revision`` block of the acceptance request: what was disputed and why a person 

677 agreed.""" 

678 dispute = carried.get("dispute") 

679 if not dispute: 

680 return {} 

681 return { 

682 "revision": { 

683 "disputed": dispute.get("disputed") or [], 

684 "rationale": dispute.get("rationale"), 

685 "note": REVISION_NOTE, 

686 } 

687 } 

688 

689 

690def _approved_contents(state: dict[str, Any]) -> dict[str, bytes]: 

691 """The approved content of each frozen file, by the path it was written to.""" 

692 renamed = state.get("renamed") or {} 

693 return { 

694 renamed.get(item["path"], item["path"]): item["content"].encode("utf-8") 

695 for item in state.get("tests") or [] 

696 } 

697 

698 

699def _put_back(workspace: Path, path: str, digest: str, content: bytes | None) -> str | None: 

700 """Write the approved content of a frozen file that changed; what had happened to it 

701 (``deleted`` or ``modified``), or ``None`` when it is as approved or cannot be restored.""" 

702 target = workspace / path 

703 change = _frozen_change(target, digest) 

704 if change is None or content is None or sha256_bytes(content) != digest: 

705 return None 

706 if target.is_symlink(): 706 ↛ 707line 706 didn't jump to line 707 because the condition on line 706 was never true

707 target.unlink() 

708 safe = contained_path(workspace, Path(path)) 

709 safe.parent.mkdir(parents=True, exist_ok=True) 

710 safe.write_bytes(content) 

711 return change 

712 

713 

714def _frozen_change(target: Path, digest: str) -> str | None: 

715 """What happened to a frozen file: ``deleted`` when it is no longer a regular file, 

716 ``modified`` when its content differs, ``None`` when it is as approved.""" 

717 if target.is_symlink(): 717 ↛ 718line 717 didn't jump to line 718 because the condition on line 717 was never true

718 return "deleted" 

719 try: 

720 current = target.read_bytes() 

721 except OSError: 

722 return "deleted" 

723 return None if sha256_bytes(current) == digest else "modified" 

724 

725 

726__all__ = ["ACCEPTANCE_ID", "WRITE_UNDONE_RULE", "AcceptanceTests", "validate_tests"]