Coverage for src/governed_harness/orchestration/acceptance.py: 89%
328 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Independent, frozen acceptance tests (#52, N4).
3The gate checked the agent's work with the agent's own tests: approved steps of a small model
4covered some sections at 31-40 %, and a single systematic defect produced hundreds of errors
5while the agent's fifty tests passed. Under ``verification.acceptanceTests.mode: agent``, in
6SPECIFICATION a separate call (kind ``acceptance``, read-only, the provider and model of
7``acceptanceTests.author``) writes acceptance tests from the criteria and requirements only and
8returns them as files. The harness does not write them yet: a person approves them, bound to
9their digest (``harness acceptance decide``). On approval the harness writes them under
10``acceptanceTests.directory``, records their digests (frozen) and runs them on the workspace as it
11is before the change: a test that already passes there does not test the change (a MEDIUM
12finding). Every later VERIFICATION checks that the frozen files are unchanged (a modified or
13deleted file is a HIGH finding) and that they pass; the implement request names them as frozen.
14A proposal without tests is recorded and the run continues.
16Since #56, under ``testing.strategy: bdd`` the same call writes the criteria as Gherkin feature
17files (under ``testing.featuresDirectory``, default ``features``) instead of pytest files: a
18person approves the scenarios, the harness freezes them and runs the BDD runner of the project
19(``testing.bddCommand`` or the standards pack's runner) before the change (it must fail: the
20steps are not defined yet) and in every VERIFICATION; the implement request asks for the step
21definitions.
23Since 2.0 (#94), under ``acceptanceTests.protect`` the frozen files are write-protected for the
24implementing agent: they are among the paths the agent sandbox denies writes to, and a frozen
25file an agent invocation changed or deleted anyway is put back as approved, with a HIGH
26``acceptance.write-undone`` finding. Under ``acceptanceTests.disputes`` a person may send a run
27back to SPECIFICATION to have wrong tests written again (``revise``; see
28``acceptance_dispute``): the earlier frozen files are removed and stay recorded with their
29digests under ``previous``, the acceptance call receives what was disputed, and the new tests
30run once on a copy of the baseline (the workspace already holds the change)."""
32from __future__ import annotations
34from pathlib import Path, PurePosixPath
35from typing import TYPE_CHECKING, Any
37from governed_harness.capabilities import grants_from_rules
38from governed_harness.capabilities.authorizer import contained_path
39from governed_harness.domain.enums import (
40 ActorType,
41 DecisionKind,
42 FindingSeverity,
43 PhaseId,
44 ResultStatus,
45 ValidationKind,
46)
47from governed_harness.domain.errors import NotFoundError, PolicyViolationError
48from governed_harness.domain.models import (
49 Actor,
50 ChangeSet,
51 Execution,
52 Finding,
53 PhaseExecution,
54 Task,
55)
56from governed_harness.evidence import sha256_json
57from governed_harness.evidence.hashing import sha256_bytes
58from governed_harness.orchestration.engine_types import PhaseOutcome
59from governed_harness.orchestration.workspace_ops import materialized
60from governed_harness.runtime import CancellationToken, SafeProcessRunner
61from governed_harness.runtime.process_runner import CommandSpec
62from governed_harness.validators import CommandValidator, ValidatorOutput
64if TYPE_CHECKING:
65 from governed_harness.orchestration.hosts import ResultsHost
67ACCEPTANCE_ID = "harness.acceptance-tests"
68WRITE_UNDONE_RULE = "acceptance.write-undone"
69REVISION_NOTE = (
70 "A person agreed that the acceptance tests approved before were wrong, for the reasons "
71 "listed. Write the acceptance tests again from the criteria and requirements; do not repeat "
72 "the disputed assertions."
73)
74MAX_FILES = 20
75MAX_FILE_BYTES = 200_000
78def _check_feature(index: int, path: str, content: Any, directory: str) -> None:
79 """A feature file lives under ``directory`` and has a Feature and a Scenario."""
80 if ( 80 ↛ 85line 80 didn't jump to line 85 because the condition on line 80 was never true
81 not path.startswith(directory + "/")
82 or ".." in PurePosixPath(path).parts
83 or not path.endswith(".feature")
84 ):
85 raise ValueError(f"feature file {index} must be {directory}/<name>.feature, got {path!r}")
86 if not isinstance(content, str) or "Feature:" not in content or ("Scenario" not in content): 86 ↛ 87line 86 didn't jump to line 87 because the condition on line 86 was never true
87 raise ValueError(f"feature file {path} needs a Feature and a Scenario")
90def _check_test_path(index: int, path: str, directory: str) -> None:
91 """A pytest file lives under ``directory`` and is named ``test_<name>.py``."""
92 if ( 92 ↛ 98line 92 didn't jump to line 98 because the condition on line 92 was never true
93 not path.startswith(directory + "/")
94 or ".." in PurePosixPath(path).parts
95 or not PurePosixPath(path).name.startswith("test_")
96 or not path.endswith(".py")
97 ):
98 raise ValueError(f"test file {index} must be {directory}/test_<name>.py, got {path!r}")
101def _test_file(index: int, entry: Any, directory: str, gherkin: bool) -> dict[str, str]:
102 """One file of the answer, checked: its place, its name and its content."""
103 if not isinstance(entry, dict): 103 ↛ 104line 103 didn't jump to line 104 because the condition on line 103 was never true
104 raise ValueError(f"test file {index} is not an object")
105 path = str(entry.get("path") or "")
106 content = entry.get("content")
107 if gherkin:
108 _check_feature(index, path, content, directory)
109 else:
110 _check_test_path(index, path, directory)
111 if not isinstance(content, str) or not content.strip(): 111 ↛ 112line 111 didn't jump to line 112 because the condition on line 111 was never true
112 raise ValueError(f"test file {path} has no content")
113 if len(content.encode("utf-8")) > MAX_FILE_BYTES: 113 ↛ 114line 113 didn't jump to line 114 because the condition on line 113 was never true
114 raise ValueError(f"test file {path} is larger than {MAX_FILE_BYTES} bytes")
115 return {"path": path, "content": content}
118def validate_tests(
119 result: dict[str, Any], directory: str, *, gherkin: bool = False
120) -> list[dict[str, str]]:
121 raw = result.get("tests")
122 if not isinstance(raw, list):
123 raise ValueError("the acceptance result needs a 'tests' list")
124 if len(raw) > MAX_FILES: 124 ↛ 125line 124 didn't jump to line 125 because the condition on line 124 was never true
125 raise ValueError(f"at most {MAX_FILES} acceptance test files are allowed")
126 files: list[dict[str, str]] = []
127 seen: set[str] = set()
128 for index, entry in enumerate(raw, start=1):
129 item = _test_file(index, entry, directory, gherkin)
130 if item["path"] in seen: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true
131 raise ValueError(f"test file {item['path']} appears twice")
132 seen.add(item["path"])
133 files.append(item)
134 return files
137def _free_path(workspace: Path, path: str, execution_id: str) -> str:
138 """``path`` when no file is there; otherwise the same name with the run's suffix (and a
139 counter if that is taken too), in the same directory and with the same extension, so the
140 runner still collects it (``test_<name>_<run>.py``, ``<name>_<run>.feature``)."""
141 if not (workspace / path).exists():
142 return path
143 pure = PurePosixPath(path)
144 suffix = execution_id.rsplit("_", 1)[-1][-8:]
145 candidate = str(pure.with_name(f"{pure.stem}_{suffix}{pure.suffix}"))
146 number = 2
147 while (workspace / candidate).exists(): 147 ↛ 148line 147 didn't jump to line 148 because the condition on line 147 was never true
148 candidate = str(pure.with_name(f"{pure.stem}_{suffix}_{number}{pure.suffix}"))
149 number += 1
150 return candidate
153class AcceptanceTests:
154 def __init__(self, results: ResultsHost) -> None:
155 self.results = results
157 @property
158 def config(self) -> Any:
159 verification = self.results.project.verification
160 return verification.acceptance_tests if verification else None
162 @property
163 def bdd(self) -> bool:
164 """``testing.strategy: bdd`` (#56): the acceptance tests are Gherkin scenarios."""
165 engineering = self.results.engineering
166 if not engineering.configured:
167 return False
168 return engineering.strategy(self.results.project.project_id).strategy == "bdd"
170 @property
171 def enabled(self) -> bool:
172 return bool(self.config and self.config.enabled) or self.bdd
174 def _command(self, paths: list[str]) -> tuple[str, ...]:
175 if self.bdd:
176 return self.results.engineering.bdd_command()
177 return ("python", "-m", "pytest", "-q", *paths)
179 def _key(self, execution: Execution) -> str:
180 return f"acceptance:{execution.execution_id}"
182 def state(self, execution: Execution) -> dict[str, Any] | None:
183 value = self.results.flag_json(self._key(execution))
184 return value if isinstance(value, dict) else None
186 # ----- SPECIFICATION -------------------------------------------------------------------
187 def propose(
188 self, execution: Execution, phase: PhaseExecution, task: Task
189 ) -> PhaseOutcome | None:
190 if not self.enabled:
191 return None
192 state = self.state(execution)
193 carried = _carried(state)
194 if state is not None and not carried:
195 if state["status"] == "PROPOSED": 195 ↛ 196line 195 didn't jump to line 196 because the condition on line 195 was never true
196 return PhaseOutcome(
197 ResultStatus.BLOCKED,
198 f"{len(state['tests'])} acceptance test file(s) wait for a person: harness "
199 f"acceptance decide --run {execution.execution_id} --decision APPROVE "
200 f"--digest {state['digest']}",
201 (state["ref"],),
202 )
203 return None
204 gherkin = self.bdd
205 directory = self.results.engineering.features_directory() if gherkin else self.config.path
206 outcome = self.results.call_agent(
207 execution,
208 phase,
209 "acceptance",
210 {
211 "directory": directory,
212 **({"format": "gherkin"} if gherkin else {}),
213 **_revision_request(carried),
214 },
215 task=task,
216 instruction_values={
217 "directory": directory,
218 "format": "gherkin" if gherkin else "pytest",
219 },
220 validate=lambda result: validate_tests(result, directory, gherkin=gherkin),
221 )
222 if outcome.status is not ResultStatus.PASSED or outcome.result is None:
223 return PhaseOutcome(
224 ResultStatus.BLOCKED,
225 f"The acceptance-test call did not answer ({outcome.status}): {outcome.summary}",
226 outcome.evidence_refs,
227 )
228 try:
229 tests = validate_tests(outcome.result, directory, gherkin=gherkin)
230 except ValueError as error:
231 self.results.record_finding(
232 execution,
233 validator_id=ACCEPTANCE_ID,
234 rule_id="acceptance.malformed",
235 category="acceptance-tests",
236 severity=FindingSeverity.HIGH,
237 message=f"The proposed acceptance tests are not valid: {error}",
238 evidence_refs=outcome.evidence_refs,
239 )
240 return PhaseOutcome(
241 ResultStatus.BLOCKED,
242 f"The proposed acceptance tests are not valid: {error}",
243 outcome.evidence_refs,
244 )
245 digest = sha256_json({"tests": tests})
246 status = "PROPOSED" if tests else "EMPTY"
247 record = {
248 "status": status,
249 "digest": digest,
250 "directory": directory,
251 "format": "gherkin" if gherkin else "pytest",
252 "tests": tests,
253 "invocationId": outcome.invocation_id,
254 **carried,
255 }
256 ref = self.results.record_json(
257 execution,
258 PhaseId.SPECIFICATION,
259 record,
260 kind="acceptance-tests-proposal",
261 summary=f"Proposed acceptance tests: {len(tests)} file(s)",
262 supports=tuple(item.criterion_id for item in task.acceptance_criteria),
263 )
264 self.results.set_flag_json(self._key(execution), {**record, "ref": ref})
265 self.results.s.events.append(
266 execution.execution_id,
267 "acceptance.tests.proposed",
268 {"digest": digest, "files": [item["path"] for item in tests], "evidenceRef": ref},
269 phase_execution_id=phase.phase_execution_id,
270 )
271 if not tests:
272 return None
273 return PhaseOutcome(
274 ResultStatus.BLOCKED,
275 f"{len(tests)} acceptance test file(s) wait for a person: harness acceptance decide "
276 f"--run {execution.execution_id} --decision APPROVE --digest {digest}",
277 (ref,),
278 )
280 def decide(
281 self,
282 execution: Execution,
283 *,
284 decision: DecisionKind,
285 digest: str,
286 actor: Actor,
287 rationale: str,
288 ) -> dict[str, Any]:
289 """Approve (write and freeze) or reject the proposed acceptance tests."""
290 if actor.actor_type is not ActorType.HUMAN: 290 ↛ 291line 290 didn't jump to line 291 because the condition on line 290 was never true
291 raise PolicyViolationError("only a person can decide acceptance tests")
292 if decision not in {DecisionKind.APPROVE, DecisionKind.REJECT}: 292 ↛ 293line 292 didn't jump to line 293 because the condition on line 292 was never true
293 raise PolicyViolationError("acceptance tests are approved or rejected")
294 state = self.state(execution)
295 if not state or state.get("status") != "PROPOSED": 295 ↛ 296line 295 didn't jump to line 296 because the condition on line 295 was never true
296 raise NotFoundError(f"run {execution.execution_id} has no proposed acceptance tests")
297 if state["digest"] != digest: 297 ↛ 298line 297 didn't jump to line 298 because the condition on line 297 was never true
298 raise PolicyViolationError("the digest does not match the proposed acceptance tests")
299 frozen: dict[str, str] = {}
300 renamed: dict[str, str] = {}
301 if decision is DecisionKind.APPROVE: 301 ↛ 303line 301 didn't jump to line 303 because the condition on line 301 was always true
302 frozen, renamed = self._write(execution, state["tests"])
303 status = "APPROVED" if decision is DecisionKind.APPROVE else "REJECTED"
304 state = {
305 **state,
306 "status": status,
307 "frozen": frozen,
308 "decidedBy": actor.actor_id,
309 "rationale": rationale,
310 }
311 payload: dict[str, Any] = {"decision": decision.value, "digest": digest, "frozen": frozen}
312 if renamed:
313 state["renamed"] = renamed
314 payload["renamed"] = renamed
315 self.results.set_flag_json(self._key(execution), state)
316 self.results.s.events.append(
317 execution.execution_id, "acceptance.tests.decided", payload, actor=actor
318 )
319 if frozen: 319 ↛ 322line 319 didn't jump to line 322 because the condition on line 319 was always true
320 self._fail_before(execution, list(frozen))
321 state = self.state(execution) or state
322 return state
324 def _write(
325 self, execution: Execution, tests: list[dict[str, str]]
326 ) -> tuple[dict[str, str], dict[str, str]]:
327 """Write the approved files and return their digests (frozen) and the proposed paths
328 written under another name. A path where a file already exists (the frozen acceptance
329 test of an earlier run in the same workspace, or any file of the project) is never
330 overwritten (#82): the file gets a run-unique name next to it, so no earlier run's
331 frozen file changes and no agent is blamed for a test that disappeared."""
332 workspace = self.results.s.paths.workspace
333 frozen: dict[str, str] = {}
334 renamed: dict[str, str] = {}
335 for item in tests:
336 path = _free_path(workspace, item["path"], execution.execution_id)
337 if path != item["path"]:
338 renamed[item["path"]] = path
339 target = contained_path(workspace, Path(path))
340 target.parent.mkdir(parents=True, exist_ok=True)
341 data = item["content"].encode("utf-8")
342 target.write_bytes(data)
343 frozen[path] = sha256_bytes(data)
344 return frozen, renamed
346 def revise(self, execution: Execution, dispute: dict[str, Any]) -> None:
347 """A person agreed that the frozen tests are wrong (#94): remove the files the harness
348 wrote and leave the run ready for a new proposal. What was approved stays recorded
349 (digest, frozen files and their digests) under ``previous``."""
350 state = self.state(execution)
351 if not state or state.get("status") != "APPROVED": 351 ↛ 352line 351 didn't jump to line 352 because the condition on line 351 was never true
352 raise NotFoundError(f"run {execution.execution_id} has no frozen acceptance tests")
353 workspace = self.results.s.paths.workspace
354 for path in state.get("frozen") or {}:
355 contained_path(workspace, Path(path)).unlink(missing_ok=True)
356 kept = ("digest", "frozen", "renamed", "decidedBy", "rationale", "ref")
357 previous = [
358 *(state.get("previous") or []),
359 {key: state[key] for key in kept if key in state},
360 ]
361 self.results.set_flag_json(
362 self._key(execution),
363 {
364 "status": "REVISING",
365 "revision": len(previous),
366 "previous": previous,
367 "dispute": dispute,
368 },
369 )
370 self.results.s.events.append(
371 execution.execution_id,
372 "acceptance.tests.revision",
373 {"revision": len(previous), "removed": sorted(state.get("frozen") or {})},
374 )
376 def _run_tests(
377 self, execution: Execution, runner: SafeProcessRunner, root: Path, paths: list[str]
378 ) -> Any:
379 results = self.results
380 engine = results.engine
381 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{ACCEPTANCE_ID}", version="1")
382 return runner.run(
383 CommandSpec(
384 argv=self._command(paths),
385 cwd=root,
386 timeout_seconds=900.0,
387 max_output_bytes=results.project.runtime.max_output_bytes,
388 ),
389 actor=actor,
390 grants=grants_from_rules(
391 execution.execution_id, actor, results.s.resolved.effective_capabilities
392 ),
393 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)),
394 )
396 def _before_change(self, execution: Execution, paths: list[str]) -> Any:
397 """The frozen tests on the code before the change: the workspace itself in the first
398 SPECIFICATION and, for a revision (the workspace already holds the change), a copy
399 with every other changed path as on the baseline. ``None`` when that copy cannot be
400 built."""
401 results = self.results
402 workspace = results.s.paths.workspace
403 if not (self.state(execution) or {}).get("revision"):
404 return self._run_tests(execution, results.engine._runner(execution), workspace, paths)
405 contents = results.baseline_contents(execution)
406 changes = results.baseline_changes(execution)
407 if contents is None or changes is None: 407 ↛ 408line 407 didn't jump to line 408 because the condition on line 407 was never true
408 return None
409 reverted = [item.path for item in changes.changes if item.path not in paths]
410 scratch = results.s.paths.harness_dir / "tmp"
411 with materialized(workspace, scratch, contents, reverted) as copy:
412 if copy is None: 412 ↛ 413line 412 didn't jump to line 413 because the condition on line 412 was never true
413 return None
414 return self._run_tests(execution, SafeProcessRunner(copy), copy, paths)
416 def _fail_before(self, execution: Execution, paths: list[str]) -> None:
417 """The frozen tests run on the code before the change: passing there means they do not
418 test the change."""
419 results = self.results
420 process = self._before_change(execution, paths)
421 if process is None: 421 ↛ 422line 421 didn't jump to line 422 because the condition on line 421 was never true
422 state = self.state(execution) or {}
423 state["failBefore"] = {
424 "status": "NOT_RUN",
425 "reason": "the baseline of the run could not be rebuilt",
426 }
427 results.set_flag_json(self._key(execution), state)
428 return
429 ref = results.record_json(
430 execution,
431 PhaseId.SPECIFICATION,
432 {
433 "tests": paths,
434 "status": process.status.value,
435 "exitCode": process.exit_code,
436 "stdoutRef": results.s.artifacts.put(process.stdout, media_type="text/plain").uri,
437 },
438 kind="acceptance-fail-before",
439 summary=f"Acceptance tests before the change: {process.status.value}",
440 )
441 state = self.state(execution) or {}
442 state["failBefore"] = {"status": process.status.value, "ref": ref}
443 if process.status is ResultStatus.PASSED:
444 finding = results.record_finding(
445 execution,
446 validator_id=ACCEPTANCE_ID,
447 rule_id="acceptance.passes-before",
448 category="acceptance-tests",
449 severity=FindingSeverity.MEDIUM,
450 message=(
451 "The approved acceptance tests already pass before the change; they do not "
452 "test the new behaviour"
453 ),
454 evidence_refs=(ref,),
455 )
456 state["failBeforeFinding"] = finding.finding_id
457 results.set_flag_json(self._key(execution), state)
459 # ----- IMPLEMENTATION and VERIFICATION --------------------------------------------------
460 def untouched(self, execution: Execution) -> set[str]:
461 """The frozen files the harness wrote on approval that are still as approved: they
462 are the harness's, not the agent's edits (#81)."""
463 state = self.state(execution)
464 if not state or state.get("status") != "APPROVED":
465 return set()
466 workspace = self.results.s.paths.workspace
467 kept: set[str] = set()
468 for path, digest in (state.get("frozen") or {}).items():
469 target = workspace / path
470 if target.is_file() and sha256_bytes(target.read_bytes()) == digest: 470 ↛ 468line 470 didn't jump to line 468 because the condition on line 470 was always true
471 kept.add(path)
472 return kept
474 # ----- write protection (#94) ------------------------------------------------------------
475 @property
476 def protects(self) -> bool:
477 return bool(self.config and self.config.protect)
479 def _frozen(self, execution: Execution) -> dict[str, str]:
480 state = self.state(execution)
481 if not state or state.get("status") != "APPROVED":
482 return {}
483 return dict(state.get("frozen") or {})
485 def protected_paths(self, execution: Execution) -> tuple[Path, ...]:
486 """The frozen files, for the agent sandbox to deny writes to (``protect``)."""
487 if not self.protects:
488 return ()
489 workspace = self.results.s.paths.workspace
490 return tuple(workspace / path for path in sorted(self._frozen(execution)))
492 def restore_frozen(self, execution: Execution) -> list[str]:
493 """After an agent invocation: put back every frozen file that is no longer as
494 approved, and report it. Returns the restored paths."""
495 frozen = self._frozen(execution) if self.protects else {}
496 if not frozen:
497 return []
498 approved = _approved_contents(self.state(execution) or {})
499 workspace = self.results.s.paths.workspace
500 undone: list[str] = []
501 for path, digest in sorted(frozen.items()):
502 what = _put_back(workspace, path, digest, approved.get(path))
503 if what is not None:
504 undone.append(f"{path} ({what})")
505 if undone:
506 self._report_undone(execution, undone)
507 return [item.split(" ", 1)[0] for item in undone]
509 def _report_undone(self, execution: Execution, undone: list[str]) -> None:
510 finding = self.results.record_finding(
511 execution,
512 validator_id=ACCEPTANCE_ID,
513 rule_id=WRITE_UNDONE_RULE,
514 category="acceptance-tests",
515 severity=FindingSeverity.HIGH,
516 message=(
517 "The agent changed frozen acceptance test file(s); the harness put the approved "
518 f"content back: {', '.join(undone)}"
519 ),
520 path=undone[0].split(" ", 1)[0],
521 recommendation=(
522 "The frozen acceptance tests are write-protected "
523 "(verification.acceptanceTests.protect): change the code, or dispute a test in "
524 "the self-report."
525 ),
526 introduced=True,
527 )
528 self.results.s.events.append(
529 execution.execution_id,
530 "acceptance.write.undone",
531 {"files": undone, "findingId": finding.finding_id},
532 )
534 def request_extra(self, execution: Execution) -> dict[str, Any] | None:
535 extra = self._request_extra(execution)
536 if extra is not None and self.protects:
537 extra["note"] += " The files are write-protected: a change to them is undone."
538 return extra
540 def _request_extra(self, execution: Execution) -> dict[str, Any] | None:
541 state = self.state(execution)
542 if not state or state.get("status") != "APPROVED" or not state.get("frozen"):
543 return None
544 if state.get("format") == "gherkin":
545 return {
546 "paths": sorted(state["frozen"]),
547 "frozen": True,
548 "format": "gherkin",
549 "runner": list(self._command([])),
550 "note": "Write the step definitions and the code that make these scenarios "
551 "pass; do not modify or delete the feature files.",
552 }
553 return {
554 "paths": sorted(state["frozen"]),
555 "frozen": True,
556 "note": "Make these tests pass; do not modify or delete them.",
557 }
559 def validation(self, execution: Execution, change_set: ChangeSet) -> ValidatorOutput | None:
560 state = self.state(execution)
561 if not self.enabled or not state or state.get("status") != "APPROVED":
562 return None
563 frozen: dict[str, str] = state.get("frozen") or {}
564 if not frozen: 564 ↛ 565line 564 didn't jump to line 565 because the condition on line 564 was never true
565 return None
566 results = self.results
567 findings = self._frozen_findings(execution, frozen)
568 output = self._run_frozen(execution, change_set, frozen)
569 failed = bool(findings) or output.result.status is not ResultStatus.PASSED
570 result = results.record_validation(
571 execution,
572 validator_id=ACCEPTANCE_ID,
573 digest=change_set.digest,
574 status=output.result.status if not findings else ResultStatus.FAILED,
575 kind=ValidationKind.VALIDATION_FAILURE if failed else ValidationKind.SUCCESS,
576 mandatory=True,
577 summary=(
578 f"frozen acceptance tests: {output.result.summary}"
579 + (f"; {len(findings)} frozen file(s) changed" if findings else "")
580 ),
581 findings=(*findings, *output.findings),
582 evidence_refs=output.result.evidence_refs,
583 )
584 return ValidatorOutput(result, ())
586 def _frozen_findings(self, execution: Execution, frozen: dict[str, str]) -> list[Finding]:
587 """A HIGH finding for every frozen acceptance test deleted or modified."""
588 results = self.results
589 workspace = results.s.paths.workspace
590 findings: list[Finding] = []
591 for path, digest in sorted(frozen.items()):
592 target = workspace / path
593 current = sha256_bytes(target.read_bytes()) if target.is_file() else None
594 if current != digest:
595 findings.append(
596 results.record_finding(
597 execution,
598 validator_id=ACCEPTANCE_ID,
599 rule_id="acceptance.modified",
600 category="acceptance-tests",
601 severity=FindingSeverity.HIGH,
602 message=(
603 f"The frozen acceptance test {path} was "
604 + ("deleted" if current is None else "modified")
605 ),
606 path=path,
607 recommendation="Restore the approved acceptance test; change the code.",
608 introduced=True,
609 )
610 )
611 return findings
613 def _run_frozen(
614 self, execution: Execution, change_set: ChangeSet, frozen: dict[str, str]
615 ) -> ValidatorOutput:
616 """Run the frozen acceptance tests and record their tool invocations and findings."""
617 results = self.results
618 workspace = results.s.paths.workspace
619 engine = results.engine
620 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{ACCEPTANCE_ID}", version="1")
621 from governed_harness.configuration.models import ValidatorDefinition
622 from governed_harness.validators import ValidationContext
624 output = CommandValidator(ACCEPTANCE_ID).execute(
625 ValidationContext(
626 execution_id=execution.execution_id,
627 workspace=workspace,
628 task=engine.run_task(execution),
629 change_set=change_set,
630 definition=engine._bounded_definition(
631 ValidatorDefinition(
632 id=ACCEPTANCE_ID,
633 command=self._command(sorted(frozen)),
634 mandatory=True,
635 )
636 ),
637 grants=grants_from_rules(
638 execution.execution_id, actor, results.s.resolved.effective_capabilities
639 ),
640 artifact_store=results.s.artifacts,
641 process_runner=engine._runner(execution),
642 provenance=engine._provenance(execution).model_copy(update={"actor": actor}),
643 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)),
644 max_output_bytes=results.project.runtime.max_output_bytes,
645 parse_output=True,
646 )
647 )
648 for tool in output.tool_invocations:
649 engine._save_tool(execution, tool)
650 for finding in output.findings:
651 results.s.state.put(
652 "finding",
653 finding.finding_id,
654 finding,
655 execution_id=execution.execution_id,
656 project_id=execution.project_id,
657 )
658 results.s.events.append(
659 execution.execution_id,
660 "finding.recorded",
661 finding.model_dump(mode="json"),
662 actor=finding.provenance.actor,
663 )
664 return output
667def _carried(state: dict[str, Any] | None) -> dict[str, Any]:
668 """What a proposal made for a revision (#94) carries over: its number, the tests approved
669 before and the dispute; empty for a first proposal."""
670 if not state or state.get("status") != "REVISING":
671 return {}
672 return {key: state[key] for key in ("revision", "previous", "dispute") if key in state}
675def _revision_request(carried: dict[str, Any]) -> dict[str, Any]:
676 """The ``revision`` block of the acceptance request: what was disputed and why a person
677 agreed."""
678 dispute = carried.get("dispute")
679 if not dispute:
680 return {}
681 return {
682 "revision": {
683 "disputed": dispute.get("disputed") or [],
684 "rationale": dispute.get("rationale"),
685 "note": REVISION_NOTE,
686 }
687 }
690def _approved_contents(state: dict[str, Any]) -> dict[str, bytes]:
691 """The approved content of each frozen file, by the path it was written to."""
692 renamed = state.get("renamed") or {}
693 return {
694 renamed.get(item["path"], item["path"]): item["content"].encode("utf-8")
695 for item in state.get("tests") or []
696 }
699def _put_back(workspace: Path, path: str, digest: str, content: bytes | None) -> str | None:
700 """Write the approved content of a frozen file that changed; what had happened to it
701 (``deleted`` or ``modified``), or ``None`` when it is as approved or cannot be restored."""
702 target = workspace / path
703 change = _frozen_change(target, digest)
704 if change is None or content is None or sha256_bytes(content) != digest:
705 return None
706 if target.is_symlink(): 706 ↛ 707line 706 didn't jump to line 707 because the condition on line 706 was never true
707 target.unlink()
708 safe = contained_path(workspace, Path(path))
709 safe.parent.mkdir(parents=True, exist_ok=True)
710 safe.write_bytes(content)
711 return change
714def _frozen_change(target: Path, digest: str) -> str | None:
715 """What happened to a frozen file: ``deleted`` when it is no longer a regular file,
716 ``modified`` when its content differs, ``None`` when it is as approved."""
717 if target.is_symlink(): 717 ↛ 718line 717 didn't jump to line 718 because the condition on line 717 was never true
718 return "deleted"
719 try:
720 current = target.read_bytes()
721 except OSError:
722 return "deleted"
723 return None if sha256_bytes(current) == digest else "modified"
726__all__ = ["ACCEPTANCE_ID", "WRITE_UNDONE_RULE", "AcceptanceTests", "validate_tests"]