Coverage for src/governed_harness/orchestration/gate_contract.py: 95%

211 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""The gate contract in the provider request and ``harness check`` (#52, N3). 

2 

3The evaluation's agents did not know what the gate would run: about one turn in ten was a 

4denied attempt to run Ruff or Mypy, and correct work was rejected for a rule the agent never 

5saw. Under ``runtime.gateContract`` the implement request carries the contract: the validators 

6(command, mandatory or optional), the enabled checks and their policies, the review rules, the 

7blocking severities, the absolute workspace path and a command, ``harness check``, that runs the 

8same validators and checks on the workspace without recording anything. 

9 

10``harness check`` reads the project configuration and a check state file the harness writes 

11before each implement call (``.harness/check/<run>.json``: the task and the baseline), so it 

12needs no write access to ``.harness`` and can run inside the agent sandbox. Its artifacts go to 

13a temporary directory that is removed afterwards.""" 

14 

15from __future__ import annotations 

16 

17import json 

18import tempfile 

19from collections.abc import Callable 

20from dataclasses import replace 

21from pathlib import Path 

22from typing import TYPE_CHECKING, Any 

23 

24from governed_harness import __version__ 

25from governed_harness.capabilities import grants_from_rules 

26from governed_harness.checks import parse_unified_diff 

27from governed_harness.configuration import ConfigurationResolver 

28from governed_harness.configuration.models import ( 

29 ResolvedConfiguration, 

30 ValidatorDefinition, 

31 VerificationConfig, 

32) 

33from governed_harness.domain.enums import ActorType, FindingSeverity, ResultStatus 

34from governed_harness.domain.errors import NotFoundError 

35from governed_harness.domain.models import ( 

36 Actor, 

37 CapabilityGrant, 

38 ChangeSet, 

39 Execution, 

40 Provenance, 

41 Task, 

42) 

43from governed_harness.evidence import LocalArtifactStore 

44from governed_harness.runtime import CancellationToken, SafeProcessRunner 

45from governed_harness.runtime.snapshots import SnapshotSettings, SnapshotStore 

46from governed_harness.runtime.state_location import resolve_state_location 

47from governed_harness.validators import CommandValidator, ValidationContext 

48from governed_harness.validators.review import RULES 

49 

50if TYPE_CHECKING: 

51 from governed_harness.orchestration.hosts import ResultsHost 

52 from governed_harness.validators.base import ValidatorOutput 

53 

54CHECK_DIRECTORY = "check" 

55 

56 

57def check_state_path(harness_dir: Path, execution_id: str) -> Path: 

58 return harness_dir / CHECK_DIRECTORY / f"{execution_id}.json" 

59 

60 

61class GateContract: 

62 def __init__(self, results: ResultsHost) -> None: 

63 self.results = results 

64 

65 @property 

66 def enabled(self) -> bool: 

67 return bool(self.results.project.runtime.gate_contract) 

68 

69 def validators(self) -> list[ValidatorDefinition]: 

70 resolved = self.results.s.resolved 

71 definitions = list(resolved.effective_validators) 

72 verification = resolved.project.verification 

73 for invariant in (verification.invariants if verification else None) or (): 73 ↛ 74line 73 didn't jump to line 74 because the loop on line 73 never started

74 definitions.append( 

75 ValidatorDefinition( 

76 id=f"invariant.{invariant.invariant_id}", command=invariant.command 

77 ) 

78 ) 

79 return definitions 

80 

81 def contract( 

82 self, 

83 execution: Execution, 

84 task: Task | None = None, 

85 grants: list[CapabilityGrant] | None = None, 

86 ) -> dict[str, Any]: 

87 """The gate contract of an implement call. With ``task``, also what the gate holds the 

88 change to from the task: the requirement identifiers and the naming rule that 

89 requirement traceability checks (#84) and the assumptions (#79). With the agent's 

90 ``grants``, a check command only when the agent may run it (#84): ``harness check`` 

91 when its grants allow it, else the validator commands they allow (``checkCommands``, 

92 possibly none).""" 

93 value = self._contract(execution) 

94 if task is not None: 94 ↛ 97line 94 didn't jump to line 97 because the condition on line 94 was always true

95 value.update(task_terms(task)) 

96 value.update(traceability_terms(task, self.results.project.requirement_traceability)) 

97 if grants is not None and not runnable(value["checkCommand"], grants): 

98 del value["checkCommand"] 

99 value["checkCommands"] = [ 

100 {"id": item["id"], "command": item["command"]} 

101 for item in value["validators"] 

102 if item["command"] and runnable(item["command"], grants) 

103 ] 

104 return value 

105 

106 @staticmethod 

107 def instructions_note(contract: dict[str, Any]) -> str: 

108 """What the implement instructions add when the contract has no ``harness check`` 

109 command the agent may run (#84); empty otherwise.""" 

110 if "checkCommand" in contract: 

111 return "" 

112 if contract.get("checkCommands"): 

113 return ( 

114 "Your permissions do not include the harness check command: instead, run the " 

115 "validator commands listed in gate.checkCommands before you finish." 

116 ) 

117 return ( 

118 "Your permissions include no command of the gate: do not try to run the check " 

119 "command or the validators; the harness runs them after you finish." 

120 ) 

121 

122 def _contract(self, execution: Execution) -> dict[str, Any]: 

123 resolved = self.results.s.resolved 

124 project = resolved.project 

125 workspace = str(self.results.s.paths.workspace) 

126 verification = project.verification 

127 checks = self._checks(verification) if verification is not None else {} 

128 if self.results.engineering.configured: 

129 layers = self.results.architecture.rules() 

130 if layers is not None and layers.layers: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true

131 checks["layers"] = layers.as_dict() 

132 skipped = {"review.possible-secret"} if self.results.secrets_in_context else set() 

133 return { 

134 "workspace": workspace, 

135 "validators": [ 

136 { 

137 "id": item.validator_id, 

138 "command": list(item.command or ()), 

139 "mandatory": item.mandatory, 

140 } 

141 for item in self.validators() 

142 ], 

143 "checks": checks, 

144 "requirementTraceability": project.requirement_traceability, 

145 "reviewRules": [ 

146 {"ruleId": rule.rule_id, "severity": rule.severity, "message": rule.message} 

147 for rule in RULES 

148 if rule.rule_id not in skipped 

149 ], 

150 "blockingSeverities": list( 

151 resolved.effective_policies.get("findingBlockSeverities", ["HIGH", "CRITICAL"]) 

152 ), 

153 "checkCommand": [ 

154 "harness", 

155 "check", 

156 "--path", 

157 workspace, 

158 "--run", 

159 execution.execution_id, 

160 ], 

161 } 

162 

163 def _checks(self, verification: VerificationConfig) -> dict[str, Any]: 

164 """The diff checks of ``verification`` the gate runs, as the contract shows them.""" 

165 checks: dict[str, Any] = {} 

166 for name, alias in ( 

167 ("interface", "interface"), 

168 ("constraints", "constraints"), 

169 ("weakened_controls", "weakenedControls"), 

170 ("ratchet", "ratchet"), 

171 ("secrets", "secrets"), 

172 ("security_patterns", "securityPatterns"), 

173 ("differential", "differential"), 

174 ): 

175 value = getattr(verification, name) 

176 if value not in {None, "off", False}: 

177 checks[alias] = value 

178 if verification.architecture is not None: 

179 checks["architecture"] = verification.architecture.model_dump( 

180 mode="json", by_alias=True 

181 ) 

182 if verification.test_quality is not None: 

183 checks["testQuality"] = verification.test_quality.model_dump(mode="json", by_alias=True) 

184 if verification.risk_factors is not None: 

185 checks["riskFactors"] = self.results.verification.risk_actions() 

186 if verification.principles is not None: 

187 checks["principles"] = verification.principles.model_dump(mode="json", by_alias=True) 

188 return checks 

189 

190 def write_check_state(self, execution: Execution, task: Task) -> Path: 

191 """The task and the baseline ``harness check`` compares the workspace with.""" 

192 path = check_state_path(self.results.s.paths.harness_dir, execution.execution_id) 

193 path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) 

194 value = { 

195 "executionId": execution.execution_id, 

196 "task": task.model_dump(mode="json", by_alias=True), 

197 # The stored baseline (text snapshot or manifest), read from the artifact store. 

198 "baselineRef": self.results.s.state.get_flag(f"baseline:{execution.execution_id}"), 

199 } 

200 path.write_text(json.dumps(value, sort_keys=True), encoding="utf-8") 

201 return path 

202 

203 

204TRACEABILITY_RULE = ( 

205 "A requirement is traced when a test names its identifier: the name of a test file, class " 

206 "or function contains the identifier as a run of whole tokens in any case (for A1: " 

207 "test_a1_rounding, test_A1, TestA1Rounding; for req_alphabet: test_req_alphabet_rejects), " 

208 "or the identifier appears as a whole word in a test's source, docstring or string " 

209 "constants (pytest.param(..., id='A1'), # covers A1). Test files are Python test_*.py or " 

210 "*_test.py, and Node *.test.*, *.spec.* or files under test/, tests/ or __tests__/. Name " 

211 "each test after the identifier of the requirement it checks." 

212) 

213"""How requirement traceability decides that a test names a requirement (see 

214``governed_harness.validators.traceability``), as the gate contract states it (#84).""" 

215 

216 

217def traceability_terms(task: Task, policy: str) -> dict[str, Any]: 

218 """The requirement identifiers the traceability check looks for in the tests and its naming 

219 rule (#84); nothing when the check is off.""" 

220 from governed_harness.validators.traceability import requirement_identifier 

221 

222 if policy == "off": 

223 return {} 

224 identified: list[dict[str, str]] = [] 

225 skipped: list[str] = [] 

226 for requirement in task.requirements: 

227 found = requirement_identifier(requirement) 

228 if found is None: 

229 skipped.append(requirement.requirement_id) 

230 continue 

231 identified.append( 

232 { 

233 "identifier": found.identifier, 

234 "requirementId": requirement.requirement_id, 

235 "text": requirement.text[:300], 

236 } 

237 ) 

238 return { 

239 "traceability": { 

240 "policy": policy, 

241 "requirements": identified, 

242 "notChecked": skipped, 

243 "rule": TRACEABILITY_RULE, 

244 } 

245 } 

246 

247 

248def runnable(argv: list[str], grants: list[CapabilityGrant]) -> bool: 

249 """Whether the agent's ``process.execute`` grants allow ``argv`` (#84).""" 

250 from governed_harness.capabilities.authorizer import CapabilityAuthorizer, CapabilityDenied 

251 

252 if not argv or not grants: 

253 return False 

254 try: 

255 CapabilityAuthorizer().authorize_command(actor=grants[0].actor, argv=argv, grants=grants) 

256 except CapabilityDenied: 

257 return False 

258 return True 

259 

260 

261def task_terms(task: Task) -> dict[str, Any]: 

262 """What the task revision adds to the gate contract: the points the agent review left open 

263 (#79), as assumptions the implementation states it took.""" 

264 from governed_harness.orchestration.intent_convergence import assumptions_of 

265 

266 assumptions = assumptions_of(task) 

267 return {"assumptions": assumptions} if assumptions else {} 

268 

269 

270def permissions( 

271 kind: str, grants: list[CapabilityGrant], read_only: bool, network: bool 

272) -> dict[str, Any]: 

273 """What a call kind may do, derived from the capability grants of the agent actor 

274 (``governance.phasePermissions``): read-only kinds get no write scope.""" 

275 scopes: dict[str, list[str]] = {} 

276 for grant in grants: 

277 if grant.approval_required: 277 ↛ 278line 277 didn't jump to line 278 because the condition on line 277 was never true

278 continue 

279 scopes.setdefault(grant.capability, []).extend(grant.scope) 

280 write = [] if read_only else sorted(set(scopes.get("filesystem.write", []))) 

281 return { 

282 "callKind": kind, 

283 "readOnly": read_only, 

284 "filesystem": { 

285 "read": sorted(set(scopes.get("filesystem.read", []))), 

286 "write": write, 

287 }, 

288 "process": sorted(set(scopes.get("process.execute", []))), 

289 "network": network, 

290 "other": sorted( 

291 name 

292 for name in scopes 

293 if name not in {"filesystem.read", "filesystem.write", "process.execute"} 

294 ), 

295 } 

296 

297 

298# ----- harness check ------------------------------------------------------------------------ 

299def run_check(path: Path, execution_id: str | None = None) -> dict[str, Any]: 

300 """Run the gate's validators and the diff checks on the workspace; nothing is recorded.""" 

301 resolved = ConfigurationResolver().resolve(path) 

302 workspace = resolved.workspace_root.resolve() 

303 state = _check_state(workspace / ".harness", execution_id) 

304 task = Task.model_validate(state["task"]) if state else None 

305 baseline_ref = state.get("baselineRef") if state else None 

306 run_id = str(state["executionId"]) if state else "check" 

307 blocking_names = resolved.effective_policies.get("findingBlockSeverities", ["HIGH", "CRITICAL"]) 

308 blocking = {FindingSeverity(str(name)) for name in blocking_names} 

309 diff_text, digest, baseline_text = _check_baseline(resolved, workspace, baseline_ref) 

310 validators = _check_validators(resolved, workspace, run_id, task, digest) 

311 checks = _check_diff(resolved, workspace, task, diff_text, baseline_text) 

312 failed_validators = [ 

313 item for item in validators if item["mandatory"] and item["status"] != "PASSED" 

314 ] 

315 blocking_issues = [ 

316 issue 

317 for check in checks 

318 if check["policy"] == "enforce" 

319 for issue in check["issues"] 

320 if FindingSeverity(issue["severity"]) in blocking 

321 ] 

322 status = ResultStatus.FAILED if failed_validators or blocking_issues else ResultStatus.PASSED 

323 return { 

324 "status": status.value, 

325 "executionId": state["executionId"] if state else None, 

326 "workspace": str(workspace), 

327 "validators": validators, 

328 "checks": checks, 

329 "blockingIssues": len(blocking_issues), 

330 "note": ( 

331 "Nothing was recorded. The gate also runs the independent review and, when " 

332 "configured, requirement traceability, coverage, SARIF and the baseline comparison." 

333 ), 

334 } 

335 

336 

337def _check_baseline( 

338 resolved: ResolvedConfiguration, workspace: Path, baseline_ref: Any 

339) -> tuple[str, str, Callable[[str], str | None] | None]: 

340 """The diff against the run's baseline snapshot (its text and digest) and a reader of the 

341 baseline files; without a baseline an empty diff and no reader.""" 

342 diff_text = "" 

343 digest = "sha256:check" 

344 baseline_text: Callable[[str], str | None] | None = None 

345 if isinstance(baseline_ref, str): 

346 # Read-only: the artifact store and the snapshot listing without the snapshot cache, 

347 # so the check runs inside the agent sandbox, which keeps .harness read-only. 

348 # The registry may live outside the workspace (runtime.stateDir, #55); it is only read. 

349 location = resolve_state_location( 

350 workspace, resolved.project.project_id, resolved.project.runtime.state_dir, create=False 

351 ) 

352 snapshots = SnapshotStore( 

353 workspace, 

354 LocalArtifactStore(location.artifacts), 

355 replace(SnapshotSettings.from_config(resolved.project.workspace), cache=False), 

356 ) 

357 stored = snapshots.load(baseline_ref) 

358 diff = snapshots.diff(stored, snapshots.take()) 

359 diff_text = diff.unified_diff.decode("utf-8", "replace") 

360 digest = diff.digest 

361 

362 def baseline_text(path: str) -> str | None: 

363 data = snapshots.content(stored, path) 

364 return data.decode("utf-8", "replace") if data is not None else None 

365 

366 return diff_text, digest, baseline_text 

367 

368 

369def _check_validators( 

370 resolved: ResolvedConfiguration, 

371 workspace: Path, 

372 run_id: str, 

373 task: Task | None, 

374 digest: str, 

375) -> list[dict[str, Any]]: 

376 """Run every validator of the gate in a scratch artifact store and report each one.""" 

377 validators: list[dict[str, Any]] = [] 

378 with tempfile.TemporaryDirectory(prefix="harness-check-") as scratch: 

379 store = LocalArtifactStore(Path(scratch) / "artifacts") 

380 change_set = ChangeSet( 

381 change_set_id="check", execution_id=run_id, files=(), diff_ref="check", digest=digest 

382 ) 

383 for definition in _definitions(resolved): 

384 output = _run_validator( 

385 resolved, workspace, run_id, task, change_set, definition, store 

386 ) 

387 validators.append(_validator_entry(definition, output, store)) 

388 return validators 

389 

390 

391def _run_validator( 

392 resolved: ResolvedConfiguration, 

393 workspace: Path, 

394 run_id: str, 

395 task: Task | None, 

396 change_set: ChangeSet, 

397 definition: ValidatorDefinition, 

398 store: LocalArtifactStore, 

399) -> ValidatorOutput: 

400 """Run one validator of the gate with the grants of the configuration.""" 

401 actor = Actor( 

402 actor_type=ActorType.TOOL, 

403 actor_id=f"validator.{definition.validator_id}", 

404 version="1", 

405 ) 

406 return CommandValidator(definition.validator_id).execute( 

407 ValidationContext( 

408 execution_id=run_id, 

409 workspace=workspace, 

410 task=task or _placeholder_task(resolved), 

411 change_set=change_set, 

412 definition=definition, 

413 grants=grants_from_rules(run_id, actor, resolved.effective_capabilities), 

414 artifact_store=store, 

415 process_runner=SafeProcessRunner(workspace), 

416 provenance=Provenance(actor=actor, core_version=__version__), 

417 cancellation=CancellationToken(lambda: False), 

418 max_output_bytes=resolved.project.runtime.max_output_bytes, 

419 parse_output=True, 

420 ) 

421 ) 

422 

423 

424def _validator_entry( 

425 definition: ValidatorDefinition, output: ValidatorOutput, store: LocalArtifactStore 

426) -> dict[str, Any]: 

427 """The report of one validator: status, summary, the first problems and its output.""" 

428 return { 

429 "id": definition.validator_id, 

430 "mandatory": definition.mandatory, 

431 "status": output.result.status.value, 

432 "summary": output.result.summary, 

433 "problems": [ 

434 { 

435 "rule": item.rule_id, 

436 "path": item.location.path if item.location else None, 

437 "line": item.location.start_line if item.location else None, 

438 "message": item.message, 

439 } 

440 for item in output.findings[1:51] 

441 ], 

442 "output": _tail(store, output.result.evidence_refs), 

443 } 

444 

445 

446def _check_diff( 

447 resolved: ResolvedConfiguration, 

448 workspace: Path, 

449 task: Task | None, 

450 diff_text: str, 

451 baseline_text: Callable[[str], str | None] | None, 

452) -> list[dict[str, Any]]: 

453 """The checks that read only the diff and the files, when the run has a task and a 

454 baseline and verification or architecture rules are configured.""" 

455 from governed_harness.orchestration.architecture import effective_rules 

456 from governed_harness.orchestration.verification_checks import ( 

457 current_files, 

458 pure_checks, 

459 ) 

460 

461 checks: list[dict[str, Any]] = [] 

462 verification = resolved.project.verification 

463 architecture = resolved.project.architecture 

464 layers = effective_rules(architecture, workspace / ".harness") 

465 if ( 

466 (verification is not None or layers is not None) 

467 and task is not None 

468 and (baseline_text is not None) 

469 ): 

470 parsed = parse_unified_diff(diff_text) 

471 for item in pure_checks( 

472 verification or VerificationConfig(), 

473 task, 

474 parsed, 

475 current_files(workspace, parsed), 

476 workspace, 

477 baseline_text, 

478 layers=layers, 

479 layer_policy=architecture.policy if architecture is not None else "enforce", 

480 layer_differential=bool(architecture is not None and architecture.differential), 

481 ): 

482 checks.append( 

483 { 

484 "id": item.validator_id, 

485 "policy": item.policy, 

486 "issues": [ 

487 { 

488 "rule": issue.rule_id, 

489 "severity": issue.severity.value, 

490 "path": issue.path, 

491 "line": issue.line, 

492 "message": issue.message, 

493 } 

494 for issue in item.issues 

495 ], 

496 } 

497 ) 

498 return checks 

499 

500 

501def _definitions(resolved: ResolvedConfiguration) -> list[ValidatorDefinition]: 

502 definitions = list(resolved.effective_validators) 

503 verification = resolved.project.verification 

504 for invariant in (verification.invariants if verification else None) or (): 504 ↛ 505line 504 didn't jump to line 505 because the loop on line 504 never started

505 definitions.append( 

506 ValidatorDefinition(id=f"invariant.{invariant.invariant_id}", command=invariant.command) 

507 ) 

508 return definitions 

509 

510 

511def _check_state(harness_dir: Path, execution_id: str | None) -> dict[str, Any] | None: 

512 directory = harness_dir / CHECK_DIRECTORY 

513 if execution_id: 

514 candidate = directory / f"{execution_id}.json" 

515 if not candidate.is_file(): 

516 raise NotFoundError(f"no check state for run {execution_id}") 

517 else: 

518 files = sorted(directory.glob("*.json"), key=lambda item: item.stat().st_mtime) 

519 if not files: 

520 return None 

521 candidate = files[-1] 

522 value = json.loads(candidate.read_text(encoding="utf-8")) 

523 return value if isinstance(value, dict) else None 

524 

525 

526def _placeholder_task(resolved: ResolvedConfiguration) -> Task: 

527 return Task.model_validate( 

528 { 

529 "taskId": "check", 

530 "projectId": resolved.project.project_id, 

531 "title": "harness check", 

532 "intent": "Run the gate's validators on the workspace.", 

533 "acceptanceCriteria": [{"criterionId": "check", "text": "The validators pass."}], 

534 } 

535 ) 

536 

537 

538def _tail(store: LocalArtifactStore, refs: tuple[str, ...], limit: int = 2000) -> str: 

539 try: 

540 report = json.loads(store.get(refs[0])) 

541 stdout = store.get(report["stdoutRef"]).decode("utf-8", "replace") 

542 stderr = store.get(report["stderrRef"]).decode("utf-8", "replace") 

543 except (OSError, ValueError, KeyError, IndexError, TypeError): 

544 return "" 

545 text = (stdout + ("\n" + stderr if stderr.strip() else "")).strip() 

546 return text[-limit:] 

547 

548 

549__all__ = [ 

550 "TRACEABILITY_RULE", 

551 "GateContract", 

552 "check_state_path", 

553 "permissions", 

554 "run_check", 

555 "runnable", 

556 "task_terms", 

557 "traceability_terms", 

558] 

559 

560 

561def gate_definitions(resolved: ResolvedConfiguration) -> list[ValidatorDefinition]: 

562 """The validators of the gate (the project's and the invariants), as the check runs them.""" 

563 return _definitions(resolved)