Coverage for src/governed_harness/orchestration/gate_contract.py: 95%
211 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""The gate contract in the provider request and ``harness check`` (#52, N3).
3The evaluation's agents did not know what the gate would run: about one turn in ten was a
4denied attempt to run Ruff or Mypy, and correct work was rejected for a rule the agent never
5saw. Under ``runtime.gateContract`` the implement request carries the contract: the validators
6(command, mandatory or optional), the enabled checks and their policies, the review rules, the
7blocking severities, the absolute workspace path and a command, ``harness check``, that runs the
8same validators and checks on the workspace without recording anything.
10``harness check`` reads the project configuration and a check state file the harness writes
11before each implement call (``.harness/check/<run>.json``: the task and the baseline), so it
12needs no write access to ``.harness`` and can run inside the agent sandbox. Its artifacts go to
13a temporary directory that is removed afterwards."""
15from __future__ import annotations
17import json
18import tempfile
19from collections.abc import Callable
20from dataclasses import replace
21from pathlib import Path
22from typing import TYPE_CHECKING, Any
24from governed_harness import __version__
25from governed_harness.capabilities import grants_from_rules
26from governed_harness.checks import parse_unified_diff
27from governed_harness.configuration import ConfigurationResolver
28from governed_harness.configuration.models import (
29 ResolvedConfiguration,
30 ValidatorDefinition,
31 VerificationConfig,
32)
33from governed_harness.domain.enums import ActorType, FindingSeverity, ResultStatus
34from governed_harness.domain.errors import NotFoundError
35from governed_harness.domain.models import (
36 Actor,
37 CapabilityGrant,
38 ChangeSet,
39 Execution,
40 Provenance,
41 Task,
42)
43from governed_harness.evidence import LocalArtifactStore
44from governed_harness.runtime import CancellationToken, SafeProcessRunner
45from governed_harness.runtime.snapshots import SnapshotSettings, SnapshotStore
46from governed_harness.runtime.state_location import resolve_state_location
47from governed_harness.validators import CommandValidator, ValidationContext
48from governed_harness.validators.review import RULES
50if TYPE_CHECKING:
51 from governed_harness.orchestration.hosts import ResultsHost
52 from governed_harness.validators.base import ValidatorOutput
54CHECK_DIRECTORY = "check"
57def check_state_path(harness_dir: Path, execution_id: str) -> Path:
58 return harness_dir / CHECK_DIRECTORY / f"{execution_id}.json"
61class GateContract:
62 def __init__(self, results: ResultsHost) -> None:
63 self.results = results
65 @property
66 def enabled(self) -> bool:
67 return bool(self.results.project.runtime.gate_contract)
69 def validators(self) -> list[ValidatorDefinition]:
70 resolved = self.results.s.resolved
71 definitions = list(resolved.effective_validators)
72 verification = resolved.project.verification
73 for invariant in (verification.invariants if verification else None) or (): 73 ↛ 74line 73 didn't jump to line 74 because the loop on line 73 never started
74 definitions.append(
75 ValidatorDefinition(
76 id=f"invariant.{invariant.invariant_id}", command=invariant.command
77 )
78 )
79 return definitions
81 def contract(
82 self,
83 execution: Execution,
84 task: Task | None = None,
85 grants: list[CapabilityGrant] | None = None,
86 ) -> dict[str, Any]:
87 """The gate contract of an implement call. With ``task``, also what the gate holds the
88 change to from the task: the requirement identifiers and the naming rule that
89 requirement traceability checks (#84) and the assumptions (#79). With the agent's
90 ``grants``, a check command only when the agent may run it (#84): ``harness check``
91 when its grants allow it, else the validator commands they allow (``checkCommands``,
92 possibly none)."""
93 value = self._contract(execution)
94 if task is not None: 94 ↛ 97line 94 didn't jump to line 97 because the condition on line 94 was always true
95 value.update(task_terms(task))
96 value.update(traceability_terms(task, self.results.project.requirement_traceability))
97 if grants is not None and not runnable(value["checkCommand"], grants):
98 del value["checkCommand"]
99 value["checkCommands"] = [
100 {"id": item["id"], "command": item["command"]}
101 for item in value["validators"]
102 if item["command"] and runnable(item["command"], grants)
103 ]
104 return value
106 @staticmethod
107 def instructions_note(contract: dict[str, Any]) -> str:
108 """What the implement instructions add when the contract has no ``harness check``
109 command the agent may run (#84); empty otherwise."""
110 if "checkCommand" in contract:
111 return ""
112 if contract.get("checkCommands"):
113 return (
114 "Your permissions do not include the harness check command: instead, run the "
115 "validator commands listed in gate.checkCommands before you finish."
116 )
117 return (
118 "Your permissions include no command of the gate: do not try to run the check "
119 "command or the validators; the harness runs them after you finish."
120 )
122 def _contract(self, execution: Execution) -> dict[str, Any]:
123 resolved = self.results.s.resolved
124 project = resolved.project
125 workspace = str(self.results.s.paths.workspace)
126 verification = project.verification
127 checks = self._checks(verification) if verification is not None else {}
128 if self.results.engineering.configured:
129 layers = self.results.architecture.rules()
130 if layers is not None and layers.layers: 130 ↛ 131line 130 didn't jump to line 131 because the condition on line 130 was never true
131 checks["layers"] = layers.as_dict()
132 skipped = {"review.possible-secret"} if self.results.secrets_in_context else set()
133 return {
134 "workspace": workspace,
135 "validators": [
136 {
137 "id": item.validator_id,
138 "command": list(item.command or ()),
139 "mandatory": item.mandatory,
140 }
141 for item in self.validators()
142 ],
143 "checks": checks,
144 "requirementTraceability": project.requirement_traceability,
145 "reviewRules": [
146 {"ruleId": rule.rule_id, "severity": rule.severity, "message": rule.message}
147 for rule in RULES
148 if rule.rule_id not in skipped
149 ],
150 "blockingSeverities": list(
151 resolved.effective_policies.get("findingBlockSeverities", ["HIGH", "CRITICAL"])
152 ),
153 "checkCommand": [
154 "harness",
155 "check",
156 "--path",
157 workspace,
158 "--run",
159 execution.execution_id,
160 ],
161 }
163 def _checks(self, verification: VerificationConfig) -> dict[str, Any]:
164 """The diff checks of ``verification`` the gate runs, as the contract shows them."""
165 checks: dict[str, Any] = {}
166 for name, alias in (
167 ("interface", "interface"),
168 ("constraints", "constraints"),
169 ("weakened_controls", "weakenedControls"),
170 ("ratchet", "ratchet"),
171 ("secrets", "secrets"),
172 ("security_patterns", "securityPatterns"),
173 ("differential", "differential"),
174 ):
175 value = getattr(verification, name)
176 if value not in {None, "off", False}:
177 checks[alias] = value
178 if verification.architecture is not None:
179 checks["architecture"] = verification.architecture.model_dump(
180 mode="json", by_alias=True
181 )
182 if verification.test_quality is not None:
183 checks["testQuality"] = verification.test_quality.model_dump(mode="json", by_alias=True)
184 if verification.risk_factors is not None:
185 checks["riskFactors"] = self.results.verification.risk_actions()
186 if verification.principles is not None:
187 checks["principles"] = verification.principles.model_dump(mode="json", by_alias=True)
188 return checks
190 def write_check_state(self, execution: Execution, task: Task) -> Path:
191 """The task and the baseline ``harness check`` compares the workspace with."""
192 path = check_state_path(self.results.s.paths.harness_dir, execution.execution_id)
193 path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
194 value = {
195 "executionId": execution.execution_id,
196 "task": task.model_dump(mode="json", by_alias=True),
197 # The stored baseline (text snapshot or manifest), read from the artifact store.
198 "baselineRef": self.results.s.state.get_flag(f"baseline:{execution.execution_id}"),
199 }
200 path.write_text(json.dumps(value, sort_keys=True), encoding="utf-8")
201 return path
204TRACEABILITY_RULE = (
205 "A requirement is traced when a test names its identifier: the name of a test file, class "
206 "or function contains the identifier as a run of whole tokens in any case (for A1: "
207 "test_a1_rounding, test_A1, TestA1Rounding; for req_alphabet: test_req_alphabet_rejects), "
208 "or the identifier appears as a whole word in a test's source, docstring or string "
209 "constants (pytest.param(..., id='A1'), # covers A1). Test files are Python test_*.py or "
210 "*_test.py, and Node *.test.*, *.spec.* or files under test/, tests/ or __tests__/. Name "
211 "each test after the identifier of the requirement it checks."
212)
213"""How requirement traceability decides that a test names a requirement (see
214``governed_harness.validators.traceability``), as the gate contract states it (#84)."""
217def traceability_terms(task: Task, policy: str) -> dict[str, Any]:
218 """The requirement identifiers the traceability check looks for in the tests and its naming
219 rule (#84); nothing when the check is off."""
220 from governed_harness.validators.traceability import requirement_identifier
222 if policy == "off":
223 return {}
224 identified: list[dict[str, str]] = []
225 skipped: list[str] = []
226 for requirement in task.requirements:
227 found = requirement_identifier(requirement)
228 if found is None:
229 skipped.append(requirement.requirement_id)
230 continue
231 identified.append(
232 {
233 "identifier": found.identifier,
234 "requirementId": requirement.requirement_id,
235 "text": requirement.text[:300],
236 }
237 )
238 return {
239 "traceability": {
240 "policy": policy,
241 "requirements": identified,
242 "notChecked": skipped,
243 "rule": TRACEABILITY_RULE,
244 }
245 }
248def runnable(argv: list[str], grants: list[CapabilityGrant]) -> bool:
249 """Whether the agent's ``process.execute`` grants allow ``argv`` (#84)."""
250 from governed_harness.capabilities.authorizer import CapabilityAuthorizer, CapabilityDenied
252 if not argv or not grants:
253 return False
254 try:
255 CapabilityAuthorizer().authorize_command(actor=grants[0].actor, argv=argv, grants=grants)
256 except CapabilityDenied:
257 return False
258 return True
261def task_terms(task: Task) -> dict[str, Any]:
262 """What the task revision adds to the gate contract: the points the agent review left open
263 (#79), as assumptions the implementation states it took."""
264 from governed_harness.orchestration.intent_convergence import assumptions_of
266 assumptions = assumptions_of(task)
267 return {"assumptions": assumptions} if assumptions else {}
270def permissions(
271 kind: str, grants: list[CapabilityGrant], read_only: bool, network: bool
272) -> dict[str, Any]:
273 """What a call kind may do, derived from the capability grants of the agent actor
274 (``governance.phasePermissions``): read-only kinds get no write scope."""
275 scopes: dict[str, list[str]] = {}
276 for grant in grants:
277 if grant.approval_required: 277 ↛ 278line 277 didn't jump to line 278 because the condition on line 277 was never true
278 continue
279 scopes.setdefault(grant.capability, []).extend(grant.scope)
280 write = [] if read_only else sorted(set(scopes.get("filesystem.write", [])))
281 return {
282 "callKind": kind,
283 "readOnly": read_only,
284 "filesystem": {
285 "read": sorted(set(scopes.get("filesystem.read", []))),
286 "write": write,
287 },
288 "process": sorted(set(scopes.get("process.execute", []))),
289 "network": network,
290 "other": sorted(
291 name
292 for name in scopes
293 if name not in {"filesystem.read", "filesystem.write", "process.execute"}
294 ),
295 }
298# ----- harness check ------------------------------------------------------------------------
299def run_check(path: Path, execution_id: str | None = None) -> dict[str, Any]:
300 """Run the gate's validators and the diff checks on the workspace; nothing is recorded."""
301 resolved = ConfigurationResolver().resolve(path)
302 workspace = resolved.workspace_root.resolve()
303 state = _check_state(workspace / ".harness", execution_id)
304 task = Task.model_validate(state["task"]) if state else None
305 baseline_ref = state.get("baselineRef") if state else None
306 run_id = str(state["executionId"]) if state else "check"
307 blocking_names = resolved.effective_policies.get("findingBlockSeverities", ["HIGH", "CRITICAL"])
308 blocking = {FindingSeverity(str(name)) for name in blocking_names}
309 diff_text, digest, baseline_text = _check_baseline(resolved, workspace, baseline_ref)
310 validators = _check_validators(resolved, workspace, run_id, task, digest)
311 checks = _check_diff(resolved, workspace, task, diff_text, baseline_text)
312 failed_validators = [
313 item for item in validators if item["mandatory"] and item["status"] != "PASSED"
314 ]
315 blocking_issues = [
316 issue
317 for check in checks
318 if check["policy"] == "enforce"
319 for issue in check["issues"]
320 if FindingSeverity(issue["severity"]) in blocking
321 ]
322 status = ResultStatus.FAILED if failed_validators or blocking_issues else ResultStatus.PASSED
323 return {
324 "status": status.value,
325 "executionId": state["executionId"] if state else None,
326 "workspace": str(workspace),
327 "validators": validators,
328 "checks": checks,
329 "blockingIssues": len(blocking_issues),
330 "note": (
331 "Nothing was recorded. The gate also runs the independent review and, when "
332 "configured, requirement traceability, coverage, SARIF and the baseline comparison."
333 ),
334 }
337def _check_baseline(
338 resolved: ResolvedConfiguration, workspace: Path, baseline_ref: Any
339) -> tuple[str, str, Callable[[str], str | None] | None]:
340 """The diff against the run's baseline snapshot (its text and digest) and a reader of the
341 baseline files; without a baseline an empty diff and no reader."""
342 diff_text = ""
343 digest = "sha256:check"
344 baseline_text: Callable[[str], str | None] | None = None
345 if isinstance(baseline_ref, str):
346 # Read-only: the artifact store and the snapshot listing without the snapshot cache,
347 # so the check runs inside the agent sandbox, which keeps .harness read-only.
348 # The registry may live outside the workspace (runtime.stateDir, #55); it is only read.
349 location = resolve_state_location(
350 workspace, resolved.project.project_id, resolved.project.runtime.state_dir, create=False
351 )
352 snapshots = SnapshotStore(
353 workspace,
354 LocalArtifactStore(location.artifacts),
355 replace(SnapshotSettings.from_config(resolved.project.workspace), cache=False),
356 )
357 stored = snapshots.load(baseline_ref)
358 diff = snapshots.diff(stored, snapshots.take())
359 diff_text = diff.unified_diff.decode("utf-8", "replace")
360 digest = diff.digest
362 def baseline_text(path: str) -> str | None:
363 data = snapshots.content(stored, path)
364 return data.decode("utf-8", "replace") if data is not None else None
366 return diff_text, digest, baseline_text
369def _check_validators(
370 resolved: ResolvedConfiguration,
371 workspace: Path,
372 run_id: str,
373 task: Task | None,
374 digest: str,
375) -> list[dict[str, Any]]:
376 """Run every validator of the gate in a scratch artifact store and report each one."""
377 validators: list[dict[str, Any]] = []
378 with tempfile.TemporaryDirectory(prefix="harness-check-") as scratch:
379 store = LocalArtifactStore(Path(scratch) / "artifacts")
380 change_set = ChangeSet(
381 change_set_id="check", execution_id=run_id, files=(), diff_ref="check", digest=digest
382 )
383 for definition in _definitions(resolved):
384 output = _run_validator(
385 resolved, workspace, run_id, task, change_set, definition, store
386 )
387 validators.append(_validator_entry(definition, output, store))
388 return validators
391def _run_validator(
392 resolved: ResolvedConfiguration,
393 workspace: Path,
394 run_id: str,
395 task: Task | None,
396 change_set: ChangeSet,
397 definition: ValidatorDefinition,
398 store: LocalArtifactStore,
399) -> ValidatorOutput:
400 """Run one validator of the gate with the grants of the configuration."""
401 actor = Actor(
402 actor_type=ActorType.TOOL,
403 actor_id=f"validator.{definition.validator_id}",
404 version="1",
405 )
406 return CommandValidator(definition.validator_id).execute(
407 ValidationContext(
408 execution_id=run_id,
409 workspace=workspace,
410 task=task or _placeholder_task(resolved),
411 change_set=change_set,
412 definition=definition,
413 grants=grants_from_rules(run_id, actor, resolved.effective_capabilities),
414 artifact_store=store,
415 process_runner=SafeProcessRunner(workspace),
416 provenance=Provenance(actor=actor, core_version=__version__),
417 cancellation=CancellationToken(lambda: False),
418 max_output_bytes=resolved.project.runtime.max_output_bytes,
419 parse_output=True,
420 )
421 )
424def _validator_entry(
425 definition: ValidatorDefinition, output: ValidatorOutput, store: LocalArtifactStore
426) -> dict[str, Any]:
427 """The report of one validator: status, summary, the first problems and its output."""
428 return {
429 "id": definition.validator_id,
430 "mandatory": definition.mandatory,
431 "status": output.result.status.value,
432 "summary": output.result.summary,
433 "problems": [
434 {
435 "rule": item.rule_id,
436 "path": item.location.path if item.location else None,
437 "line": item.location.start_line if item.location else None,
438 "message": item.message,
439 }
440 for item in output.findings[1:51]
441 ],
442 "output": _tail(store, output.result.evidence_refs),
443 }
446def _check_diff(
447 resolved: ResolvedConfiguration,
448 workspace: Path,
449 task: Task | None,
450 diff_text: str,
451 baseline_text: Callable[[str], str | None] | None,
452) -> list[dict[str, Any]]:
453 """The checks that read only the diff and the files, when the run has a task and a
454 baseline and verification or architecture rules are configured."""
455 from governed_harness.orchestration.architecture import effective_rules
456 from governed_harness.orchestration.verification_checks import (
457 current_files,
458 pure_checks,
459 )
461 checks: list[dict[str, Any]] = []
462 verification = resolved.project.verification
463 architecture = resolved.project.architecture
464 layers = effective_rules(architecture, workspace / ".harness")
465 if (
466 (verification is not None or layers is not None)
467 and task is not None
468 and (baseline_text is not None)
469 ):
470 parsed = parse_unified_diff(diff_text)
471 for item in pure_checks(
472 verification or VerificationConfig(),
473 task,
474 parsed,
475 current_files(workspace, parsed),
476 workspace,
477 baseline_text,
478 layers=layers,
479 layer_policy=architecture.policy if architecture is not None else "enforce",
480 layer_differential=bool(architecture is not None and architecture.differential),
481 ):
482 checks.append(
483 {
484 "id": item.validator_id,
485 "policy": item.policy,
486 "issues": [
487 {
488 "rule": issue.rule_id,
489 "severity": issue.severity.value,
490 "path": issue.path,
491 "line": issue.line,
492 "message": issue.message,
493 }
494 for issue in item.issues
495 ],
496 }
497 )
498 return checks
501def _definitions(resolved: ResolvedConfiguration) -> list[ValidatorDefinition]:
502 definitions = list(resolved.effective_validators)
503 verification = resolved.project.verification
504 for invariant in (verification.invariants if verification else None) or (): 504 ↛ 505line 504 didn't jump to line 505 because the loop on line 504 never started
505 definitions.append(
506 ValidatorDefinition(id=f"invariant.{invariant.invariant_id}", command=invariant.command)
507 )
508 return definitions
511def _check_state(harness_dir: Path, execution_id: str | None) -> dict[str, Any] | None:
512 directory = harness_dir / CHECK_DIRECTORY
513 if execution_id:
514 candidate = directory / f"{execution_id}.json"
515 if not candidate.is_file():
516 raise NotFoundError(f"no check state for run {execution_id}")
517 else:
518 files = sorted(directory.glob("*.json"), key=lambda item: item.stat().st_mtime)
519 if not files:
520 return None
521 candidate = files[-1]
522 value = json.loads(candidate.read_text(encoding="utf-8"))
523 return value if isinstance(value, dict) else None
526def _placeholder_task(resolved: ResolvedConfiguration) -> Task:
527 return Task.model_validate(
528 {
529 "taskId": "check",
530 "projectId": resolved.project.project_id,
531 "title": "harness check",
532 "intent": "Run the gate's validators on the workspace.",
533 "acceptanceCriteria": [{"criterionId": "check", "text": "The validators pass."}],
534 }
535 )
538def _tail(store: LocalArtifactStore, refs: tuple[str, ...], limit: int = 2000) -> str:
539 try:
540 report = json.loads(store.get(refs[0]))
541 stdout = store.get(report["stdoutRef"]).decode("utf-8", "replace")
542 stderr = store.get(report["stderrRef"]).decode("utf-8", "replace")
543 except (OSError, ValueError, KeyError, IndexError, TypeError):
544 return ""
545 text = (stdout + ("\n" + stderr if stderr.strip() else "")).strip()
546 return text[-limit:]
549__all__ = [
550 "TRACEABILITY_RULE",
551 "GateContract",
552 "check_state_path",
553 "permissions",
554 "run_check",
555 "runnable",
556 "task_terms",
557 "traceability_terms",
558]
561def gate_definitions(resolved: ResolvedConfiguration) -> list[ValidatorDefinition]:
562 """The validators of the gate (the project's and the invariants), as the check runs them."""
563 return _definitions(resolved)