Coverage for src/governed_harness/application/agent_contract.py: 41%
98 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Application helpers of the agent contract (wave 11, #97): the schemas, the validation of a
2contract document, the registry of script contracts and the evidence of what an agent call
3received and returned."""
5from __future__ import annotations
7import json
8from pathlib import Path
9from typing import TYPE_CHECKING, Any, Final
11from pydantic import BaseModel
13from governed_harness.agents.contract import (
14 RESULT_KEYS,
15 AgentCallRequestContract,
16 AgentCallResponse,
17 request_digest,
18 validate_request_contract,
19 validate_response_contract,
20)
21from governed_harness.agents.script_contracts import ScriptContract, registry
22from governed_harness.configuration.agent_contract import AgentContractConfig
23from governed_harness.domain.errors import ConfigurationError, NotFoundError
25if TYPE_CHECKING:
26 from governed_harness.orchestration.engine import EngineServices
28SCHEMAS: Final[dict[str, type[BaseModel]]] = {
29 "request": AgentCallRequestContract,
30 "response": AgentCallResponse,
31 "script": ScriptContract,
32}
33DOCUMENT_KINDS: Final[tuple[str, ...]] = ("request", "response", "script", "output")
34RECORD_EVENT: Final = "agent.call.recorded"
35ITERATION_EVENT: Final = "agent.iteration"
38def contract_schema(name: str) -> dict[str, Any]:
39 """The JSON Schema of the ``request``, ``response`` or ``script`` contract."""
40 model = SCHEMAS.get(name)
41 if model is None:
42 raise ConfigurationError(f"unknown contract {name!r}; known: {', '.join(SCHEMAS)}")
43 return model.model_json_schema(by_alias=True, ref_template="#/$defs/{model}")
46def validate_document(name: str, path: Path, call_kind: str | None = None) -> dict[str, Any]:
47 """Validate a JSON file against a contract. ``output`` is the JSON an agent prints, checked
48 against the limits of ``call_kind``. A violation is returned as a structured result."""
49 if name not in DOCUMENT_KINDS:
50 raise ConfigurationError(f"unknown document {name!r}; known: {', '.join(DOCUMENT_KINDS)}")
51 try:
52 value = json.loads(path.read_text(encoding="utf-8-sig"))
53 except (OSError, ValueError) as error:
54 raise ConfigurationError(f"cannot read {path} as JSON: {error}") from error
55 try:
56 if name == "output":
57 problems = output_problems(value, call_kind or "")
58 if problems:
59 raise ValueError("; ".join(problems))
60 elif name == "request":
61 validate_request_contract(value)
62 elif name == "response":
63 validate_response_contract(value)
64 else:
65 ScriptContract.model_validate(value)
66 except ValueError as error:
67 return {"document": name, "valid": False, "violation": str(error), "file": str(path)}
68 return {"document": name, "valid": True, "violation": None, "file": str(path)}
71def output_problems(value: Any, call_kind: str) -> list[str]:
72 """What the JSON an agent printed does against the response contract of its call kind."""
73 from governed_harness.agents.contract import check_answer
75 if call_kind not in RESULT_KEYS:
76 raise ConfigurationError(f"--kind must be one of: {', '.join(RESULT_KEYS)}")
77 if not isinstance(value, dict):
78 return ["the output must be one JSON object"]
79 problems: list[str] = []
80 if value.get("status") not in {"PASSED", "FAILED", "BLOCKED"}:
81 problems.append("status must be PASSED, FAILED or BLOCKED")
82 if not isinstance(value.get("summary"), str):
83 problems.append("summary must be a string")
84 result = value.get("result")
85 if result is not None and not isinstance(result, dict):
86 problems.append("result must be an object")
87 return problems
88 return problems + check_answer(call_kind, result, AgentContractConfig())
91def script_contracts(path: Path, script_id: str | None = None) -> dict[str, Any]:
92 """Every contract an agent may use in the project at ``path`` (``harness check``, the
93 validator commands of the gate, the tools of the embedded MCP server), or one by id."""
94 from governed_harness.configuration import ConfigurationResolver
95 from governed_harness.embedded.mcp_server import TOOLS
96 from governed_harness.orchestration.gate_contract import gate_definitions
98 try:
99 definitions = gate_definitions(ConfigurationResolver().resolve(path))
100 except ConfigurationError:
101 definitions = []
102 contracts = registry(definitions, TOOLS)
103 if script_id is not None:
104 if script_id not in contracts:
105 raise NotFoundError(f"no script contract {script_id!r}; known: {', '.join(contracts)}")
106 return contracts[script_id].model_dump(mode="json", by_alias=True)
107 return {
108 "contractVersion": "1.0",
109 "count": len(contracts),
110 "scripts": [
111 {"id": item.script_id, "kind": item.kind, "summary": item.summary}
112 for item in contracts.values()
113 ],
114 }
117def agent_calls(
118 services: EngineServices, execution_id: str, kind: str | None, iteration: int | None
119) -> dict[str, Any]:
120 """The agent calls of a run (``agent.call.recorded`` events), the iterations each agent
121 used, and, for one call (``kind`` and ``iteration``), exactly what was sent and received."""
122 events = services.events.list(execution_id)
123 recorded = [item.payload for item in events if item.event_type == RECORD_EVENT]
124 begun: dict[str, int] = {}
125 cap = 0
126 for item in events:
127 if item.event_type == ITERATION_EVENT:
128 name = str(item.payload.get("callKind"))
129 begun[name] = begun.get(name, 0) + 1
130 cap = int(item.payload.get("max", cap))
131 summary = {
132 "executionId": execution_id,
133 "iterations": {name: {"used": count, "max": cap} for name, count in sorted(begun.items())},
134 "calls": [
135 {
136 key: payload.get(key)
137 for key in (
138 "callKind",
139 "iteration",
140 "provider",
141 "size",
142 "finalStatus",
143 "limitReached",
144 "sentBytes",
145 "receivedBytes",
146 "evidenceRef",
147 )
148 }
149 for payload in recorded
150 if kind is None or payload.get("callKind") == kind
151 ],
152 }
153 if kind is None:
154 return summary
155 return {**summary, "call": _call_detail(services, recorded, kind, iteration)}
158def _call_detail(
159 services: EngineServices, recorded: list[dict[str, Any]], kind: str, iteration: int | None
160) -> dict[str, Any]:
161 matching = [
162 item
163 for item in recorded
164 if item.get("callKind") == kind
165 and (iteration is None or item.get("iteration") == iteration)
166 ]
167 if not matching: 167 ↛ 168line 167 didn't jump to line 168 because the condition on line 167 was never true
168 raise NotFoundError(f"no recorded {kind} call" + (f" {iteration}" if iteration else ""))
169 payload = matching[-1]
170 record = json.loads(services.artifacts.get(str(payload["evidenceRef"])))
171 sent = record["sent"]
172 request_ref = sent.get("requestRef")
173 request = json.loads(services.artifacts.get(str(request_ref))) if request_ref else None
174 return {
175 "record": record,
176 "request": request,
177 "parentReceived": record["received"],
178 "verification": _verification(sent, request),
179 }
182def _verification(sent: dict[str, Any], request: dict[str, Any] | None) -> dict[str, Any]:
183 """Whether the stored request is the one the provider got and carries nothing beyond what
184 the contract lists (``None`` where the request was not stored)."""
185 if request is None:
186 return {
187 "requestStored": False,
188 "note": "The request of this kind is not stored whole; its digest is the prompt "
189 "digest of the invocation.",
190 }
191 contract = request.get("agentContract") or {}
192 keys = sorted(request)
193 return {
194 "requestStored": True,
195 "digestMatches": request_digest(request) == sent.get("digest"),
196 "keysMatchRecord": keys == sent.get("topLevelKeys"),
197 "keysMatchContract": keys == sorted(contract.get("allowedContext") or []),
198 "extraKeys": sorted(set(keys) - set(contract.get("allowedContext") or [])),
199 }