Coverage for src/governed_harness/application/agent_contract.py: 41%

98 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Application helpers of the agent contract (wave 11, #97): the schemas, the validation of a 

2contract document, the registry of script contracts and the evidence of what an agent call 

3received and returned.""" 

4 

5from __future__ import annotations 

6 

7import json 

8from pathlib import Path 

9from typing import TYPE_CHECKING, Any, Final 

10 

11from pydantic import BaseModel 

12 

13from governed_harness.agents.contract import ( 

14 RESULT_KEYS, 

15 AgentCallRequestContract, 

16 AgentCallResponse, 

17 request_digest, 

18 validate_request_contract, 

19 validate_response_contract, 

20) 

21from governed_harness.agents.script_contracts import ScriptContract, registry 

22from governed_harness.configuration.agent_contract import AgentContractConfig 

23from governed_harness.domain.errors import ConfigurationError, NotFoundError 

24 

25if TYPE_CHECKING: 

26 from governed_harness.orchestration.engine import EngineServices 

27 

28SCHEMAS: Final[dict[str, type[BaseModel]]] = { 

29 "request": AgentCallRequestContract, 

30 "response": AgentCallResponse, 

31 "script": ScriptContract, 

32} 

33DOCUMENT_KINDS: Final[tuple[str, ...]] = ("request", "response", "script", "output") 

34RECORD_EVENT: Final = "agent.call.recorded" 

35ITERATION_EVENT: Final = "agent.iteration" 

36 

37 

38def contract_schema(name: str) -> dict[str, Any]: 

39 """The JSON Schema of the ``request``, ``response`` or ``script`` contract.""" 

40 model = SCHEMAS.get(name) 

41 if model is None: 

42 raise ConfigurationError(f"unknown contract {name!r}; known: {', '.join(SCHEMAS)}") 

43 return model.model_json_schema(by_alias=True, ref_template="#/$defs/{model}") 

44 

45 

46def validate_document(name: str, path: Path, call_kind: str | None = None) -> dict[str, Any]: 

47 """Validate a JSON file against a contract. ``output`` is the JSON an agent prints, checked 

48 against the limits of ``call_kind``. A violation is returned as a structured result.""" 

49 if name not in DOCUMENT_KINDS: 

50 raise ConfigurationError(f"unknown document {name!r}; known: {', '.join(DOCUMENT_KINDS)}") 

51 try: 

52 value = json.loads(path.read_text(encoding="utf-8-sig")) 

53 except (OSError, ValueError) as error: 

54 raise ConfigurationError(f"cannot read {path} as JSON: {error}") from error 

55 try: 

56 if name == "output": 

57 problems = output_problems(value, call_kind or "") 

58 if problems: 

59 raise ValueError("; ".join(problems)) 

60 elif name == "request": 

61 validate_request_contract(value) 

62 elif name == "response": 

63 validate_response_contract(value) 

64 else: 

65 ScriptContract.model_validate(value) 

66 except ValueError as error: 

67 return {"document": name, "valid": False, "violation": str(error), "file": str(path)} 

68 return {"document": name, "valid": True, "violation": None, "file": str(path)} 

69 

70 

71def output_problems(value: Any, call_kind: str) -> list[str]: 

72 """What the JSON an agent printed does against the response contract of its call kind.""" 

73 from governed_harness.agents.contract import check_answer 

74 

75 if call_kind not in RESULT_KEYS: 

76 raise ConfigurationError(f"--kind must be one of: {', '.join(RESULT_KEYS)}") 

77 if not isinstance(value, dict): 

78 return ["the output must be one JSON object"] 

79 problems: list[str] = [] 

80 if value.get("status") not in {"PASSED", "FAILED", "BLOCKED"}: 

81 problems.append("status must be PASSED, FAILED or BLOCKED") 

82 if not isinstance(value.get("summary"), str): 

83 problems.append("summary must be a string") 

84 result = value.get("result") 

85 if result is not None and not isinstance(result, dict): 

86 problems.append("result must be an object") 

87 return problems 

88 return problems + check_answer(call_kind, result, AgentContractConfig()) 

89 

90 

91def script_contracts(path: Path, script_id: str | None = None) -> dict[str, Any]: 

92 """Every contract an agent may use in the project at ``path`` (``harness check``, the 

93 validator commands of the gate, the tools of the embedded MCP server), or one by id.""" 

94 from governed_harness.configuration import ConfigurationResolver 

95 from governed_harness.embedded.mcp_server import TOOLS 

96 from governed_harness.orchestration.gate_contract import gate_definitions 

97 

98 try: 

99 definitions = gate_definitions(ConfigurationResolver().resolve(path)) 

100 except ConfigurationError: 

101 definitions = [] 

102 contracts = registry(definitions, TOOLS) 

103 if script_id is not None: 

104 if script_id not in contracts: 

105 raise NotFoundError(f"no script contract {script_id!r}; known: {', '.join(contracts)}") 

106 return contracts[script_id].model_dump(mode="json", by_alias=True) 

107 return { 

108 "contractVersion": "1.0", 

109 "count": len(contracts), 

110 "scripts": [ 

111 {"id": item.script_id, "kind": item.kind, "summary": item.summary} 

112 for item in contracts.values() 

113 ], 

114 } 

115 

116 

117def agent_calls( 

118 services: EngineServices, execution_id: str, kind: str | None, iteration: int | None 

119) -> dict[str, Any]: 

120 """The agent calls of a run (``agent.call.recorded`` events), the iterations each agent 

121 used, and, for one call (``kind`` and ``iteration``), exactly what was sent and received.""" 

122 events = services.events.list(execution_id) 

123 recorded = [item.payload for item in events if item.event_type == RECORD_EVENT] 

124 begun: dict[str, int] = {} 

125 cap = 0 

126 for item in events: 

127 if item.event_type == ITERATION_EVENT: 

128 name = str(item.payload.get("callKind")) 

129 begun[name] = begun.get(name, 0) + 1 

130 cap = int(item.payload.get("max", cap)) 

131 summary = { 

132 "executionId": execution_id, 

133 "iterations": {name: {"used": count, "max": cap} for name, count in sorted(begun.items())}, 

134 "calls": [ 

135 { 

136 key: payload.get(key) 

137 for key in ( 

138 "callKind", 

139 "iteration", 

140 "provider", 

141 "size", 

142 "finalStatus", 

143 "limitReached", 

144 "sentBytes", 

145 "receivedBytes", 

146 "evidenceRef", 

147 ) 

148 } 

149 for payload in recorded 

150 if kind is None or payload.get("callKind") == kind 

151 ], 

152 } 

153 if kind is None: 

154 return summary 

155 return {**summary, "call": _call_detail(services, recorded, kind, iteration)} 

156 

157 

158def _call_detail( 

159 services: EngineServices, recorded: list[dict[str, Any]], kind: str, iteration: int | None 

160) -> dict[str, Any]: 

161 matching = [ 

162 item 

163 for item in recorded 

164 if item.get("callKind") == kind 

165 and (iteration is None or item.get("iteration") == iteration) 

166 ] 

167 if not matching: 167 ↛ 168line 167 didn't jump to line 168 because the condition on line 167 was never true

168 raise NotFoundError(f"no recorded {kind} call" + (f" {iteration}" if iteration else "")) 

169 payload = matching[-1] 

170 record = json.loads(services.artifacts.get(str(payload["evidenceRef"]))) 

171 sent = record["sent"] 

172 request_ref = sent.get("requestRef") 

173 request = json.loads(services.artifacts.get(str(request_ref))) if request_ref else None 

174 return { 

175 "record": record, 

176 "request": request, 

177 "parentReceived": record["received"], 

178 "verification": _verification(sent, request), 

179 } 

180 

181 

182def _verification(sent: dict[str, Any], request: dict[str, Any] | None) -> dict[str, Any]: 

183 """Whether the stored request is the one the provider got and carries nothing beyond what 

184 the contract lists (``None`` where the request was not stored).""" 

185 if request is None: 

186 return { 

187 "requestStored": False, 

188 "note": "The request of this kind is not stored whole; its digest is the prompt " 

189 "digest of the invocation.", 

190 } 

191 contract = request.get("agentContract") or {} 

192 keys = sorted(request) 

193 return { 

194 "requestStored": True, 

195 "digestMatches": request_digest(request) == sent.get("digest"), 

196 "keysMatchRecord": keys == sent.get("topLevelKeys"), 

197 "keysMatchContract": keys == sorted(contract.get("allowedContext") or []), 

198 "extraKeys": sorted(set(keys) - set(contract.get("allowedContext") or [])), 

199 }