Coverage for src/governed_harness/ladder/probes.py: 89%

152 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Evaluation of behaviour probes (#55, items 3 and 4). 

2 

3A probe runs a command once per variant and asserts on what it printed. This module holds the 

4deterministic part: filling a variant into the command, reading the output as JSON or text, 

5evaluating each assertion and classifying whether the probe could run at all. Running the 

6process is the orchestration's job (with the harness's process runner and grants). 

7 

8Classification of one probe on one workspace state: 

9 

10* ``READY``: every variant ran to completion (no timeout, no missing or refused executable); 

11 its assertions may pass or fail (an output that is not JSON fails the JSON assertions: on the 

12 baseline that is the expected "fails before"); 

13* ``UNAVAILABLE``: some variant could not run, so the probe says nothing about the behaviour. 

14 

15An unavailable probe never passes: VERIFICATION records it ``BLOCKED``.""" 

16 

17from __future__ import annotations 

18 

19import json 

20import re 

21from dataclasses import dataclass, field 

22from typing import Any, Literal 

23 

24from governed_harness.ladder.jsonpath import select 

25 

26Readiness = Literal["READY", "UNAVAILABLE"] 

27MAX_SHOWN = 300 

28 

29 

30@dataclass(frozen=True) 

31class VariantRun: 

32 """What one variant of a probe produced.""" 

33 

34 name: str 

35 ran: bool 

36 """The process started and finished on its own (not timed out, not refused, not missing).""" 

37 exit_code: int | None 

38 stdout: str 

39 stderr: str = "" 

40 problem: str | None = None 

41 """Why the variant did not run (``ran`` false), or why its output could not be read.""" 

42 

43 

44@dataclass(frozen=True) 

45class AssertionResult: 

46 index: int 

47 kind: str 

48 passed: bool 

49 detail: str 

50 variant: str | None = None 

51 

52 def as_dict(self) -> dict[str, Any]: 

53 value: dict[str, Any] = { 

54 "index": self.index, 

55 "kind": self.kind, 

56 "passed": self.passed, 

57 "detail": self.detail, 

58 } 

59 if self.variant is not None: 

60 value["variant"] = self.variant 

61 return value 

62 

63 

64@dataclass(frozen=True) 

65class ProbeEvaluation: 

66 probe_id: str 

67 readiness: Readiness 

68 passed: bool 

69 results: tuple[AssertionResult, ...] = () 

70 problems: tuple[str, ...] = () 

71 variants: tuple[str, ...] = field(default_factory=tuple) 

72 

73 def as_dict(self) -> dict[str, Any]: 

74 return { 

75 "probeId": self.probe_id, 

76 "readiness": self.readiness, 

77 "passed": self.passed, 

78 "variants": list(self.variants), 

79 "problems": list(self.problems), 

80 "assertions": [item.as_dict() for item in self.results], 

81 } 

82 

83 

84_PLACEHOLDER = re.compile(r"\{([A-Za-z_]\w*)\}", re.ASCII) 

85NOT_JSON = object() 

86"""The document of a variant whose output is not JSON (``output: json``).""" 

87 

88 

89def fill(template: str, values: dict[str, str]) -> str: 

90 """``template`` with each ``{name}`` replaced by the variant's value (validated earlier).""" 

91 return _PLACEHOLDER.sub(lambda match: values.get(match.group(1), match.group(0)), template) 

92 

93 

94def _shown(value: Any) -> str: 

95 text = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str) 

96 return text if len(text) <= MAX_SHOWN else text[: MAX_SHOWN - 3] + "..." 

97 

98 

99def _document(run: VariantRun, output: str) -> Any: 

100 if output != "json": 

101 return run.stdout 

102 try: 

103 return json.loads(run.stdout) 

104 except json.JSONDecodeError: 

105 return NOT_JSON 

106 

107 

108def _values(document: Any, path: str | None, output: str) -> list[Any]: 

109 if path is None: 109 ↛ 110line 109 didn't jump to line 110 because the condition on line 109 was never true

110 return [document] 

111 if output != "json": 111 ↛ 112line 111 didn't jump to line 112 because the condition on line 111 was never true

112 return [] 

113 return select(document, path) 

114 

115 

116def _ordered(values: list[Any], order: str) -> bool: 

117 try: 

118 pairs = list(zip(values, values[1:], strict=False)) 

119 if order == "ascending": 119 ↛ 121line 119 didn't jump to line 121 because the condition on line 119 was always true

120 return all(left <= right for left, right in pairs) 

121 return all(left >= right for left, right in pairs) 

122 except TypeError: 

123 return False 

124 

125 

126def _before(sequence: list[Any] | str, before: str, after: str) -> bool: 

127 if isinstance(sequence, str): 127 ↛ 131line 127 didn't jump to line 131 because the condition on line 127 was always true

128 first = sequence.find(before) 

129 second = sequence.find(after) 

130 return first >= 0 and second >= 0 and first < second 

131 texts = [ 

132 item if isinstance(item, str) else json.dumps(item, sort_keys=True) for item in sequence 

133 ] 

134 if before not in texts or after not in texts: 

135 return False 

136 return texts.index(before) < texts.index(after) 

137 

138 

139def _single( 

140 index: int, assertion: Any, run: VariantRun, document: Any, output: str 

141) -> AssertionResult: 

142 kind = assertion.kind 

143 if document is NOT_JSON and kind != "exitCode" and (kind != "text"): 

144 return AssertionResult( 

145 index, 

146 kind, 

147 False, 

148 f"the output is not JSON (exit code {run.exit_code})", 

149 run.name, 

150 ) 

151 if kind == "exitCode": 

152 passed = run.exit_code == assertion.equals 

153 return AssertionResult( 

154 index, kind, passed, f"exit code {run.exit_code}, expected {assertion.equals}", run.name 

155 ) 

156 if kind == "jsonPath": 

157 return _json_path(index, assertion, run, _values(document, assertion.path, output)) 

158 if kind == "order": 

159 return _order(index, assertion, run, document, output) 

160 return _text(index, assertion, run) 

161 

162 

163def _json_path(index: int, assertion: Any, run: VariantRun, values: list[Any]) -> AssertionResult: 

164 """``present``, ``matches`` or ``equals`` of the values a path selects.""" 

165 kind = assertion.kind 

166 if assertion.present is not None: 

167 passed = bool(values) is assertion.present 

168 state = "present" if values else "absent" 

169 want = "present" if assertion.present else "absent" 

170 return AssertionResult( 

171 index, kind, passed, f"{assertion.path} is {state}, expected {want}", run.name 

172 ) 

173 if assertion.matches is not None: 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true

174 pattern = re.compile(assertion.matches) 

175 passed = bool(values) and all( 

176 isinstance(item, str) and pattern.search(item) is not None for item in values 

177 ) 

178 return AssertionResult( 

179 index, 

180 kind, 

181 passed, 

182 f"{assertion.path} = {_shown(values)} must match {assertion.matches!r}", 

183 run.name, 

184 ) 

185 actual: Any = values[0] if len(values) == 1 else values 

186 passed = bool(values) and actual == assertion.equals 

187 return AssertionResult( 

188 index, 

189 kind, 

190 passed, 

191 f"{assertion.path} = {_shown(actual) if values else 'absent'}, expected " 

192 f"{_shown(assertion.equals)}", 

193 run.name, 

194 ) 

195 

196 

197def _order( 

198 index: int, assertion: Any, run: VariantRun, document: Any, output: str 

199) -> AssertionResult: 

200 """The values of a path in ``order``, or one text ``before`` another.""" 

201 kind = assertion.kind 

202 sequence: list[Any] | str 

203 if assertion.path is not None: 

204 sequence = _values(document, assertion.path, output) 

205 else: 

206 sequence = run.stdout if output != "json" else json.dumps(document, sort_keys=True) 

207 if assertion.order is not None: 

208 passed = ( 

209 isinstance(sequence, list) and len(sequence) > 0 and _ordered(sequence, assertion.order) 

210 ) 

211 return AssertionResult( 

212 index, 

213 kind, 

214 passed, 

215 f"{assertion.path} = {_shown(sequence)} must be {assertion.order}", 

216 run.name, 

217 ) 

218 passed = _before(sequence, str(assertion.before), str(assertion.after)) 

219 return AssertionResult( 

220 index, 

221 kind, 

222 passed, 

223 f"{assertion.before!r} must appear before {assertion.after!r}", 

224 run.name, 

225 ) 

226 

227 

228def _text(index: int, assertion: Any, run: VariantRun) -> AssertionResult: 

229 kind = assertion.kind 

230 if assertion.contains is not None: 

231 passed = assertion.contains in run.stdout 

232 return AssertionResult( 

233 index, kind, passed, f"output must contain {assertion.contains!r}", run.name 

234 ) 

235 passed = re.search(str(assertion.matches), run.stdout) is not None 

236 return AssertionResult( 

237 index, kind, passed, f"output must match {assertion.matches!r}", run.name 

238 ) 

239 

240 

241def _observed(run: VariantRun, document: Any, assertion: Any, output: str) -> str: 

242 """What a ``differs`` assertion compares for one variant.""" 

243 if assertion.path is None: 243 ↛ 244line 243 didn't jump to line 244 because the condition on line 243 was never true

244 return run.stdout 

245 if document is NOT_JSON: 

246 return "not JSON" 

247 return _shown(_values(document, assertion.path, output)) 

248 

249 

250def _chosen(assertion: Any, runs: list[VariantRun]) -> list[VariantRun]: 

251 """The runs an assertion checks: the variants it names, or every variant.""" 

252 return [run for run in runs if not assertion.variants or run.name in assertion.variants] 

253 

254 

255def _per_variant( 

256 index: int, assertion: Any, runs: list[VariantRun], documents: dict[str, Any], output: str 

257) -> list[AssertionResult]: 

258 """An assertion other than ``differs`` on each variant it names (#74); one that names no 

259 variant of the probe fails instead of checking nothing.""" 

260 chosen = _chosen(assertion, runs) 

261 if not chosen: 

262 return [ 

263 AssertionResult( 

264 index, 

265 assertion.kind, 

266 False, 

267 f"names no variant of the probe: {', '.join(assertion.variants)}", 

268 ) 

269 ] 

270 return [_single(index, assertion, run, documents[run.name], output) for run in chosen] 

271 

272 

273def _differs( 

274 index: int, assertion: Any, runs: list[VariantRun], documents: dict[str, Any], output: str 

275) -> AssertionResult: 

276 chosen = _chosen(assertion, runs) 

277 observed = [_observed(run, documents[run.name], assertion, output) for run in chosen] 

278 passed = len(chosen) >= 2 and len(set(observed)) > 1 

279 return AssertionResult( 

280 index, 

281 "differs", 

282 passed, 

283 f"{assertion.path or 'the output'} across " 

284 f"{', '.join(run.name for run in chosen)}: " 

285 + ("differs" if passed else "the same in every variant"), 

286 ) 

287 

288 

289def evaluate(probe: Any, runs: list[VariantRun]) -> ProbeEvaluation: 

290 """Evaluate the assertions of ``probe`` (a ``ProbeDefinition``) on the runs of its variants, 

291 in the order of ``probe.expanded_variants()``.""" 

292 names = tuple(run.name for run in runs) 

293 problems = [ 

294 f"variant {run.name}: {run.problem or 'did not run'}" for run in runs if not run.ran 

295 ] 

296 documents = {run.name: _document(run, probe.output) for run in runs if run.ran} 

297 if problems: 

298 return ProbeEvaluation(probe.probe_id, "UNAVAILABLE", False, (), tuple(problems), names) 

299 results: list[AssertionResult] = [] 

300 for index, assertion in enumerate(probe.assertions, start=1): 

301 if assertion.kind == "differs": 

302 results.append(_differs(index, assertion, runs, documents, probe.output)) 

303 else: 

304 results.extend(_per_variant(index, assertion, runs, documents, probe.output)) 

305 return ProbeEvaluation( 

306 probe.probe_id, "READY", all(item.passed for item in results), tuple(results), (), names 

307 ) 

308 

309 

310__all__ = [ 

311 "AssertionResult", 

312 "ProbeEvaluation", 

313 "Readiness", 

314 "VariantRun", 

315 "evaluate", 

316 "fill", 

317]