Coverage for src/governed_harness/ladder/probes.py: 89%
152 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Evaluation of behaviour probes (#55, items 3 and 4).
3A probe runs a command once per variant and asserts on what it printed. This module holds the
4deterministic part: filling a variant into the command, reading the output as JSON or text,
5evaluating each assertion and classifying whether the probe could run at all. Running the
6process is the orchestration's job (with the harness's process runner and grants).
8Classification of one probe on one workspace state:
10* ``READY``: every variant ran to completion (no timeout, no missing or refused executable);
11 its assertions may pass or fail (an output that is not JSON fails the JSON assertions: on the
12 baseline that is the expected "fails before");
13* ``UNAVAILABLE``: some variant could not run, so the probe says nothing about the behaviour.
15An unavailable probe never passes: VERIFICATION records it ``BLOCKED``."""
17from __future__ import annotations
19import json
20import re
21from dataclasses import dataclass, field
22from typing import Any, Literal
24from governed_harness.ladder.jsonpath import select
26Readiness = Literal["READY", "UNAVAILABLE"]
27MAX_SHOWN = 300
30@dataclass(frozen=True)
31class VariantRun:
32 """What one variant of a probe produced."""
34 name: str
35 ran: bool
36 """The process started and finished on its own (not timed out, not refused, not missing)."""
37 exit_code: int | None
38 stdout: str
39 stderr: str = ""
40 problem: str | None = None
41 """Why the variant did not run (``ran`` false), or why its output could not be read."""
44@dataclass(frozen=True)
45class AssertionResult:
46 index: int
47 kind: str
48 passed: bool
49 detail: str
50 variant: str | None = None
52 def as_dict(self) -> dict[str, Any]:
53 value: dict[str, Any] = {
54 "index": self.index,
55 "kind": self.kind,
56 "passed": self.passed,
57 "detail": self.detail,
58 }
59 if self.variant is not None:
60 value["variant"] = self.variant
61 return value
64@dataclass(frozen=True)
65class ProbeEvaluation:
66 probe_id: str
67 readiness: Readiness
68 passed: bool
69 results: tuple[AssertionResult, ...] = ()
70 problems: tuple[str, ...] = ()
71 variants: tuple[str, ...] = field(default_factory=tuple)
73 def as_dict(self) -> dict[str, Any]:
74 return {
75 "probeId": self.probe_id,
76 "readiness": self.readiness,
77 "passed": self.passed,
78 "variants": list(self.variants),
79 "problems": list(self.problems),
80 "assertions": [item.as_dict() for item in self.results],
81 }
84_PLACEHOLDER = re.compile(r"\{([A-Za-z_]\w*)\}", re.ASCII)
85NOT_JSON = object()
86"""The document of a variant whose output is not JSON (``output: json``)."""
89def fill(template: str, values: dict[str, str]) -> str:
90 """``template`` with each ``{name}`` replaced by the variant's value (validated earlier)."""
91 return _PLACEHOLDER.sub(lambda match: values.get(match.group(1), match.group(0)), template)
94def _shown(value: Any) -> str:
95 text = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str)
96 return text if len(text) <= MAX_SHOWN else text[: MAX_SHOWN - 3] + "..."
99def _document(run: VariantRun, output: str) -> Any:
100 if output != "json":
101 return run.stdout
102 try:
103 return json.loads(run.stdout)
104 except json.JSONDecodeError:
105 return NOT_JSON
108def _values(document: Any, path: str | None, output: str) -> list[Any]:
109 if path is None: 109 ↛ 110line 109 didn't jump to line 110 because the condition on line 109 was never true
110 return [document]
111 if output != "json": 111 ↛ 112line 111 didn't jump to line 112 because the condition on line 111 was never true
112 return []
113 return select(document, path)
116def _ordered(values: list[Any], order: str) -> bool:
117 try:
118 pairs = list(zip(values, values[1:], strict=False))
119 if order == "ascending": 119 ↛ 121line 119 didn't jump to line 121 because the condition on line 119 was always true
120 return all(left <= right for left, right in pairs)
121 return all(left >= right for left, right in pairs)
122 except TypeError:
123 return False
126def _before(sequence: list[Any] | str, before: str, after: str) -> bool:
127 if isinstance(sequence, str): 127 ↛ 131line 127 didn't jump to line 131 because the condition on line 127 was always true
128 first = sequence.find(before)
129 second = sequence.find(after)
130 return first >= 0 and second >= 0 and first < second
131 texts = [
132 item if isinstance(item, str) else json.dumps(item, sort_keys=True) for item in sequence
133 ]
134 if before not in texts or after not in texts:
135 return False
136 return texts.index(before) < texts.index(after)
139def _single(
140 index: int, assertion: Any, run: VariantRun, document: Any, output: str
141) -> AssertionResult:
142 kind = assertion.kind
143 if document is NOT_JSON and kind != "exitCode" and (kind != "text"):
144 return AssertionResult(
145 index,
146 kind,
147 False,
148 f"the output is not JSON (exit code {run.exit_code})",
149 run.name,
150 )
151 if kind == "exitCode":
152 passed = run.exit_code == assertion.equals
153 return AssertionResult(
154 index, kind, passed, f"exit code {run.exit_code}, expected {assertion.equals}", run.name
155 )
156 if kind == "jsonPath":
157 return _json_path(index, assertion, run, _values(document, assertion.path, output))
158 if kind == "order":
159 return _order(index, assertion, run, document, output)
160 return _text(index, assertion, run)
163def _json_path(index: int, assertion: Any, run: VariantRun, values: list[Any]) -> AssertionResult:
164 """``present``, ``matches`` or ``equals`` of the values a path selects."""
165 kind = assertion.kind
166 if assertion.present is not None:
167 passed = bool(values) is assertion.present
168 state = "present" if values else "absent"
169 want = "present" if assertion.present else "absent"
170 return AssertionResult(
171 index, kind, passed, f"{assertion.path} is {state}, expected {want}", run.name
172 )
173 if assertion.matches is not None: 173 ↛ 174line 173 didn't jump to line 174 because the condition on line 173 was never true
174 pattern = re.compile(assertion.matches)
175 passed = bool(values) and all(
176 isinstance(item, str) and pattern.search(item) is not None for item in values
177 )
178 return AssertionResult(
179 index,
180 kind,
181 passed,
182 f"{assertion.path} = {_shown(values)} must match {assertion.matches!r}",
183 run.name,
184 )
185 actual: Any = values[0] if len(values) == 1 else values
186 passed = bool(values) and actual == assertion.equals
187 return AssertionResult(
188 index,
189 kind,
190 passed,
191 f"{assertion.path} = {_shown(actual) if values else 'absent'}, expected "
192 f"{_shown(assertion.equals)}",
193 run.name,
194 )
197def _order(
198 index: int, assertion: Any, run: VariantRun, document: Any, output: str
199) -> AssertionResult:
200 """The values of a path in ``order``, or one text ``before`` another."""
201 kind = assertion.kind
202 sequence: list[Any] | str
203 if assertion.path is not None:
204 sequence = _values(document, assertion.path, output)
205 else:
206 sequence = run.stdout if output != "json" else json.dumps(document, sort_keys=True)
207 if assertion.order is not None:
208 passed = (
209 isinstance(sequence, list) and len(sequence) > 0 and _ordered(sequence, assertion.order)
210 )
211 return AssertionResult(
212 index,
213 kind,
214 passed,
215 f"{assertion.path} = {_shown(sequence)} must be {assertion.order}",
216 run.name,
217 )
218 passed = _before(sequence, str(assertion.before), str(assertion.after))
219 return AssertionResult(
220 index,
221 kind,
222 passed,
223 f"{assertion.before!r} must appear before {assertion.after!r}",
224 run.name,
225 )
228def _text(index: int, assertion: Any, run: VariantRun) -> AssertionResult:
229 kind = assertion.kind
230 if assertion.contains is not None:
231 passed = assertion.contains in run.stdout
232 return AssertionResult(
233 index, kind, passed, f"output must contain {assertion.contains!r}", run.name
234 )
235 passed = re.search(str(assertion.matches), run.stdout) is not None
236 return AssertionResult(
237 index, kind, passed, f"output must match {assertion.matches!r}", run.name
238 )
241def _observed(run: VariantRun, document: Any, assertion: Any, output: str) -> str:
242 """What a ``differs`` assertion compares for one variant."""
243 if assertion.path is None: 243 ↛ 244line 243 didn't jump to line 244 because the condition on line 243 was never true
244 return run.stdout
245 if document is NOT_JSON:
246 return "not JSON"
247 return _shown(_values(document, assertion.path, output))
250def _chosen(assertion: Any, runs: list[VariantRun]) -> list[VariantRun]:
251 """The runs an assertion checks: the variants it names, or every variant."""
252 return [run for run in runs if not assertion.variants or run.name in assertion.variants]
255def _per_variant(
256 index: int, assertion: Any, runs: list[VariantRun], documents: dict[str, Any], output: str
257) -> list[AssertionResult]:
258 """An assertion other than ``differs`` on each variant it names (#74); one that names no
259 variant of the probe fails instead of checking nothing."""
260 chosen = _chosen(assertion, runs)
261 if not chosen:
262 return [
263 AssertionResult(
264 index,
265 assertion.kind,
266 False,
267 f"names no variant of the probe: {', '.join(assertion.variants)}",
268 )
269 ]
270 return [_single(index, assertion, run, documents[run.name], output) for run in chosen]
273def _differs(
274 index: int, assertion: Any, runs: list[VariantRun], documents: dict[str, Any], output: str
275) -> AssertionResult:
276 chosen = _chosen(assertion, runs)
277 observed = [_observed(run, documents[run.name], assertion, output) for run in chosen]
278 passed = len(chosen) >= 2 and len(set(observed)) > 1
279 return AssertionResult(
280 index,
281 "differs",
282 passed,
283 f"{assertion.path or 'the output'} across "
284 f"{', '.join(run.name for run in chosen)}: "
285 + ("differs" if passed else "the same in every variant"),
286 )
289def evaluate(probe: Any, runs: list[VariantRun]) -> ProbeEvaluation:
290 """Evaluate the assertions of ``probe`` (a ``ProbeDefinition``) on the runs of its variants,
291 in the order of ``probe.expanded_variants()``."""
292 names = tuple(run.name for run in runs)
293 problems = [
294 f"variant {run.name}: {run.problem or 'did not run'}" for run in runs if not run.ran
295 ]
296 documents = {run.name: _document(run, probe.output) for run in runs if run.ran}
297 if problems:
298 return ProbeEvaluation(probe.probe_id, "UNAVAILABLE", False, (), tuple(problems), names)
299 results: list[AssertionResult] = []
300 for index, assertion in enumerate(probe.assertions, start=1):
301 if assertion.kind == "differs":
302 results.append(_differs(index, assertion, runs, documents, probe.output))
303 else:
304 results.extend(_per_variant(index, assertion, runs, documents, probe.output))
305 return ProbeEvaluation(
306 probe.probe_id, "READY", all(item.passed for item in results), tuple(results), (), names
307 )
310__all__ = [
311 "AssertionResult",
312 "ProbeEvaluation",
313 "Readiness",
314 "VariantRun",
315 "evaluate",
316 "fill",
317]