Coverage for src/governed_harness/orchestration/ladder_probes.py: 87%
61 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Running behaviour probes (#55, items 3 and 4).
3A probe is run once per variant, without a shell, with the harness's process runner (so the
4capability grants apply: a project probe's executable is granted by the configuration, a task
5probe's executable must already be granted) and an environment of its ``passEnv`` names plus the
6variant's own variables. The output is evaluated by ``ladder.probes``. A probe runs on the
7workspace after the change (VERIFICATION) or on a scratch copy of the baseline (preflight in
8PLANNING); either way an executable that is missing, refused or too slow makes the probe
9``UNAVAILABLE``, never ``PASSED``."""
11from __future__ import annotations
13import time
14from dataclasses import dataclass
15from pathlib import Path
16from typing import TYPE_CHECKING, Any
18from governed_harness.capabilities import grants_from_rules
19from governed_harness.domain.enums import ActorType
20from governed_harness.domain.models import Actor, Execution, ProbeDefinition
21from governed_harness.ladder.probes import ProbeEvaluation, VariantRun, evaluate, fill
22from governed_harness.runtime import CancellationToken, SafeProcessRunner
23from governed_harness.runtime.process_runner import CommandSpec
25if TYPE_CHECKING:
26 from governed_harness.orchestration.hosts import EngineHost
28PROBE_ACTOR_PREFIX = "validator.probe."
29_STREAM_CHARS = 2_000
32@dataclass(frozen=True)
33class ProbeRun:
34 """A probe's evaluation and the evidence of its variants (stored output)."""
36 evaluation: ProbeEvaluation
37 variants: tuple[dict[str, Any], ...]
38 duration_ms: int
40 def as_dict(self) -> dict[str, Any]:
41 return {
42 **self.evaluation.as_dict(),
43 "variantRuns": list(self.variants),
44 "durationMs": self.duration_ms,
45 }
48def probe_actor(probe: ProbeDefinition) -> Actor:
49 return Actor(
50 actor_type=ActorType.TOOL, actor_id=f"{PROBE_ACTOR_PREFIX}{probe.probe_id}", version="1"
51 )
54def run_probe(
55 engine: EngineHost,
56 execution: Execution,
57 probe: ProbeDefinition,
58 root: Path,
59 runner: SafeProcessRunner,
60) -> ProbeRun:
61 """Run every variant of ``probe`` in ``root`` (the workspace or a scratch copy)."""
62 services = engine.s
63 actor = probe_actor(probe)
64 grants = grants_from_rules(
65 execution.execution_id, actor, services.resolved.effective_capabilities
66 )
67 cancellation = CancellationToken(lambda: engine.is_cancelled(execution.execution_id))
68 runs: list[VariantRun] = []
69 records: list[dict[str, Any]] = []
70 started = time.perf_counter()
71 for variant in probe.expanded_variants():
72 argv = tuple(fill(item, variant.values) for item in probe.command)
73 cwd = root / fill(probe.cwd, variant.values)
74 record: dict[str, Any] = {"variant": variant.name, "argv": list(argv)}
75 try:
76 result = runner.run(
77 CommandSpec(
78 argv=argv,
79 cwd=cwd,
80 timeout_seconds=float(engine._bounded_timeout(probe.timeout_seconds)),
81 allowed_environment=(*probe.pass_env, *variant.env),
82 max_output_bytes=services.resolved.project.runtime.max_output_bytes,
83 ),
84 actor=actor,
85 grants=grants,
86 extra_env=variant.env or None,
87 cancellation=cancellation,
88 )
89 except FileNotFoundError:
90 problem = f"{argv[0]} not found"
91 runs.append(VariantRun(variant.name, False, None, "", problem=problem))
92 records.append({**record, "ran": False, "problem": problem})
93 continue
94 except Exception as error: # noqa: BLE001 # a refused or failed start is "unavailable"
95 problem = f"{type(error).__name__}: {error}"
96 runs.append(VariantRun(variant.name, False, None, "", problem=problem))
97 records.append({**record, "ran": False, "problem": problem})
98 continue
99 stdout = result.stdout.decode("utf-8", "replace")
100 stderr = result.stderr.decode("utf-8", "replace")
101 ran = not result.timed_out and not result.cancelled
102 stopped = None
103 if result.timed_out: 103 ↛ 104line 103 didn't jump to line 104 because the condition on line 103 was never true
104 stopped = "timed out"
105 elif result.cancelled: 105 ↛ 106line 105 didn't jump to line 106 because the condition on line 105 was never true
106 stopped = "cancelled"
107 runs.append(VariantRun(variant.name, ran, result.exit_code, stdout, stderr, stopped))
108 stdout_ref = services.artifacts.put(
109 result.stdout,
110 media_type="text/plain",
111 metadata={"stream": "stdout", "probe": probe.probe_id},
112 )
113 stderr_ref = services.artifacts.put(
114 result.stderr,
115 media_type="text/plain",
116 metadata={"stream": "stderr", "probe": probe.probe_id},
117 )
118 records.append(
119 {
120 **record,
121 "ran": ran,
122 "exitCode": result.exit_code,
123 "problem": stopped,
124 "stdoutRef": stdout_ref.uri,
125 "stderrRef": stderr_ref.uri,
126 "stdoutHead": stdout[:_STREAM_CHARS],
127 }
128 )
129 duration = max(0, round((time.perf_counter() - started) * 1000))
130 return ProbeRun(evaluate(probe, runs), tuple(records), duration)
133__all__ = ["PROBE_ACTOR_PREFIX", "ProbeRun", "probe_actor", "run_probe"]