Coverage for src/governed_harness/orchestration/ladder_probes.py: 87%

61 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Running behaviour probes (#55, items 3 and 4). 

2 

3A probe is run once per variant, without a shell, with the harness's process runner (so the 

4capability grants apply: a project probe's executable is granted by the configuration, a task 

5probe's executable must already be granted) and an environment of its ``passEnv`` names plus the 

6variant's own variables. The output is evaluated by ``ladder.probes``. A probe runs on the 

7workspace after the change (VERIFICATION) or on a scratch copy of the baseline (preflight in 

8PLANNING); either way an executable that is missing, refused or too slow makes the probe 

9``UNAVAILABLE``, never ``PASSED``.""" 

10 

11from __future__ import annotations 

12 

13import time 

14from dataclasses import dataclass 

15from pathlib import Path 

16from typing import TYPE_CHECKING, Any 

17 

18from governed_harness.capabilities import grants_from_rules 

19from governed_harness.domain.enums import ActorType 

20from governed_harness.domain.models import Actor, Execution, ProbeDefinition 

21from governed_harness.ladder.probes import ProbeEvaluation, VariantRun, evaluate, fill 

22from governed_harness.runtime import CancellationToken, SafeProcessRunner 

23from governed_harness.runtime.process_runner import CommandSpec 

24 

25if TYPE_CHECKING: 

26 from governed_harness.orchestration.hosts import EngineHost 

27 

28PROBE_ACTOR_PREFIX = "validator.probe." 

29_STREAM_CHARS = 2_000 

30 

31 

32@dataclass(frozen=True) 

33class ProbeRun: 

34 """A probe's evaluation and the evidence of its variants (stored output).""" 

35 

36 evaluation: ProbeEvaluation 

37 variants: tuple[dict[str, Any], ...] 

38 duration_ms: int 

39 

40 def as_dict(self) -> dict[str, Any]: 

41 return { 

42 **self.evaluation.as_dict(), 

43 "variantRuns": list(self.variants), 

44 "durationMs": self.duration_ms, 

45 } 

46 

47 

48def probe_actor(probe: ProbeDefinition) -> Actor: 

49 return Actor( 

50 actor_type=ActorType.TOOL, actor_id=f"{PROBE_ACTOR_PREFIX}{probe.probe_id}", version="1" 

51 ) 

52 

53 

54def run_probe( 

55 engine: EngineHost, 

56 execution: Execution, 

57 probe: ProbeDefinition, 

58 root: Path, 

59 runner: SafeProcessRunner, 

60) -> ProbeRun: 

61 """Run every variant of ``probe`` in ``root`` (the workspace or a scratch copy).""" 

62 services = engine.s 

63 actor = probe_actor(probe) 

64 grants = grants_from_rules( 

65 execution.execution_id, actor, services.resolved.effective_capabilities 

66 ) 

67 cancellation = CancellationToken(lambda: engine.is_cancelled(execution.execution_id)) 

68 runs: list[VariantRun] = [] 

69 records: list[dict[str, Any]] = [] 

70 started = time.perf_counter() 

71 for variant in probe.expanded_variants(): 

72 argv = tuple(fill(item, variant.values) for item in probe.command) 

73 cwd = root / fill(probe.cwd, variant.values) 

74 record: dict[str, Any] = {"variant": variant.name, "argv": list(argv)} 

75 try: 

76 result = runner.run( 

77 CommandSpec( 

78 argv=argv, 

79 cwd=cwd, 

80 timeout_seconds=float(engine._bounded_timeout(probe.timeout_seconds)), 

81 allowed_environment=(*probe.pass_env, *variant.env), 

82 max_output_bytes=services.resolved.project.runtime.max_output_bytes, 

83 ), 

84 actor=actor, 

85 grants=grants, 

86 extra_env=variant.env or None, 

87 cancellation=cancellation, 

88 ) 

89 except FileNotFoundError: 

90 problem = f"{argv[0]} not found" 

91 runs.append(VariantRun(variant.name, False, None, "", problem=problem)) 

92 records.append({**record, "ran": False, "problem": problem}) 

93 continue 

94 except Exception as error: # noqa: BLE001 # a refused or failed start is "unavailable" 

95 problem = f"{type(error).__name__}: {error}" 

96 runs.append(VariantRun(variant.name, False, None, "", problem=problem)) 

97 records.append({**record, "ran": False, "problem": problem}) 

98 continue 

99 stdout = result.stdout.decode("utf-8", "replace") 

100 stderr = result.stderr.decode("utf-8", "replace") 

101 ran = not result.timed_out and not result.cancelled 

102 stopped = None 

103 if result.timed_out: 103 ↛ 104line 103 didn't jump to line 104 because the condition on line 103 was never true

104 stopped = "timed out" 

105 elif result.cancelled: 105 ↛ 106line 105 didn't jump to line 106 because the condition on line 105 was never true

106 stopped = "cancelled" 

107 runs.append(VariantRun(variant.name, ran, result.exit_code, stdout, stderr, stopped)) 

108 stdout_ref = services.artifacts.put( 

109 result.stdout, 

110 media_type="text/plain", 

111 metadata={"stream": "stdout", "probe": probe.probe_id}, 

112 ) 

113 stderr_ref = services.artifacts.put( 

114 result.stderr, 

115 media_type="text/plain", 

116 metadata={"stream": "stderr", "probe": probe.probe_id}, 

117 ) 

118 records.append( 

119 { 

120 **record, 

121 "ran": ran, 

122 "exitCode": result.exit_code, 

123 "problem": stopped, 

124 "stdoutRef": stdout_ref.uri, 

125 "stderrRef": stderr_ref.uri, 

126 "stdoutHead": stdout[:_STREAM_CHARS], 

127 } 

128 ) 

129 duration = max(0, round((time.perf_counter() - started) * 1000)) 

130 return ProbeRun(evaluate(probe, runs), tuple(records), duration) 

131 

132 

133__all__ = ["PROBE_ACTOR_PREFIX", "ProbeRun", "probe_actor", "run_probe"]