Coverage for src/governed_harness/benchmark/runner.py: 100%

60 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1from __future__ import annotations 

2 

3import platform 

4import statistics 

5import sys 

6import tempfile 

7import time 

8from collections.abc import Callable 

9from datetime import UTC, datetime, timedelta 

10from pathlib import Path 

11 

12from governed_harness.capabilities import grants_from_rules 

13from governed_harness.configuration.models import CapabilityRule 

14from governed_harness.domain.enums import ActorType, PhaseId, ResultStatus 

15from governed_harness.domain.models import Actor 

16from governed_harness.events import SQLiteEventStore 

17from governed_harness.evidence import LocalArtifactStore, sha256_json 

18from governed_harness.gates import GateEngine, GateInput 

19from governed_harness.orchestration import NormativeStateMachine 

20from governed_harness.runtime import CommandSpec, SafeProcessRunner 

21 

22 

23def _measure(operation: Callable[[], object], iterations: int) -> dict[str, float]: 

24 samples: list[float] = [] 

25 for _ in range(iterations): 

26 start = time.perf_counter_ns() 

27 operation() 

28 samples.append((time.perf_counter_ns() - start) / 1_000_000) 

29 ordered = sorted(samples) 

30 p95 = ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))] 

31 total_ms = sum(samples) 

32 return { 

33 "iterations": float(iterations), 

34 "totalMs": total_ms, 

35 "meanMs": statistics.fmean(samples), 

36 "medianMs": statistics.median(samples), 

37 "p95Ms": p95, 

38 "operationsPerSecond": iterations / (total_ms / 1000) if total_ms else 0.0, 

39 } 

40 

41 

42def run_benchmarks(*, iterations: int = 1000) -> dict[str, object]: 

43 with tempfile.TemporaryDirectory(prefix="harness-bench-") as directory: 

44 root = Path(directory) 

45 event_store = SQLiteEventStore(root / "events.db") 

46 artifact_store = LocalArtifactStore(root / "artifacts") 

47 gate = GateEngine() 

48 machine = NormativeStateMachine() 

49 hash_payload = {"phase": "VERIFICATION", "values": list(range(100))} 

50 event_index = 0 

51 

52 def append_event() -> object: 

53 nonlocal event_index 

54 event_index += 1 

55 return event_store.append("run_benchmark", "benchmark.event", {"i": event_index}) 

56 

57 core_iterations = max(10, iterations) 

58 artifact_iterations = max(10, min(iterations, 5000)) 

59 event_iterations = max(10, min(iterations, 10000)) 

60 results: dict[str, object] = { 

61 "canonicalHash": _measure(lambda: sha256_json(hash_payload), core_iterations), 

62 "stateTransition": _measure( 

63 lambda: machine.advance(PhaseId.INTENT, ResultStatus.PASSED), core_iterations 

64 ), 

65 "gateEvaluation": _measure( 

66 lambda: gate.evaluate([GateInput("tests", ResultStatus.PASSED)]), core_iterations 

67 ), 

68 "artifactPutDeduplicated": _measure( 

69 lambda: artifact_store.put(b"benchmark-evidence", redact=False), artifact_iterations 

70 ), 

71 "eventAppend": _measure(append_event, event_iterations), 

72 } 

73 

74 actor = Actor(actor_type=ActorType.TOOL, actor_id="benchmark.runner") 

75 grants = grants_from_rules( 

76 "run_benchmark", 

77 actor, 

78 (CapabilityRule(capability="process.execute", scope=(sys.executable,)),), 

79 lifetime=timedelta(minutes=5), 

80 ) 

81 runner = SafeProcessRunner(root) 

82 subprocess_iterations = max(5, min(30, iterations // 50 or 5)) 

83 

84 def direct_process() -> object: 

85 import subprocess # nosec B404 - the benchmark's own baseline process 

86 

87 return subprocess.run( # nosec B603 - fixed argv: this interpreter with -c pass, no shell 

88 [sys.executable, "-c", "pass"], 

89 cwd=root, 

90 capture_output=True, 

91 check=False, 

92 ) 

93 

94 def governed_process() -> object: 

95 return runner.run( 

96 CommandSpec( 

97 argv=(sys.executable, "-c", "pass"), 

98 cwd=root, 

99 timeout_seconds=10, 

100 ), 

101 actor=actor, 

102 grants=grants, 

103 ) 

104 

105 direct = _measure(direct_process, subprocess_iterations) 

106 governed = _measure(governed_process, subprocess_iterations) 

107 results["processDirect"] = direct 

108 results["processGoverned"] = governed 

109 results["processGovernanceOverhead"] = { 

110 "absoluteMedianMs": governed["medianMs"] - direct["medianMs"], 

111 "relativeMedianPercent": ( 

112 (governed["medianMs"] / direct["medianMs"] - 1) * 100 

113 if direct["medianMs"] 

114 else None 

115 ), 

116 "note": "Local process-launch overhead only; excludes validators, agents and human wait.", 

117 } 

118 event_store.close() 

119 return { 

120 "schemaVersion": "1.0", 

121 "generatedAt": datetime.now(UTC).isoformat(), 

122 "environment": { 

123 "python": sys.version.split()[0], 

124 "platform": platform.platform(), 

125 "processor": platform.processor() or "not reported", 

126 }, 

127 "methodology": { 

128 "clock": "time.perf_counter_ns", 

129 "warmup": "No explicit warmup; every sample is reported in the aggregate.", 

130 "scope": "Microbenchmarks on this execution environment; not general performance claims.", 

131 }, 

132 "benchmarks": results, 

133 }