Coverage for src/governed_harness/benchmark/runner.py: 100%
60 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1from __future__ import annotations
3import platform
4import statistics
5import sys
6import tempfile
7import time
8from collections.abc import Callable
9from datetime import UTC, datetime, timedelta
10from pathlib import Path
12from governed_harness.capabilities import grants_from_rules
13from governed_harness.configuration.models import CapabilityRule
14from governed_harness.domain.enums import ActorType, PhaseId, ResultStatus
15from governed_harness.domain.models import Actor
16from governed_harness.events import SQLiteEventStore
17from governed_harness.evidence import LocalArtifactStore, sha256_json
18from governed_harness.gates import GateEngine, GateInput
19from governed_harness.orchestration import NormativeStateMachine
20from governed_harness.runtime import CommandSpec, SafeProcessRunner
23def _measure(operation: Callable[[], object], iterations: int) -> dict[str, float]:
24 samples: list[float] = []
25 for _ in range(iterations):
26 start = time.perf_counter_ns()
27 operation()
28 samples.append((time.perf_counter_ns() - start) / 1_000_000)
29 ordered = sorted(samples)
30 p95 = ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))]
31 total_ms = sum(samples)
32 return {
33 "iterations": float(iterations),
34 "totalMs": total_ms,
35 "meanMs": statistics.fmean(samples),
36 "medianMs": statistics.median(samples),
37 "p95Ms": p95,
38 "operationsPerSecond": iterations / (total_ms / 1000) if total_ms else 0.0,
39 }
42def run_benchmarks(*, iterations: int = 1000) -> dict[str, object]:
43 with tempfile.TemporaryDirectory(prefix="harness-bench-") as directory:
44 root = Path(directory)
45 event_store = SQLiteEventStore(root / "events.db")
46 artifact_store = LocalArtifactStore(root / "artifacts")
47 gate = GateEngine()
48 machine = NormativeStateMachine()
49 hash_payload = {"phase": "VERIFICATION", "values": list(range(100))}
50 event_index = 0
52 def append_event() -> object:
53 nonlocal event_index
54 event_index += 1
55 return event_store.append("run_benchmark", "benchmark.event", {"i": event_index})
57 core_iterations = max(10, iterations)
58 artifact_iterations = max(10, min(iterations, 5000))
59 event_iterations = max(10, min(iterations, 10000))
60 results: dict[str, object] = {
61 "canonicalHash": _measure(lambda: sha256_json(hash_payload), core_iterations),
62 "stateTransition": _measure(
63 lambda: machine.advance(PhaseId.INTENT, ResultStatus.PASSED), core_iterations
64 ),
65 "gateEvaluation": _measure(
66 lambda: gate.evaluate([GateInput("tests", ResultStatus.PASSED)]), core_iterations
67 ),
68 "artifactPutDeduplicated": _measure(
69 lambda: artifact_store.put(b"benchmark-evidence", redact=False), artifact_iterations
70 ),
71 "eventAppend": _measure(append_event, event_iterations),
72 }
74 actor = Actor(actor_type=ActorType.TOOL, actor_id="benchmark.runner")
75 grants = grants_from_rules(
76 "run_benchmark",
77 actor,
78 (CapabilityRule(capability="process.execute", scope=(sys.executable,)),),
79 lifetime=timedelta(minutes=5),
80 )
81 runner = SafeProcessRunner(root)
82 subprocess_iterations = max(5, min(30, iterations // 50 or 5))
84 def direct_process() -> object:
85 import subprocess # nosec B404 - the benchmark's own baseline process
87 return subprocess.run( # nosec B603 - fixed argv: this interpreter with -c pass, no shell
88 [sys.executable, "-c", "pass"],
89 cwd=root,
90 capture_output=True,
91 check=False,
92 )
94 def governed_process() -> object:
95 return runner.run(
96 CommandSpec(
97 argv=(sys.executable, "-c", "pass"),
98 cwd=root,
99 timeout_seconds=10,
100 ),
101 actor=actor,
102 grants=grants,
103 )
105 direct = _measure(direct_process, subprocess_iterations)
106 governed = _measure(governed_process, subprocess_iterations)
107 results["processDirect"] = direct
108 results["processGoverned"] = governed
109 results["processGovernanceOverhead"] = {
110 "absoluteMedianMs": governed["medianMs"] - direct["medianMs"],
111 "relativeMedianPercent": (
112 (governed["medianMs"] / direct["medianMs"] - 1) * 100
113 if direct["medianMs"]
114 else None
115 ),
116 "note": "Local process-launch overhead only; excludes validators, agents and human wait.",
117 }
118 event_store.close()
119 return {
120 "schemaVersion": "1.0",
121 "generatedAt": datetime.now(UTC).isoformat(),
122 "environment": {
123 "python": sys.version.split()[0],
124 "platform": platform.platform(),
125 "processor": platform.processor() or "not reported",
126 },
127 "methodology": {
128 "clock": "time.perf_counter_ns",
129 "warmup": "No explicit warmup; every sample is reported in the aggregate.",
130 "scope": "Microbenchmarks on this execution environment; not general performance claims.",
131 },
132 "benchmarks": results,
133 }