Coverage for src/governed_harness/benchmark/scenarios.py: 91%
108 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1from __future__ import annotations
3import os
4import platform
5import shutil
6import statistics
7import subprocess # nosec B404 - fixed git, node and fixture argv, no shell
8import sys
9import tempfile
10import time
11from collections.abc import Callable
12from datetime import UTC, datetime
13from pathlib import Path
15from governed_harness.application import HarnessApplication
16from governed_harness.domain.enums import DecisionKind, ResultStatus
17from governed_harness.runtime import resolve_executable
19# The fixture repository must not inherit the user's Git configuration: global commit signing or
20# a global core.hooksPath would make the baseline commit fail or prompt.
21_GIT_ISOLATION = ("-c", "commit.gpgsign=false", "-c", f"core.hooksPath={os.devnull}")
24def _git_init(root: Path) -> None:
25 env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"}
27 def git(*args: str) -> None:
28 subprocess.run(["git", *_GIT_ISOLATION, *args], cwd=root, check=True, env=env) # nosec B603 B607 - git from PATH, fixed fixture argv, no shell
30 git("init", "-q")
31 git("config", "user.email", "benchmark@example.invalid")
32 git("config", "user.name", "Benchmark Fixture")
33 git("add", ".")
34 git("commit", "-qm", "baseline")
37def _python_fixture(root: Path) -> tuple[Path, tuple[str, ...]]:
38 (root / "src" / "sample").mkdir(parents=True)
39 (root / "tests").mkdir()
40 (root / "src" / "sample" / "__init__.py").write_text(
41 "from .pricing import apply_discount\n", encoding="utf-8", newline="\n"
42 )
43 (root / "src" / "sample" / "pricing.py").write_text(
44 "def apply_discount(subtotal: float, threshold: float, rate: float) -> float:\n"
45 " return subtotal\n",
46 encoding="utf-8",
47 newline="\n",
48 )
49 (root / "tests" / "test_pricing.py").write_text(
50 "from sample import apply_discount\n\n"
51 "def test_below_threshold() -> None:\n"
52 " assert apply_discount(99, 100, 0.1) == 99\n",
53 encoding="utf-8",
54 newline="\n",
55 )
56 (root / "pyproject.toml").write_text(
57 "[build-system]\nrequires=[]\nbuild-backend='setuptools.build_meta'\n\n"
58 "[project]\nname='benchmark-python-fixture'\nversion='0.1.0'\n\n"
59 "[tool.pytest.ini_options]\ntestpaths=['tests']\npythonpath=['src']\n",
60 encoding="utf-8",
61 newline="\n",
62 )
63 task = root / "task.yaml"
64 task.write_text(
65 "title: Implement threshold discount\n"
66 "intent: Apply a discount at or above the threshold.\n"
67 "acceptanceCriteria:\n"
68 " - A subtotal of 100 with a ten percent rate returns 90.\n"
69 " - A subtotal below the threshold is unchanged.\n"
70 "implementation:\n"
71 " mode: patch\n"
72 " patches:\n"
73 " - path: src/sample/pricing.py\n"
74 " operation: replace\n"
75 " content: |\n"
76 " def apply_discount(subtotal: float, threshold: float, rate: float) -> float:\n"
77 " return subtotal * (1 - rate) if subtotal >= threshold else subtotal\n"
78 " - path: tests/test_pricing.py\n"
79 " operation: append\n"
80 " content: |\n"
81 "\n"
82 " def test_at_threshold() -> None:\n"
83 " assert apply_discount(100, 100, 0.1) == 90\n",
84 encoding="utf-8",
85 newline="\n",
86 )
87 _git_init(root)
88 return task, (sys.executable, "-m", "pytest", "-q")
91def _node_fixture(root: Path) -> tuple[Path, tuple[str, ...]]:
92 (root / "src").mkdir(parents=True)
93 (root / "test").mkdir()
94 (root / "src" / "resolve-config.js").write_text(
95 "export function resolveConfig(defaults, repository, task) {\n"
96 " return { ...defaults, ...task, ...repository };\n"
97 "}\n",
98 encoding="utf-8",
99 newline="\n",
100 )
101 (root / "test" / "resolve-config.test.js").write_text(
102 "import assert from 'node:assert/strict';\n"
103 "import test from 'node:test';\n"
104 "import { resolveConfig } from '../src/resolve-config.js';\n\n"
105 "test('repository overrides defaults', () => {\n"
106 " assert.deepEqual(resolveConfig({mode:'safe'}, {mode:'strict'}, {}), {mode:'strict'});\n"
107 "});\n",
108 encoding="utf-8",
109 newline="\n",
110 )
111 (root / "package.json").write_text(
112 "{\n"
113 ' "name": "benchmark-node-fixture",\n'
114 ' "version": "0.1.0",\n'
115 ' "private": true,\n'
116 ' "type": "module",\n'
117 ' "scripts": {\n'
118 ' "test": "node --test",\n'
119 ' "lint": "node --check src/resolve-config.js",\n'
120 ' "typecheck": "node --check src/resolve-config.js"\n'
121 " }\n"
122 "}\n",
123 encoding="utf-8",
124 newline="\n",
125 )
126 task = root / "task.yaml"
127 task.write_text(
128 "title: Correct configuration precedence\n"
129 "intent: Task values must override repository values, which override defaults.\n"
130 "acceptanceCriteria:\n"
131 " - Task values have the highest precedence.\n"
132 " - Existing repository precedence remains covered.\n"
133 "implementation:\n"
134 " mode: patch\n"
135 " patches:\n"
136 " - path: src/resolve-config.js\n"
137 " operation: replace\n"
138 " content: |\n"
139 " export function resolveConfig(defaults, repository, task) {\n"
140 " return { ...defaults, ...repository, ...task };\n"
141 " }\n"
142 " - path: test/resolve-config.test.js\n"
143 " operation: append\n"
144 " content: |\n"
145 "\n"
146 " test('task overrides repository', () => {\n"
147 " assert.deepEqual(resolveConfig({mode:'safe'}, {mode:'strict'}, {mode:'task'}), {mode:'task'});\n"
148 " });\n",
149 encoding="utf-8",
150 newline="\n",
151 )
152 _git_init(root)
153 return task, ("npm", "test", "--silent")
156def _apply_direct_change(stack: str, root: Path) -> None:
157 if stack == "python":
158 (root / "src" / "sample" / "pricing.py").write_text(
159 "def apply_discount(subtotal: float, threshold: float, rate: float) -> float:\n"
160 " return subtotal * (1 - rate) if subtotal >= threshold else subtotal\n",
161 encoding="utf-8",
162 newline="\n",
163 )
164 with (root / "tests" / "test_pricing.py").open(
165 "a", encoding="utf-8", newline="\n"
166 ) as handle:
167 handle.write(
168 "\n\ndef test_at_threshold() -> None:\n"
169 " assert apply_discount(100, 100, 0.1) == 90\n"
170 )
171 else:
172 (root / "src" / "resolve-config.js").write_text(
173 "export function resolveConfig(defaults, repository, task) {\n"
174 " return { ...defaults, ...repository, ...task };\n"
175 "}\n",
176 encoding="utf-8",
177 newline="\n",
178 )
179 with (root / "test" / "resolve-config.test.js").open(
180 "a", encoding="utf-8", newline="\n"
181 ) as handle:
182 handle.write(
183 "\n\ntest('task overrides repository', () => {\n"
184 " assert.deepEqual(resolveConfig({mode:'safe'}, {mode:'strict'}, {mode:'task'}), {mode:'task'});\n"
185 "});\n"
186 )
189def _summarize(samples: list[float]) -> dict[str, float]:
190 ordered = sorted(samples)
191 p95 = ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))]
192 return {
193 "iterations": float(len(samples)),
194 "meanMs": statistics.fmean(samples),
195 "medianMs": statistics.median(samples),
196 "p95Ms": p95,
197 "minMs": min(samples),
198 "maxMs": max(samples),
199 }
202def _run_direct(
203 stack: str, fixture: Callable[[Path], tuple[Path, tuple[str, ...]]]
204) -> tuple[float, dict[str, object]]:
205 with tempfile.TemporaryDirectory(prefix=f"harness-direct-{stack}-") as directory:
206 root = Path(directory)
207 _, command = fixture(root)
208 start = time.perf_counter_ns()
209 _apply_direct_change(stack, root)
210 argv = [resolve_executable(command[0]), *command[1:]]
211 result = subprocess.run(argv, cwd=root, capture_output=True, check=False) # nosec B603 - the fixture's own resolved command, no shell
212 elapsed_ms = (time.perf_counter_ns() - start) / 1_000_000
213 if result.returncode != 0: 213 ↛ 214line 213 didn't jump to line 214 because the condition on line 213 was never true
214 raise RuntimeError(
215 f"direct {stack} fixture failed: {result.stderr.decode('utf-8', 'replace')[:500]}"
216 )
217 return elapsed_ms, {"exitCode": result.returncode}
220def _run_governed(
221 stack: str, fixture: Callable[[Path], tuple[Path, tuple[str, ...]]]
222) -> tuple[float, dict[str, object]]:
223 with tempfile.TemporaryDirectory(prefix=f"harness-governed-{stack}-") as directory:
224 root = Path(directory)
225 task_file, _ = fixture(root)
226 app = HarnessApplication()
227 start = time.perf_counter_ns()
228 app.init(root)
229 task = app.create_task(root, task_file)
230 pending = app.start_run(root, task.task_id)
231 if pending.status is not ResultStatus.BLOCKED or not pending.change_set_digest: 231 ↛ 232line 231 didn't jump to line 232 because the condition on line 231 was never true
232 raise RuntimeError(
233 f"governed {stack} fixture did not reach the human gate: {pending.status}"
234 )
235 _, final = app.decide_gate(
236 root,
237 execution_id=pending.execution_id,
238 decision=DecisionKind.APPROVE,
239 change_set_digest=pending.change_set_digest,
240 actor_id="human.benchmark",
241 rationale="Programmatic benchmark approval after all deterministic checks passed",
242 )
243 elapsed_ms = (time.perf_counter_ns() - start) / 1_000_000
244 if final.status is not ResultStatus.PASSED: 244 ↛ 245line 244 didn't jump to line 245 because the condition on line 244 was never true
245 raise RuntimeError(f"governed {stack} fixture did not close: {final.status}")
246 status = app.status(root, final.execution_id)
247 return elapsed_ms, {
248 "executionId": final.execution_id,
249 "events": status["eventCount"],
250 "validations": status["validationSummary"]["total"],
251 "findings": status["findings"]["total"],
252 "eventChainValid": status["eventChainValid"],
253 }
256def run_scenario_benchmarks(*, iterations: int = 3) -> dict[str, object]:
257 if iterations < 1: 257 ↛ 258line 257 didn't jump to line 258 because the condition on line 257 was never true
258 raise ValueError("iterations must be at least 1")
259 scenarios: dict[str, object] = {}
260 for stack, fixture, available in (
261 ("python", _python_fixture, True),
262 (
263 "node",
264 _node_fixture,
265 shutil.which("node") is not None and shutil.which("npm") is not None,
266 ),
267 ):
268 if not available: 268 ↛ 269line 268 didn't jump to line 269 because the condition on line 268 was never true
269 scenarios[stack] = {"status": "NOT_AVAILABLE", "reason": "required runtime not found"}
270 continue
271 direct_samples: list[float] = []
272 governed_samples: list[float] = []
273 last_details: dict[str, object] = {}
274 for _ in range(iterations):
275 direct_ms, _ = _run_direct(stack, fixture)
276 governed_ms, last_details = _run_governed(stack, fixture)
277 direct_samples.append(direct_ms)
278 governed_samples.append(governed_ms)
279 direct = _summarize(direct_samples)
280 governed = _summarize(governed_samples)
281 overhead = governed["medianMs"] - direct["medianMs"]
282 scenarios[stack] = {
283 "status": "PASSED",
284 "directImplementationAndTests": direct,
285 "governedEndToEnd": governed,
286 "governanceOverhead": {
287 "absoluteMedianMs": overhead,
288 "relativeMedianPercent": (
289 (governed["medianMs"] / direct["medianMs"] - 1) * 100
290 if direct["medianMs"]
291 else None
292 ),
293 },
294 "lastRun": last_details,
295 }
296 return {
297 "schemaVersion": "1.0",
298 "generatedAt": datetime.now(UTC).isoformat(),
299 "environment": {
300 "python": sys.version.split()[0],
301 "node": subprocess.run( # nosec B603 B607 - node --version from PATH, no shell
302 ["node", "--version"], capture_output=True, text=True
303 ).stdout.strip()
304 if shutil.which("node")
305 else None,
306 "platform": platform.platform(),
307 },
308 "methodology": {
309 "scope": "Synthetic local fixtures; not a productivity or quality claim.",
310 "directPath": "Apply the same deterministic patch and execute the mandatory test command.",
311 "governedPath": "Initialize, persist task, execute all normative phases, validate, review, evaluate the gate, apply a programmatic benchmark approval, close, and project status.",
312 "humanWait": "Excluded: the benchmark approval is immediate and programmatic.",
313 "network": "No network access required.",
314 },
315 "scenarios": scenarios,
316 }