Coverage for src/governed_harness/benchmark/scenarios.py: 91%

108 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1from __future__ import annotations 

2 

3import os 

4import platform 

5import shutil 

6import statistics 

7import subprocess # nosec B404 - fixed git, node and fixture argv, no shell 

8import sys 

9import tempfile 

10import time 

11from collections.abc import Callable 

12from datetime import UTC, datetime 

13from pathlib import Path 

14 

15from governed_harness.application import HarnessApplication 

16from governed_harness.domain.enums import DecisionKind, ResultStatus 

17from governed_harness.runtime import resolve_executable 

18 

19# The fixture repository must not inherit the user's Git configuration: global commit signing or 

20# a global core.hooksPath would make the baseline commit fail or prompt. 

21_GIT_ISOLATION = ("-c", "commit.gpgsign=false", "-c", f"core.hooksPath={os.devnull}") 

22 

23 

24def _git_init(root: Path) -> None: 

25 env = {**os.environ, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1"} 

26 

27 def git(*args: str) -> None: 

28 subprocess.run(["git", *_GIT_ISOLATION, *args], cwd=root, check=True, env=env) # nosec B603 B607 - git from PATH, fixed fixture argv, no shell 

29 

30 git("init", "-q") 

31 git("config", "user.email", "benchmark@example.invalid") 

32 git("config", "user.name", "Benchmark Fixture") 

33 git("add", ".") 

34 git("commit", "-qm", "baseline") 

35 

36 

37def _python_fixture(root: Path) -> tuple[Path, tuple[str, ...]]: 

38 (root / "src" / "sample").mkdir(parents=True) 

39 (root / "tests").mkdir() 

40 (root / "src" / "sample" / "__init__.py").write_text( 

41 "from .pricing import apply_discount\n", encoding="utf-8", newline="\n" 

42 ) 

43 (root / "src" / "sample" / "pricing.py").write_text( 

44 "def apply_discount(subtotal: float, threshold: float, rate: float) -> float:\n" 

45 " return subtotal\n", 

46 encoding="utf-8", 

47 newline="\n", 

48 ) 

49 (root / "tests" / "test_pricing.py").write_text( 

50 "from sample import apply_discount\n\n" 

51 "def test_below_threshold() -> None:\n" 

52 " assert apply_discount(99, 100, 0.1) == 99\n", 

53 encoding="utf-8", 

54 newline="\n", 

55 ) 

56 (root / "pyproject.toml").write_text( 

57 "[build-system]\nrequires=[]\nbuild-backend='setuptools.build_meta'\n\n" 

58 "[project]\nname='benchmark-python-fixture'\nversion='0.1.0'\n\n" 

59 "[tool.pytest.ini_options]\ntestpaths=['tests']\npythonpath=['src']\n", 

60 encoding="utf-8", 

61 newline="\n", 

62 ) 

63 task = root / "task.yaml" 

64 task.write_text( 

65 "title: Implement threshold discount\n" 

66 "intent: Apply a discount at or above the threshold.\n" 

67 "acceptanceCriteria:\n" 

68 " - A subtotal of 100 with a ten percent rate returns 90.\n" 

69 " - A subtotal below the threshold is unchanged.\n" 

70 "implementation:\n" 

71 " mode: patch\n" 

72 " patches:\n" 

73 " - path: src/sample/pricing.py\n" 

74 " operation: replace\n" 

75 " content: |\n" 

76 " def apply_discount(subtotal: float, threshold: float, rate: float) -> float:\n" 

77 " return subtotal * (1 - rate) if subtotal >= threshold else subtotal\n" 

78 " - path: tests/test_pricing.py\n" 

79 " operation: append\n" 

80 " content: |\n" 

81 "\n" 

82 " def test_at_threshold() -> None:\n" 

83 " assert apply_discount(100, 100, 0.1) == 90\n", 

84 encoding="utf-8", 

85 newline="\n", 

86 ) 

87 _git_init(root) 

88 return task, (sys.executable, "-m", "pytest", "-q") 

89 

90 

91def _node_fixture(root: Path) -> tuple[Path, tuple[str, ...]]: 

92 (root / "src").mkdir(parents=True) 

93 (root / "test").mkdir() 

94 (root / "src" / "resolve-config.js").write_text( 

95 "export function resolveConfig(defaults, repository, task) {\n" 

96 " return { ...defaults, ...task, ...repository };\n" 

97 "}\n", 

98 encoding="utf-8", 

99 newline="\n", 

100 ) 

101 (root / "test" / "resolve-config.test.js").write_text( 

102 "import assert from 'node:assert/strict';\n" 

103 "import test from 'node:test';\n" 

104 "import { resolveConfig } from '../src/resolve-config.js';\n\n" 

105 "test('repository overrides defaults', () => {\n" 

106 " assert.deepEqual(resolveConfig({mode:'safe'}, {mode:'strict'}, {}), {mode:'strict'});\n" 

107 "});\n", 

108 encoding="utf-8", 

109 newline="\n", 

110 ) 

111 (root / "package.json").write_text( 

112 "{\n" 

113 ' "name": "benchmark-node-fixture",\n' 

114 ' "version": "0.1.0",\n' 

115 ' "private": true,\n' 

116 ' "type": "module",\n' 

117 ' "scripts": {\n' 

118 ' "test": "node --test",\n' 

119 ' "lint": "node --check src/resolve-config.js",\n' 

120 ' "typecheck": "node --check src/resolve-config.js"\n' 

121 " }\n" 

122 "}\n", 

123 encoding="utf-8", 

124 newline="\n", 

125 ) 

126 task = root / "task.yaml" 

127 task.write_text( 

128 "title: Correct configuration precedence\n" 

129 "intent: Task values must override repository values, which override defaults.\n" 

130 "acceptanceCriteria:\n" 

131 " - Task values have the highest precedence.\n" 

132 " - Existing repository precedence remains covered.\n" 

133 "implementation:\n" 

134 " mode: patch\n" 

135 " patches:\n" 

136 " - path: src/resolve-config.js\n" 

137 " operation: replace\n" 

138 " content: |\n" 

139 " export function resolveConfig(defaults, repository, task) {\n" 

140 " return { ...defaults, ...repository, ...task };\n" 

141 " }\n" 

142 " - path: test/resolve-config.test.js\n" 

143 " operation: append\n" 

144 " content: |\n" 

145 "\n" 

146 " test('task overrides repository', () => {\n" 

147 " assert.deepEqual(resolveConfig({mode:'safe'}, {mode:'strict'}, {mode:'task'}), {mode:'task'});\n" 

148 " });\n", 

149 encoding="utf-8", 

150 newline="\n", 

151 ) 

152 _git_init(root) 

153 return task, ("npm", "test", "--silent") 

154 

155 

156def _apply_direct_change(stack: str, root: Path) -> None: 

157 if stack == "python": 

158 (root / "src" / "sample" / "pricing.py").write_text( 

159 "def apply_discount(subtotal: float, threshold: float, rate: float) -> float:\n" 

160 " return subtotal * (1 - rate) if subtotal >= threshold else subtotal\n", 

161 encoding="utf-8", 

162 newline="\n", 

163 ) 

164 with (root / "tests" / "test_pricing.py").open( 

165 "a", encoding="utf-8", newline="\n" 

166 ) as handle: 

167 handle.write( 

168 "\n\ndef test_at_threshold() -> None:\n" 

169 " assert apply_discount(100, 100, 0.1) == 90\n" 

170 ) 

171 else: 

172 (root / "src" / "resolve-config.js").write_text( 

173 "export function resolveConfig(defaults, repository, task) {\n" 

174 " return { ...defaults, ...repository, ...task };\n" 

175 "}\n", 

176 encoding="utf-8", 

177 newline="\n", 

178 ) 

179 with (root / "test" / "resolve-config.test.js").open( 

180 "a", encoding="utf-8", newline="\n" 

181 ) as handle: 

182 handle.write( 

183 "\n\ntest('task overrides repository', () => {\n" 

184 " assert.deepEqual(resolveConfig({mode:'safe'}, {mode:'strict'}, {mode:'task'}), {mode:'task'});\n" 

185 "});\n" 

186 ) 

187 

188 

189def _summarize(samples: list[float]) -> dict[str, float]: 

190 ordered = sorted(samples) 

191 p95 = ordered[min(len(ordered) - 1, int(len(ordered) * 0.95))] 

192 return { 

193 "iterations": float(len(samples)), 

194 "meanMs": statistics.fmean(samples), 

195 "medianMs": statistics.median(samples), 

196 "p95Ms": p95, 

197 "minMs": min(samples), 

198 "maxMs": max(samples), 

199 } 

200 

201 

202def _run_direct( 

203 stack: str, fixture: Callable[[Path], tuple[Path, tuple[str, ...]]] 

204) -> tuple[float, dict[str, object]]: 

205 with tempfile.TemporaryDirectory(prefix=f"harness-direct-{stack}-") as directory: 

206 root = Path(directory) 

207 _, command = fixture(root) 

208 start = time.perf_counter_ns() 

209 _apply_direct_change(stack, root) 

210 argv = [resolve_executable(command[0]), *command[1:]] 

211 result = subprocess.run(argv, cwd=root, capture_output=True, check=False) # nosec B603 - the fixture's own resolved command, no shell 

212 elapsed_ms = (time.perf_counter_ns() - start) / 1_000_000 

213 if result.returncode != 0: 213 ↛ 214line 213 didn't jump to line 214 because the condition on line 213 was never true

214 raise RuntimeError( 

215 f"direct {stack} fixture failed: {result.stderr.decode('utf-8', 'replace')[:500]}" 

216 ) 

217 return elapsed_ms, {"exitCode": result.returncode} 

218 

219 

220def _run_governed( 

221 stack: str, fixture: Callable[[Path], tuple[Path, tuple[str, ...]]] 

222) -> tuple[float, dict[str, object]]: 

223 with tempfile.TemporaryDirectory(prefix=f"harness-governed-{stack}-") as directory: 

224 root = Path(directory) 

225 task_file, _ = fixture(root) 

226 app = HarnessApplication() 

227 start = time.perf_counter_ns() 

228 app.init(root) 

229 task = app.create_task(root, task_file) 

230 pending = app.start_run(root, task.task_id) 

231 if pending.status is not ResultStatus.BLOCKED or not pending.change_set_digest: 231 ↛ 232line 231 didn't jump to line 232 because the condition on line 231 was never true

232 raise RuntimeError( 

233 f"governed {stack} fixture did not reach the human gate: {pending.status}" 

234 ) 

235 _, final = app.decide_gate( 

236 root, 

237 execution_id=pending.execution_id, 

238 decision=DecisionKind.APPROVE, 

239 change_set_digest=pending.change_set_digest, 

240 actor_id="human.benchmark", 

241 rationale="Programmatic benchmark approval after all deterministic checks passed", 

242 ) 

243 elapsed_ms = (time.perf_counter_ns() - start) / 1_000_000 

244 if final.status is not ResultStatus.PASSED: 244 ↛ 245line 244 didn't jump to line 245 because the condition on line 244 was never true

245 raise RuntimeError(f"governed {stack} fixture did not close: {final.status}") 

246 status = app.status(root, final.execution_id) 

247 return elapsed_ms, { 

248 "executionId": final.execution_id, 

249 "events": status["eventCount"], 

250 "validations": status["validationSummary"]["total"], 

251 "findings": status["findings"]["total"], 

252 "eventChainValid": status["eventChainValid"], 

253 } 

254 

255 

256def run_scenario_benchmarks(*, iterations: int = 3) -> dict[str, object]: 

257 if iterations < 1: 257 ↛ 258line 257 didn't jump to line 258 because the condition on line 257 was never true

258 raise ValueError("iterations must be at least 1") 

259 scenarios: dict[str, object] = {} 

260 for stack, fixture, available in ( 

261 ("python", _python_fixture, True), 

262 ( 

263 "node", 

264 _node_fixture, 

265 shutil.which("node") is not None and shutil.which("npm") is not None, 

266 ), 

267 ): 

268 if not available: 268 ↛ 269line 268 didn't jump to line 269 because the condition on line 268 was never true

269 scenarios[stack] = {"status": "NOT_AVAILABLE", "reason": "required runtime not found"} 

270 continue 

271 direct_samples: list[float] = [] 

272 governed_samples: list[float] = [] 

273 last_details: dict[str, object] = {} 

274 for _ in range(iterations): 

275 direct_ms, _ = _run_direct(stack, fixture) 

276 governed_ms, last_details = _run_governed(stack, fixture) 

277 direct_samples.append(direct_ms) 

278 governed_samples.append(governed_ms) 

279 direct = _summarize(direct_samples) 

280 governed = _summarize(governed_samples) 

281 overhead = governed["medianMs"] - direct["medianMs"] 

282 scenarios[stack] = { 

283 "status": "PASSED", 

284 "directImplementationAndTests": direct, 

285 "governedEndToEnd": governed, 

286 "governanceOverhead": { 

287 "absoluteMedianMs": overhead, 

288 "relativeMedianPercent": ( 

289 (governed["medianMs"] / direct["medianMs"] - 1) * 100 

290 if direct["medianMs"] 

291 else None 

292 ), 

293 }, 

294 "lastRun": last_details, 

295 } 

296 return { 

297 "schemaVersion": "1.0", 

298 "generatedAt": datetime.now(UTC).isoformat(), 

299 "environment": { 

300 "python": sys.version.split()[0], 

301 "node": subprocess.run( # nosec B603 B607 - node --version from PATH, no shell 

302 ["node", "--version"], capture_output=True, text=True 

303 ).stdout.strip() 

304 if shutil.which("node") 

305 else None, 

306 "platform": platform.platform(), 

307 }, 

308 "methodology": { 

309 "scope": "Synthetic local fixtures; not a productivity or quality claim.", 

310 "directPath": "Apply the same deterministic patch and execute the mandatory test command.", 

311 "governedPath": "Initialize, persist task, execute all normative phases, validate, review, evaluate the gate, apply a programmatic benchmark approval, close, and project status.", 

312 "humanWait": "Excluded: the benchmark approval is immediate and programmatic.", 

313 "network": "No network access required.", 

314 }, 

315 "scenarios": scenarios, 

316 }