Coverage for src/governed_harness/ladder/instructions.py: 85%

185 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""``harness config lint``: contradictions between the harness configuration and the agent 

2instruction files of a repository (#55, item 14). 

3 

4Agents read instruction files (``AGENTS.md``, ``CLAUDE.md``, ``.cursorrules``, ``.cursor/rules``, 

5``.github/copilot-instructions.md``); the harness reads ``.harness/project.yaml``. When they 

6disagree, an agent follows one and the gate enforces the other. The lint reads them all, 

7deterministically (fixed patterns, no language model), and reports: 

8 

9* ``tool-version``: the same tool with different versions (``Python 3.11`` in one file, 

10 ``python 3.12`` in another or in the repository's own version files); 

11* ``coverage``: different coverage thresholds (a file against another or against the 

12 harness's ``diffCoverage``); 

13* ``tests``: one source says the tests must pass or run, another says to skip them; 

14* ``forbidden-flag``: an instruction to bypass a control: ``--no-verify``, a forced push, 

15 ``git add -A`` or ``git add .``, ``|| true``, ``--exit-zero``, ``continue-on-error``, 

16 ``HUSKY=0`` or ``SKIP=`` for hooks. 

17 

18Each conflict names the source that wins by the declared precedence (``instructions.precedence``, 

19first wins; ``harness`` is the project configuration).""" 

20 

21from __future__ import annotations 

22 

23import json 

24import re 

25import tomllib 

26from collections.abc import Callable, Iterable, Iterator 

27from dataclasses import dataclass 

28from pathlib import Path 

29from typing import Any 

30 

31HARNESS_SOURCE = "harness" 

32MAX_FILE_BYTES = 1_000_000 

33 

34_TOOLS = { 

35 "python": "python", 

36 "node": "node", 

37 "node.js": "node", 

38 "nodejs": "node", 

39 "java": "java", 

40 "go": "go", 

41 "golang": "go", 

42 "rust": "rust", 

43 "ruby": "ruby", 

44 "swift": "swift", 

45 "kotlin": "kotlin", 

46 "typescript": "typescript", 

47} 

48_VERSION = re.compile( 

49 r"\b(python|node(?:\.js|js)?|java|golang|go|rust|ruby|swift|kotlin|typescript)" 

50 r"\s*(?:version\s*)?(?:>=|==|~=|\^|v)?\s*(\d+(?:\.\d+){0,2})\b", 

51 re.IGNORECASE, 

52) 

53# A threshold after the word (``coverage of at least 80%``) or before it (``80% line 

54# coverage``); _coverage_values scans with both as one alternation would, the first first. 

55_COVERAGE_AFTER = re.compile(r"\bcoverage\b[^.\n]{0,60}?(\d{1,3}(?:\.\d+)?)\s*%", re.IGNORECASE) 

56_COVERAGE_BEFORE = re.compile( 

57 r"(\d{1,3}(?:\.\d+)?)\s*%\s*(?:test\s+|line\s+|code\s+)?coverage\b", re.IGNORECASE 

58) 

59_TESTS_REQUIRED = ( 

60 re.compile(r"\btests?\s+must\s+pass", re.IGNORECASE), 

61 re.compile(r"\brun\s+(?:the\s+|all\s+)?tests\s+before", re.IGNORECASE), 

62 re.compile(r"\ball\s+tests\s+(?:must\s+)?pass", re.IGNORECASE), 

63) 

64_TESTS_SKIPPED = re.compile( 

65 r"\b(skip(?:ping)?\s+(?:the\s+)?tests|do\s+not\s+run\s+(?:the\s+)?tests|don't\s+run\s+(?:the\s+)?tests|" 

66 r"no\s+need\s+to\s+run\s+(?:the\s+)?tests)", 

67 re.IGNORECASE, 

68) 

69_FORBIDDEN: tuple[tuple[str, re.Pattern[str], str], ...] = ( 

70 ("--no-verify", re.compile(r"--no-verify\b"), "bypasses the Git hooks"), 

71 ( 

72 "push --force", 

73 re.compile(r"\bgit\s+push\b[^\n]*\s(?:--force(?!-with-lease)\b|-f\b)"), 

74 "rewrites published history", 

75 ), 

76 ( 

77 "git add -A", 

78 re.compile(r"\bgit\s+add\s+(?:-A\b|--all\b|\.(?:\s|$))"), 

79 "stages files beyond the change", 

80 ), 

81 ("|| true", re.compile(r"\|\|\s*true\b"), "turns a failing command into success"), 

82 ("--exit-zero", re.compile(r"--exit-zero\b"), "turns a failing check into success"), 

83 ("continue-on-error", re.compile(r"\bcontinue-on-error\b"), "ignores a failing CI step"), 

84 ("HUSKY=0", re.compile(r"\bHUSKY=0\b"), "disables the Git hooks"), 

85 ("SKIP=", re.compile(r"\bSKIP=[A-Za-z0-9_,-]+"), "skips pre-commit hooks"), 

86) 

87_NEGATION = re.compile(r"\b(never|do\s+not|don't|must\s+not|avoid|forbidden|no)\b", re.IGNORECASE) 

88 

89 

90@dataclass(frozen=True) 

91class Statement: 

92 source: str 

93 line: int 

94 text: str 

95 value: str 

96 

97 

98@dataclass(frozen=True) 

99class LintIssue: 

100 kind: str 

101 severity: str 

102 message: str 

103 statements: tuple[Statement, ...] 

104 winner: str | None = None 

105 

106 def as_dict(self) -> dict[str, Any]: 

107 return { 

108 "kind": self.kind, 

109 "severity": self.severity, 

110 "message": self.message, 

111 "winner": self.winner, 

112 "statements": [ 

113 {"source": item.source, "line": item.line, "value": item.value, "text": item.text} 

114 for item in self.statements 

115 ], 

116 } 

117 

118 

119def instruction_files(workspace: Path, configured: Iterable[str]) -> list[tuple[str, str]]: 

120 """(source name, text) of each instruction file that exists; a directory (``.cursor/rules``) 

121 gives one entry per file in it.""" 

122 found: list[tuple[str, str]] = [] 

123 for item in configured: 

124 target = workspace / item 

125 paths = sorted(p for p in target.rglob("*") if p.is_file()) if target.is_dir() else [target] 

126 for path in paths: 

127 if not path.is_file() or path.is_symlink() or path.stat().st_size > MAX_FILE_BYTES: 

128 continue 

129 name = path.relative_to(workspace).as_posix() 

130 found.append((name, path.read_text(encoding="utf-8", errors="replace"))) 

131 return found 

132 

133 

134def _lines(text: str) -> Iterable[tuple[int, str]]: 

135 yield from enumerate(text.splitlines(), start=1) 

136 

137 

138def _versions(source: str, text: str) -> list[tuple[str, Statement]]: 

139 found: list[tuple[str, Statement]] = [] 

140 for number, line in _lines(text): 

141 for match in _VERSION.finditer(line): 

142 name = match.group(1).lower() 

143 tool = _TOOLS.get(name, name) 

144 found.append((tool, Statement(source, number, line.strip()[:200], match.group(2)))) 

145 return found 

146 

147 

148def repository_versions(workspace: Path) -> list[tuple[str, Statement]]: 

149 """Versions the repository itself pins: ``.python-version``, ``requires-python``, 

150 ``.nvmrc``, ``engines.node``, the ``go`` directive of ``go.mod``.""" 

151 found: list[tuple[str, Statement]] = [] 

152 for name, reader in _VERSION_READERS: 

153 path = workspace / name 

154 if path.is_file(): 

155 found.extend(reader(path)) 

156 return found 

157 

158 

159_PYTHON_VERSION_FILE = ".python-version" 

160_PYPROJECT = "pyproject.toml" 

161_NVMRC = ".nvmrc" 

162_PACKAGE_JSON = "package.json" 

163_GO_MOD = "go.mod" 

164_VERSION_NUMBER = re.compile(r"(\d+(?:\.\d+){0,2})") 

165_GO_DIRECTIVE = re.compile(r"^go\s+(\d+(?:\.\d+){0,2})", re.MULTILINE) 

166 

167 

168def _read(path: Path) -> str: 

169 return path.read_text(encoding="utf-8", errors="replace") 

170 

171 

172def _python_version_file(path: Path) -> list[tuple[str, Statement]]: 

173 value = _read(path).strip().split()[0:1] 

174 if value and re.match(r"\d", value[0]): 

175 return [("python", Statement(_PYTHON_VERSION_FILE, 1, value[0], value[0]))] 

176 return [] 

177 

178 

179def _requires_python(path: Path) -> list[tuple[str, Statement]]: 

180 try: 

181 data = tomllib.loads(_read(path)) 

182 except tomllib.TOMLDecodeError: 

183 data = {} 

184 requires = str((data.get("project") or {}).get("requires-python") or "") 

185 match = _VERSION_NUMBER.search(requires) 

186 if match: 186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true

187 return [("python", Statement(_PYPROJECT, 1, f"requires-python {requires}", match[1]))] 

188 return [] 

189 

190 

191def _nvmrc(path: Path) -> list[tuple[str, Statement]]: 

192 match = _VERSION_NUMBER.search(_read(path)) 

193 if match: 

194 return [("node", Statement(_NVMRC, 1, match[0], match[1]))] 

195 return [] 

196 

197 

198def _engines_node(path: Path) -> list[tuple[str, Statement]]: 

199 try: 

200 engines = (json.loads(path.read_text(encoding="utf-8")) or {}).get("engines") or {} 

201 except ValueError: 

202 engines = {} 

203 node = str(engines.get("node") or "") if isinstance(engines, dict) else "" 

204 match = _VERSION_NUMBER.search(node) 

205 if match: 

206 return [("node", Statement(_PACKAGE_JSON, 1, f"engines.node {node}", match[1]))] 

207 return [] 

208 

209 

210def _go_directive(path: Path) -> list[tuple[str, Statement]]: 

211 match = _GO_DIRECTIVE.search(_read(path)) 

212 if match: 

213 return [("go", Statement(_GO_MOD, 1, match[0], match[1]))] 

214 return [] 

215 

216 

217_VERSION_READERS: tuple[tuple[str, Callable[[Path], list[tuple[str, Statement]]]], ...] = ( 

218 (_PYTHON_VERSION_FILE, _python_version_file), 

219 (_PYPROJECT, _requires_python), 

220 (_NVMRC, _nvmrc), 

221 (_PACKAGE_JSON, _engines_node), 

222 (_GO_MOD, _go_directive), 

223) 

224"""The repository's own version files, in the order they are read.""" 

225 

226 

227def _compatible(left: str, right: str) -> bool: 

228 """``3`` and ``3.12`` agree (one is a prefix of the other); ``3.11`` and ``3.12`` do not.""" 

229 a, b = left.split("."), right.split(".") 

230 width = min(len(a), len(b)) 

231 return a[:width] == b[:width] 

232 

233 

234def _winner(sources: Iterable[str], precedence: list[str]) -> str | None: 

235 ranked = [ 

236 ( 

237 precedence.index(_base(source)) if _base(source) in precedence else len(precedence), 

238 source, 

239 ) 

240 for source in sources 

241 ] 

242 return min(ranked)[1] if ranked else None 

243 

244 

245def _base(source: str) -> str: 

246 if source.startswith(".cursor/rules/"): 246 ↛ 247line 246 didn't jump to line 247 because the condition on line 246 was never true

247 return ".cursor/rules" 

248 if source in {name for name, _reader in _VERSION_READERS}: 248 ↛ 249line 248 didn't jump to line 249 because the condition on line 248 was never true

249 return HARNESS_SOURCE 

250 return source 

251 

252 

253def _version_issues( 

254 versions: list[tuple[str, Statement]], precedence: list[str] 

255) -> list[LintIssue]: 

256 """One issue per tool whose versions are not prefixes of each other.""" 

257 by_tool: dict[str, list[Statement]] = {} 

258 for tool, statement in versions: 

259 by_tool.setdefault(tool, []).append(statement) 

260 issues: list[LintIssue] = [] 

261 for tool, statements in sorted(by_tool.items()): 

262 values = sorted({item.value for item in statements}) 

263 conflict = any( 

264 not _compatible(left, right) for left in values for right in values if left < right 

265 ) 

266 if conflict: 266 ↛ 261line 266 didn't jump to line 261 because the condition on line 266 was always true

267 winner = _winner((item.source for item in statements), precedence) 

268 issues.append( 

269 LintIssue( 

270 "tool-version", 

271 "HIGH", 

272 f"{tool} versions disagree: {', '.join(values)}", 

273 tuple(statements), 

274 winner, 

275 ) 

276 ) 

277 return issues 

278 

279 

280def _coverage_values(line: str) -> Iterator[str]: 

281 """The thresholds of a line, leftmost first; at the same position the threshold after the 

282 word wins, as in one alternation of both patterns.""" 

283 position = 0 

284 while True: 

285 after = _COVERAGE_AFTER.search(line, position) 

286 before = _COVERAGE_BEFORE.search(line, position) 

287 found = after if before is None or (after and after.start() <= before.start()) else before 

288 if found is None: 

289 return 

290 yield found[1] 

291 position = found.end() 

292 

293 

294def _coverage_issues( 

295 texts: list[tuple[str, str]], threshold: float | None, precedence: list[str] 

296) -> list[LintIssue]: 

297 coverage: list[Statement] = [] 

298 if threshold is not None: 

299 coverage.append( 

300 Statement(HARNESS_SOURCE, 0, "verification.testQuality.diffCoverage", f"{threshold:g}") 

301 ) 

302 for source, text in texts: 

303 for number, line in _lines(text): 

304 coverage.extend( 

305 Statement(source, number, line.strip()[:200], f"{float(value):g}") 

306 for value in _coverage_values(line) 

307 ) 

308 if len({item.value for item in coverage}) <= 1: 

309 return [] 

310 return [ 

311 LintIssue( 

312 "coverage", 

313 "MEDIUM", 

314 "coverage thresholds disagree: " 

315 + ", ".join(sorted({item.value + "%" for item in coverage})), 

316 tuple(coverage), 

317 _winner((item.source for item in coverage), precedence), 

318 ) 

319 ] 

320 

321 

322def _test_issues(texts: list[tuple[str, str]], precedence: list[str]) -> list[LintIssue]: 

323 """An issue when a file says to skip the tests (not "never skip the tests").""" 

324 required: list[Statement] = [] 

325 skipped: list[Statement] = [] 

326 for source, text in texts: 

327 for number, line in _lines(text): 

328 skip = _TESTS_SKIPPED.search(line) 

329 if skip is not None and not _NEGATION.search(line[: skip.start()]): 

330 skipped.append(Statement(source, number, line.strip()[:200], "skip tests")) 

331 elif skip is not None or any(item.search(line) for item in _TESTS_REQUIRED): 

332 required.append(Statement(source, number, line.strip()[:200], "tests must pass")) 

333 if not skipped: 

334 return [] 

335 test_statements = ( 

336 Statement(HARNESS_SOURCE, 0, "mandatory test validators", "tests must pass"), 

337 *required, 

338 *skipped, 

339 ) 

340 return [ 

341 LintIssue( 

342 "tests", 

343 "HIGH", 

344 "an instruction file tells the agent to skip the tests the gate runs", 

345 test_statements, 

346 _winner((item.source for item in test_statements), precedence), 

347 ) 

348 ] 

349 

350 

351def _forbidden_flag_issues(texts: list[tuple[str, str]]) -> list[LintIssue]: 

352 issues: list[LintIssue] = [] 

353 for source, text in texts: 

354 for number, line in _lines(text): 

355 issues.extend( 

356 LintIssue( 

357 "forbidden-flag", 

358 "HIGH", 

359 f"{source}:{number} tells the agent to use {name}, which {why}; the " 

360 "harness never does", 

361 (Statement(source, number, line.strip()[:200], name),), 

362 HARNESS_SOURCE, 

363 ) 

364 for name, why in _forbidden_flags(line) 

365 ) 

366 return issues 

367 

368 

369def _forbidden_flags(line: str) -> Iterator[tuple[str, str]]: 

370 """``(flag, why)`` of each control the line tells the agent to bypass; "never use 

371 --no-verify" agrees with the harness.""" 

372 for name, pattern, why in _FORBIDDEN: 

373 flag = pattern.search(line) 

374 if flag is not None and not _NEGATION.search(line[: flag.start()]): 

375 yield name, why 

376 

377 

378def lint_instructions( 

379 workspace: Path, 

380 files: Iterable[str], 

381 precedence: list[str], 

382 *, 

383 coverage_threshold: float | None = None, 

384) -> dict[str, Any]: 

385 """The lint report: the files read, the precedence and every issue.""" 

386 texts = instruction_files(workspace, files) 

387 versions = repository_versions(workspace) 

388 for source, text in texts: 

389 versions.extend(_versions(source, text)) 

390 issues = [ 

391 *_version_issues(versions, precedence), 

392 *_coverage_issues(texts, coverage_threshold, precedence), 

393 *_test_issues(texts, precedence), 

394 *_forbidden_flag_issues(texts), 

395 ] 

396 return { 

397 "status": "FAILED" if issues else "PASSED", 

398 "files": [source for source, _ in texts], 

399 "precedence": precedence, 

400 "issues": [item.as_dict() for item in issues], 

401 } 

402 

403 

404__all__ = [ 

405 "HARNESS_SOURCE", 

406 "LintIssue", 

407 "Statement", 

408 "instruction_files", 

409 "lint_instructions", 

410 "repository_versions", 

411]