Coverage for src/governed_harness/ladder/instructions.py: 85%
185 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""``harness config lint``: contradictions between the harness configuration and the agent
2instruction files of a repository (#55, item 14).
4Agents read instruction files (``AGENTS.md``, ``CLAUDE.md``, ``.cursorrules``, ``.cursor/rules``,
5``.github/copilot-instructions.md``); the harness reads ``.harness/project.yaml``. When they
6disagree, an agent follows one and the gate enforces the other. The lint reads them all,
7deterministically (fixed patterns, no language model), and reports:
9* ``tool-version``: the same tool with different versions (``Python 3.11`` in one file,
10 ``python 3.12`` in another or in the repository's own version files);
11* ``coverage``: different coverage thresholds (a file against another or against the
12 harness's ``diffCoverage``);
13* ``tests``: one source says the tests must pass or run, another says to skip them;
14* ``forbidden-flag``: an instruction to bypass a control: ``--no-verify``, a forced push,
15 ``git add -A`` or ``git add .``, ``|| true``, ``--exit-zero``, ``continue-on-error``,
16 ``HUSKY=0`` or ``SKIP=`` for hooks.
18Each conflict names the source that wins by the declared precedence (``instructions.precedence``,
19first wins; ``harness`` is the project configuration)."""
21from __future__ import annotations
23import json
24import re
25import tomllib
26from collections.abc import Callable, Iterable, Iterator
27from dataclasses import dataclass
28from pathlib import Path
29from typing import Any
31HARNESS_SOURCE = "harness"
32MAX_FILE_BYTES = 1_000_000
34_TOOLS = {
35 "python": "python",
36 "node": "node",
37 "node.js": "node",
38 "nodejs": "node",
39 "java": "java",
40 "go": "go",
41 "golang": "go",
42 "rust": "rust",
43 "ruby": "ruby",
44 "swift": "swift",
45 "kotlin": "kotlin",
46 "typescript": "typescript",
47}
48_VERSION = re.compile(
49 r"\b(python|node(?:\.js|js)?|java|golang|go|rust|ruby|swift|kotlin|typescript)"
50 r"\s*(?:version\s*)?(?:>=|==|~=|\^|v)?\s*(\d+(?:\.\d+){0,2})\b",
51 re.IGNORECASE,
52)
53# A threshold after the word (``coverage of at least 80%``) or before it (``80% line
54# coverage``); _coverage_values scans with both as one alternation would, the first first.
55_COVERAGE_AFTER = re.compile(r"\bcoverage\b[^.\n]{0,60}?(\d{1,3}(?:\.\d+)?)\s*%", re.IGNORECASE)
56_COVERAGE_BEFORE = re.compile(
57 r"(\d{1,3}(?:\.\d+)?)\s*%\s*(?:test\s+|line\s+|code\s+)?coverage\b", re.IGNORECASE
58)
59_TESTS_REQUIRED = (
60 re.compile(r"\btests?\s+must\s+pass", re.IGNORECASE),
61 re.compile(r"\brun\s+(?:the\s+|all\s+)?tests\s+before", re.IGNORECASE),
62 re.compile(r"\ball\s+tests\s+(?:must\s+)?pass", re.IGNORECASE),
63)
64_TESTS_SKIPPED = re.compile(
65 r"\b(skip(?:ping)?\s+(?:the\s+)?tests|do\s+not\s+run\s+(?:the\s+)?tests|don't\s+run\s+(?:the\s+)?tests|"
66 r"no\s+need\s+to\s+run\s+(?:the\s+)?tests)",
67 re.IGNORECASE,
68)
69_FORBIDDEN: tuple[tuple[str, re.Pattern[str], str], ...] = (
70 ("--no-verify", re.compile(r"--no-verify\b"), "bypasses the Git hooks"),
71 (
72 "push --force",
73 re.compile(r"\bgit\s+push\b[^\n]*\s(?:--force(?!-with-lease)\b|-f\b)"),
74 "rewrites published history",
75 ),
76 (
77 "git add -A",
78 re.compile(r"\bgit\s+add\s+(?:-A\b|--all\b|\.(?:\s|$))"),
79 "stages files beyond the change",
80 ),
81 ("|| true", re.compile(r"\|\|\s*true\b"), "turns a failing command into success"),
82 ("--exit-zero", re.compile(r"--exit-zero\b"), "turns a failing check into success"),
83 ("continue-on-error", re.compile(r"\bcontinue-on-error\b"), "ignores a failing CI step"),
84 ("HUSKY=0", re.compile(r"\bHUSKY=0\b"), "disables the Git hooks"),
85 ("SKIP=", re.compile(r"\bSKIP=[A-Za-z0-9_,-]+"), "skips pre-commit hooks"),
86)
87_NEGATION = re.compile(r"\b(never|do\s+not|don't|must\s+not|avoid|forbidden|no)\b", re.IGNORECASE)
90@dataclass(frozen=True)
91class Statement:
92 source: str
93 line: int
94 text: str
95 value: str
98@dataclass(frozen=True)
99class LintIssue:
100 kind: str
101 severity: str
102 message: str
103 statements: tuple[Statement, ...]
104 winner: str | None = None
106 def as_dict(self) -> dict[str, Any]:
107 return {
108 "kind": self.kind,
109 "severity": self.severity,
110 "message": self.message,
111 "winner": self.winner,
112 "statements": [
113 {"source": item.source, "line": item.line, "value": item.value, "text": item.text}
114 for item in self.statements
115 ],
116 }
119def instruction_files(workspace: Path, configured: Iterable[str]) -> list[tuple[str, str]]:
120 """(source name, text) of each instruction file that exists; a directory (``.cursor/rules``)
121 gives one entry per file in it."""
122 found: list[tuple[str, str]] = []
123 for item in configured:
124 target = workspace / item
125 paths = sorted(p for p in target.rglob("*") if p.is_file()) if target.is_dir() else [target]
126 for path in paths:
127 if not path.is_file() or path.is_symlink() or path.stat().st_size > MAX_FILE_BYTES:
128 continue
129 name = path.relative_to(workspace).as_posix()
130 found.append((name, path.read_text(encoding="utf-8", errors="replace")))
131 return found
134def _lines(text: str) -> Iterable[tuple[int, str]]:
135 yield from enumerate(text.splitlines(), start=1)
138def _versions(source: str, text: str) -> list[tuple[str, Statement]]:
139 found: list[tuple[str, Statement]] = []
140 for number, line in _lines(text):
141 for match in _VERSION.finditer(line):
142 name = match.group(1).lower()
143 tool = _TOOLS.get(name, name)
144 found.append((tool, Statement(source, number, line.strip()[:200], match.group(2))))
145 return found
148def repository_versions(workspace: Path) -> list[tuple[str, Statement]]:
149 """Versions the repository itself pins: ``.python-version``, ``requires-python``,
150 ``.nvmrc``, ``engines.node``, the ``go`` directive of ``go.mod``."""
151 found: list[tuple[str, Statement]] = []
152 for name, reader in _VERSION_READERS:
153 path = workspace / name
154 if path.is_file():
155 found.extend(reader(path))
156 return found
159_PYTHON_VERSION_FILE = ".python-version"
160_PYPROJECT = "pyproject.toml"
161_NVMRC = ".nvmrc"
162_PACKAGE_JSON = "package.json"
163_GO_MOD = "go.mod"
164_VERSION_NUMBER = re.compile(r"(\d+(?:\.\d+){0,2})")
165_GO_DIRECTIVE = re.compile(r"^go\s+(\d+(?:\.\d+){0,2})", re.MULTILINE)
168def _read(path: Path) -> str:
169 return path.read_text(encoding="utf-8", errors="replace")
172def _python_version_file(path: Path) -> list[tuple[str, Statement]]:
173 value = _read(path).strip().split()[0:1]
174 if value and re.match(r"\d", value[0]):
175 return [("python", Statement(_PYTHON_VERSION_FILE, 1, value[0], value[0]))]
176 return []
179def _requires_python(path: Path) -> list[tuple[str, Statement]]:
180 try:
181 data = tomllib.loads(_read(path))
182 except tomllib.TOMLDecodeError:
183 data = {}
184 requires = str((data.get("project") or {}).get("requires-python") or "")
185 match = _VERSION_NUMBER.search(requires)
186 if match: 186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true
187 return [("python", Statement(_PYPROJECT, 1, f"requires-python {requires}", match[1]))]
188 return []
191def _nvmrc(path: Path) -> list[tuple[str, Statement]]:
192 match = _VERSION_NUMBER.search(_read(path))
193 if match:
194 return [("node", Statement(_NVMRC, 1, match[0], match[1]))]
195 return []
198def _engines_node(path: Path) -> list[tuple[str, Statement]]:
199 try:
200 engines = (json.loads(path.read_text(encoding="utf-8")) or {}).get("engines") or {}
201 except ValueError:
202 engines = {}
203 node = str(engines.get("node") or "") if isinstance(engines, dict) else ""
204 match = _VERSION_NUMBER.search(node)
205 if match:
206 return [("node", Statement(_PACKAGE_JSON, 1, f"engines.node {node}", match[1]))]
207 return []
210def _go_directive(path: Path) -> list[tuple[str, Statement]]:
211 match = _GO_DIRECTIVE.search(_read(path))
212 if match:
213 return [("go", Statement(_GO_MOD, 1, match[0], match[1]))]
214 return []
217_VERSION_READERS: tuple[tuple[str, Callable[[Path], list[tuple[str, Statement]]]], ...] = (
218 (_PYTHON_VERSION_FILE, _python_version_file),
219 (_PYPROJECT, _requires_python),
220 (_NVMRC, _nvmrc),
221 (_PACKAGE_JSON, _engines_node),
222 (_GO_MOD, _go_directive),
223)
224"""The repository's own version files, in the order they are read."""
227def _compatible(left: str, right: str) -> bool:
228 """``3`` and ``3.12`` agree (one is a prefix of the other); ``3.11`` and ``3.12`` do not."""
229 a, b = left.split("."), right.split(".")
230 width = min(len(a), len(b))
231 return a[:width] == b[:width]
234def _winner(sources: Iterable[str], precedence: list[str]) -> str | None:
235 ranked = [
236 (
237 precedence.index(_base(source)) if _base(source) in precedence else len(precedence),
238 source,
239 )
240 for source in sources
241 ]
242 return min(ranked)[1] if ranked else None
245def _base(source: str) -> str:
246 if source.startswith(".cursor/rules/"): 246 ↛ 247line 246 didn't jump to line 247 because the condition on line 246 was never true
247 return ".cursor/rules"
248 if source in {name for name, _reader in _VERSION_READERS}: 248 ↛ 249line 248 didn't jump to line 249 because the condition on line 248 was never true
249 return HARNESS_SOURCE
250 return source
253def _version_issues(
254 versions: list[tuple[str, Statement]], precedence: list[str]
255) -> list[LintIssue]:
256 """One issue per tool whose versions are not prefixes of each other."""
257 by_tool: dict[str, list[Statement]] = {}
258 for tool, statement in versions:
259 by_tool.setdefault(tool, []).append(statement)
260 issues: list[LintIssue] = []
261 for tool, statements in sorted(by_tool.items()):
262 values = sorted({item.value for item in statements})
263 conflict = any(
264 not _compatible(left, right) for left in values for right in values if left < right
265 )
266 if conflict: 266 ↛ 261line 266 didn't jump to line 261 because the condition on line 266 was always true
267 winner = _winner((item.source for item in statements), precedence)
268 issues.append(
269 LintIssue(
270 "tool-version",
271 "HIGH",
272 f"{tool} versions disagree: {', '.join(values)}",
273 tuple(statements),
274 winner,
275 )
276 )
277 return issues
280def _coverage_values(line: str) -> Iterator[str]:
281 """The thresholds of a line, leftmost first; at the same position the threshold after the
282 word wins, as in one alternation of both patterns."""
283 position = 0
284 while True:
285 after = _COVERAGE_AFTER.search(line, position)
286 before = _COVERAGE_BEFORE.search(line, position)
287 found = after if before is None or (after and after.start() <= before.start()) else before
288 if found is None:
289 return
290 yield found[1]
291 position = found.end()
294def _coverage_issues(
295 texts: list[tuple[str, str]], threshold: float | None, precedence: list[str]
296) -> list[LintIssue]:
297 coverage: list[Statement] = []
298 if threshold is not None:
299 coverage.append(
300 Statement(HARNESS_SOURCE, 0, "verification.testQuality.diffCoverage", f"{threshold:g}")
301 )
302 for source, text in texts:
303 for number, line in _lines(text):
304 coverage.extend(
305 Statement(source, number, line.strip()[:200], f"{float(value):g}")
306 for value in _coverage_values(line)
307 )
308 if len({item.value for item in coverage}) <= 1:
309 return []
310 return [
311 LintIssue(
312 "coverage",
313 "MEDIUM",
314 "coverage thresholds disagree: "
315 + ", ".join(sorted({item.value + "%" for item in coverage})),
316 tuple(coverage),
317 _winner((item.source for item in coverage), precedence),
318 )
319 ]
322def _test_issues(texts: list[tuple[str, str]], precedence: list[str]) -> list[LintIssue]:
323 """An issue when a file says to skip the tests (not "never skip the tests")."""
324 required: list[Statement] = []
325 skipped: list[Statement] = []
326 for source, text in texts:
327 for number, line in _lines(text):
328 skip = _TESTS_SKIPPED.search(line)
329 if skip is not None and not _NEGATION.search(line[: skip.start()]):
330 skipped.append(Statement(source, number, line.strip()[:200], "skip tests"))
331 elif skip is not None or any(item.search(line) for item in _TESTS_REQUIRED):
332 required.append(Statement(source, number, line.strip()[:200], "tests must pass"))
333 if not skipped:
334 return []
335 test_statements = (
336 Statement(HARNESS_SOURCE, 0, "mandatory test validators", "tests must pass"),
337 *required,
338 *skipped,
339 )
340 return [
341 LintIssue(
342 "tests",
343 "HIGH",
344 "an instruction file tells the agent to skip the tests the gate runs",
345 test_statements,
346 _winner((item.source for item in test_statements), precedence),
347 )
348 ]
351def _forbidden_flag_issues(texts: list[tuple[str, str]]) -> list[LintIssue]:
352 issues: list[LintIssue] = []
353 for source, text in texts:
354 for number, line in _lines(text):
355 issues.extend(
356 LintIssue(
357 "forbidden-flag",
358 "HIGH",
359 f"{source}:{number} tells the agent to use {name}, which {why}; the "
360 "harness never does",
361 (Statement(source, number, line.strip()[:200], name),),
362 HARNESS_SOURCE,
363 )
364 for name, why in _forbidden_flags(line)
365 )
366 return issues
369def _forbidden_flags(line: str) -> Iterator[tuple[str, str]]:
370 """``(flag, why)`` of each control the line tells the agent to bypass; "never use
371 --no-verify" agrees with the harness."""
372 for name, pattern, why in _FORBIDDEN:
373 flag = pattern.search(line)
374 if flag is not None and not _NEGATION.search(line[: flag.start()]):
375 yield name, why
378def lint_instructions(
379 workspace: Path,
380 files: Iterable[str],
381 precedence: list[str],
382 *,
383 coverage_threshold: float | None = None,
384) -> dict[str, Any]:
385 """The lint report: the files read, the precedence and every issue."""
386 texts = instruction_files(workspace, files)
387 versions = repository_versions(workspace)
388 for source, text in texts:
389 versions.extend(_versions(source, text))
390 issues = [
391 *_version_issues(versions, precedence),
392 *_coverage_issues(texts, coverage_threshold, precedence),
393 *_test_issues(texts, precedence),
394 *_forbidden_flag_issues(texts),
395 ]
396 return {
397 "status": "FAILED" if issues else "PASSED",
398 "files": [source for source, _ in texts],
399 "precedence": precedence,
400 "issues": [item.as_dict() for item in issues],
401 }
404__all__ = [
405 "HARNESS_SOURCE",
406 "LintIssue",
407 "Statement",
408 "instruction_files",
409 "lint_instructions",
410 "repository_versions",
411]