Coverage for src/governed_harness/review/rules.py: 91%
298 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""The review rule catalog in three layers (#57).
3* **A, built-in**: language-neutral rules shipped with the harness
4 (``resources/review/rules.yaml``): tests, resilience, concurrency, pipeline security, quality
5 and architecture.
6* **B, language packs**: the cards of the Wave 6 standards packs of the project, read as rules;
7 the pack's ``review.yaml`` gives a card its domain and says which cards block.
8* **C, project**: rules the team writes for its framework and its business in
9 ``.harness/review/rules/DOMAIN.md`` (Markdown, one ``## rule-id`` section per rule).
11Rules merge by id with precedence C > B > A; a rule that ``supersedes`` others removes them. A
12rule is verified either by a model (``verified_by: ai``) or by a tool (``tool:harness:CHECK`` for
13the harness's own deterministic checks, ``tool:TOOL:RULE`` for a linter rule): a tool rule is
14never sent to a model. A rule whose input is missing (its tool is not configured, a file it
15``requires`` is absent, no reviewer covers its domain) is reported inactive with the reason."""
17from __future__ import annotations
19import re
20from collections.abc import Iterable, Mapping
21from dataclasses import dataclass
22from functools import lru_cache
23from importlib import resources
24from pathlib import Path
25from typing import Any, Literal
27import yaml
28from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
30from governed_harness.domain.errors import ConfigurationError
31from governed_harness.evidence.hashing import sha256_json
32from governed_harness.review.diff import glob_match
34Severity = Literal["blocking", "warning"]
35Layer = Literal["A", "B", "C"]
37RULES_DIRECTORY = ".harness/review/rules"
38HARNESS_TOOL = "harness"
39DEFAULT_PRIORITY = 50
40_RULE_ID = re.compile(r"^[a-z][a-z0-9-]*(?:\.[a-z0-9][a-z0-9-]*)+$")
41_DOMAIN = re.compile(r"^[a-z][a-z0-9-]{0,63}$")
42_LAYER_RANK: dict[str, int] = {"A": 0, "B": 1, "C": 2}
45class ConfirmationBudget(BaseModel):
46 """How much a reviewer may read to confirm a finding of the rule. A reviewer that cannot
47 confirm within it reports nothing: the catalog prefers false negatives."""
49 model_config = ConfigDict(extra="forbid", populate_by_name=True, frozen=True)
51 tool: str = "read"
52 max_reads: int = Field(default=2, alias="maxReads", ge=0, le=20)
55class Rule(BaseModel):
56 model_config = ConfigDict(extra="forbid", populate_by_name=True, frozen=True)
58 rule_id: str = Field(alias="id")
59 domain: str
60 title: str = ""
61 rule: str = Field(min_length=1)
62 severity: Severity = "warning"
63 when: str = ""
64 applies_to: tuple[str, ...] = Field(default=(), alias="appliesTo")
65 exceptions: tuple[str, ...] = ()
66 good: tuple[str, ...] = ()
67 bad: tuple[str, ...] = ()
68 budget: ConfirmationBudget = Field(default_factory=ConfirmationBudget)
69 supersedes: tuple[str, ...] = ()
70 priority: int = Field(default=DEFAULT_PRIORITY, ge=1, le=100)
71 """Lower is reported first among findings of the same severity."""
72 verified_by: tuple[str, ...] = Field(default=("ai",), alias="verifiedBy")
73 requires: tuple[str, ...] = ()
74 """Inputs the rule needs: ``tool:ID`` or ``file:GLOB``; without one it is inactive."""
75 layer: Layer = "A"
76 source: str = "builtin"
78 @field_validator("rule_id")
79 @classmethod
80 def _id(cls, value: str) -> str:
81 if not _RULE_ID.match(value):
82 raise ValueError(f"rule id must be dotted lower case (domain.name): {value!r}")
83 return value
85 @field_validator("domain")
86 @classmethod
87 def _domain(cls, value: str) -> str:
88 if not _DOMAIN.match(value): 88 ↛ 89line 88 didn't jump to line 89 because the condition on line 88 was never true
89 raise ValueError(f"domain must be lower case with dashes: {value!r}")
90 return value
92 @field_validator("verified_by", mode="before")
93 @classmethod
94 def _verified_by(cls, value: Any) -> Any:
95 items = [value] if isinstance(value, str) else list(value or ["ai"])
96 for item in items:
97 if item != "ai" and not ( 97 ↛ 100line 97 didn't jump to line 100 because the condition on line 97 was never true
98 isinstance(item, str) and item.startswith("tool:") and len(item.split(":")) >= 2
99 ):
100 raise ValueError(f"verified_by is ai or tool:TOOL[:RULE], got {item!r}")
101 if "ai" in items and len(items) > 1: 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true
102 raise ValueError("verified_by is either ai or a list of tool ids")
103 return tuple(items)
105 @property
106 def by_tool(self) -> bool:
107 return self.verified_by != ("ai",)
109 @property
110 def tools(self) -> tuple[tuple[str, str], ...]:
111 """``(tool, rule)`` of every ``tool:TOOL:RULE`` entry (rule empty for a whole tool)."""
112 found: list[tuple[str, str]] = []
113 for item in self.verified_by:
114 if item.startswith("tool:"):
115 _, tool, *rest = item.split(":", 2)
116 found.append((tool, rest[0] if rest else ""))
117 return tuple(found)
119 @property
120 def blocking(self) -> bool:
121 return self.severity == "blocking"
123 def applies(self, path: str) -> bool:
124 return not self.applies_to or any(glob_match(path, item) for item in self.applies_to)
127@dataclass(frozen=True)
128class InactiveRule:
129 rule: Rule
130 reason: str
132 def as_dict(self) -> dict[str, Any]:
133 return {
134 "id": self.rule.rule_id,
135 "domain": self.rule.domain,
136 "layer": self.rule.layer,
137 "reason": self.reason,
138 }
141@dataclass(frozen=True)
142class Catalog:
143 rules: tuple[Rule, ...]
144 inactive: tuple[InactiveRule, ...] = ()
146 @property
147 def digest(self) -> str:
148 return sha256_json([rule.model_dump(mode="json", by_alias=True) for rule in self.rules])
150 def get(self, rule_id: str) -> Rule | None:
151 return next((rule for rule in self.rules if rule.rule_id == rule_id), None)
153 def for_domain(self, domain: str, *, ai_only: bool = False) -> tuple[Rule, ...]:
154 return tuple(
155 rule
156 for rule in self.rules
157 if rule.domain == domain and (not ai_only or not rule.by_tool)
158 )
160 def tool_rules(self) -> tuple[Rule, ...]:
161 return tuple(rule for rule in self.rules if rule.by_tool)
163 def summary(self) -> dict[str, Any]:
164 layers: dict[str, int] = {"A": 0, "B": 0, "C": 0}
165 for rule in self.rules:
166 layers[rule.layer] += 1
167 return {
168 "digest": self.digest,
169 "active": len(self.rules),
170 "byLayer": layers,
171 "byTool": len(self.tool_rules()),
172 "inactive": [item.as_dict() for item in self.inactive],
173 }
176# ----- layer A ----------------------------------------------------------------------------------
177@lru_cache(maxsize=1)
178def builtin_rules() -> tuple[Rule, ...]:
179 target = resources.files("governed_harness.resources").joinpath("review", "rules.yaml")
180 data = yaml.safe_load(target.read_text(encoding="utf-8")) or {}
181 return tuple(
182 Rule.model_validate({**item, "layer": "A", "source": "builtin"})
183 for item in data.get("rules") or []
184 )
187# ----- layer B ----------------------------------------------------------------------------------
188@lru_cache(maxsize=64)
189def pack_review(pack: str) -> dict[str, Any]:
190 """The review metadata of a built-in pack (``standards/PACK/review.yaml``): card domains,
191 blocking cards, review signals and pipeline files."""
192 target = resources.files("governed_harness.resources").joinpath(
193 "standards", pack, "review.yaml"
194 )
195 if not target.is_file(): 195 ↛ 196line 195 didn't jump to line 196 because the condition on line 195 was never true
196 return {}
197 value = yaml.safe_load(target.read_text(encoding="utf-8"))
198 return value if isinstance(value, dict) else {}
201def pack_rules(packs: Iterable[Any]) -> list[Rule]:
202 """The cards of the project's standards packs as rules (layer B)."""
203 rules: list[Rule] = []
204 for pack in packs:
205 meta = pack_review(pack.pack_id)
206 domains: Mapping[str, str] = meta.get("domains") or {}
207 blocking = set(meta.get("blocking") or [])
208 for card in pack.cards:
209 verified = [
210 f"tool:{item}" for item in card.verified_by if item != "review" and ":" in item
211 ]
212 rules.append(
213 Rule.model_validate(
214 {
215 "id": card.card_id,
216 "domain": domains.get(card.card_id, "quality"),
217 "title": card.title,
218 "rule": card.rule,
219 "severity": "blocking" if card.card_id in blocking else "warning",
220 "appliesTo": list(card.applies_to),
221 "exceptions": list(card.exceptions),
222 "verifiedBy": verified or ["ai"],
223 "layer": "B",
224 "source": f"pack:{pack.pack_id}",
225 }
226 )
227 )
228 for item in meta.get("rules") or []: 228 ↛ 229line 228 didn't jump to line 229 because the loop on line 228 never started
229 rules.append(
230 Rule.model_validate({**item, "layer": "B", "source": f"pack:{pack.pack_id}"})
231 )
232 return rules
235# ----- layer C ----------------------------------------------------------------------------------
236_FIELD = re.compile(r"^[-*]\s+([A-Za-z_]+)\s*:(.*)$") # the value is stripped by the reader
237_LIST_FIELDS = {"exceptions", "appliesTo", "applies_to", "supersedes", "requires", "verifiedBy"}
238_ALIASES = {
239 "verified_by": "verifiedBy",
240 "applies_to": "appliesTo",
241 "max_reads": "maxReads",
242}
245def _frontmatter(text: str) -> tuple[dict[str, Any], str]:
246 if not text.startswith("---"):
247 return {}, text
248 parts = text.split("\n")
249 for index in range(1, len(parts)): 249 ↛ 255line 249 didn't jump to line 255 because the loop on line 249 didn't complete
250 if parts[index].strip() == "---":
251 head = yaml.safe_load("\n".join(parts[1:index])) or {}
252 if not isinstance(head, dict): 252 ↛ 253line 252 didn't jump to line 253 because the condition on line 252 was never true
253 raise ValueError("the frontmatter must be a mapping")
254 return head, "\n".join(parts[index + 1 :])
255 raise ValueError("the frontmatter is not closed with ---")
258def _split_list(value: str) -> list[str]:
259 value = value.strip()
260 if value.startswith("["): 260 ↛ 261line 260 didn't jump to line 261 because the condition on line 260 was never true
261 parsed = yaml.safe_load(value)
262 return [str(item) for item in parsed or []]
263 return [item.strip() for item in value.split(";") if item.strip()]
266def _budget(value: str) -> dict[str, Any]:
267 match = re.search(r"(\d+)", value)
268 tool = re.search(r"\b(read|grep|search|open)\w*", value)
269 return {
270 "maxReads": int(match.group(1)) if match else 2,
271 "tool": tool.group(1) if tool else "read",
272 }
275class _Section:
276 """One ``## rule-id: Title`` section read line by line: the leading ``- key: value``
277 fields, the rule's paragraph and the code blocks under ``Good:`` and ``Bad:``."""
279 def __init__(self, rule_id: str, title: str, domain: str) -> None:
280 self.data: dict[str, Any] = {"id": rule_id, "domain": domain, "title": title}
281 self.paragraph: list[str] = []
282 self.examples: dict[str, list[str]] = {"good": [], "bad": []}
283 self.target: str | None = None
284 self.block: list[str] | None = None
286 def feed(self, line: str) -> None:
287 stripped = line.strip()
288 if self.block is not None:
289 self._block_line(self.block, line, stripped)
290 return
291 if stripped.startswith("```"):
292 self.block = []
293 return
294 lowered = stripped.lower().rstrip(":")
295 if lowered in {"good", "bad"} and stripped.endswith(":"):
296 self.target = lowered
297 return
298 field = _FIELD.match(stripped)
299 if field and not self.paragraph:
300 self._field(field.group(1), field.group(2).strip())
301 return
302 if stripped and self.target is None:
303 self.paragraph.append(stripped)
305 def _block_line(self, block: list[str], line: str, stripped: str) -> None:
306 if not stripped.startswith("```"):
307 block.append(line)
308 return
309 if self.target is not None: 309 ↛ 311line 309 didn't jump to line 311 because the condition on line 309 was always true
310 self.examples[self.target].append("\n".join(block))
311 self.block = None
313 def _field(self, name: str, value: str) -> None:
314 key = _ALIASES.get(name, name)
315 if key in _LIST_FIELDS:
316 self.data[key] = _split_list(value)
317 elif key == "budget":
318 self.data["budget"] = _budget(value)
319 elif key == "priority":
320 self.data["priority"] = int(value)
321 else:
322 self.data[key] = value
324 def as_data(self) -> dict[str, Any]:
325 return {
326 **self.data,
327 "rule": " ".join(self.paragraph),
328 "good": self.examples["good"],
329 "bad": self.examples["bad"],
330 }
333def _heading(line: str) -> tuple[str, str | None] | None:
334 """A ``## rule-id: Title`` line as ``(rule id, title or None)``, read in one pass: what
335 ``^##\\s+([^:]+?)\\s*(?::\\s*(.*))?$`` captures on a line without line breaks. The id runs to
336 the first colon without its trailing whitespace; an id made only of whitespace is the last
337 space after ``##`` when there are at least two."""
338 rest = line[2:] if line.startswith("##") else ""
339 name = rest.lstrip()
340 spaces = len(rest) - len(name)
341 if not spaces:
342 return None
343 rule_id, colon, title = name.partition(":")
344 rule_id = rule_id.rstrip()
345 if not rule_id: 345 ↛ 346line 345 didn't jump to line 346 because the condition on line 345 was never true
346 if spaces < 2:
347 return None
348 rule_id = rest[spaces - 1]
349 return rule_id, (title.lstrip() if colon else None)
352def parse_rules_markdown(text: str, *, domain: str, source: str) -> list[Rule]:
353 """The rules of one project file: a ``## rule-id: Title`` section per rule, ``- key: value``
354 fields (severity, priority, when, exceptions, appliesTo, verifiedBy, supersedes, requires,
355 budget), the rule as the first paragraph, and ``Good:``/``Bad:`` code blocks."""
356 head, body = _frontmatter(text)
357 domain = str(head.get("domain") or domain)
358 sections: list[tuple[str, str, list[str]]] = []
359 for line in body.split("\n"):
360 heading = _heading(line)
361 if heading:
362 sections.append((heading[0], (heading[1] or "").strip(), []))
363 elif sections:
364 sections[-1][2].append(line)
365 rules: list[Rule] = []
366 for rule_id, title, lines in sections:
367 section = _Section(rule_id, title, domain)
368 for line in lines:
369 section.feed(line)
370 try:
371 rules.append(Rule.model_validate({**section.as_data(), "layer": "C", "source": source}))
372 except ValidationError as error:
373 raise ConfigurationError(
374 f"invalid review rule {rule_id} in {source}: {error}"
375 ) from error
376 return rules
379def project_rules(workspace: Path, directory: str = RULES_DIRECTORY) -> list[Rule]:
380 root = workspace / directory
381 if not root.is_dir():
382 return []
383 rules: list[Rule] = []
384 for path in sorted(root.glob("*.md")):
385 source = path.relative_to(workspace).as_posix()
386 try:
387 text = path.read_text(encoding="utf-8")
388 rules.extend(parse_rules_markdown(text, domain=path.stem, source=source))
389 except (OSError, ValueError, yaml.YAMLError) as error:
390 if isinstance(error, ConfigurationError):
391 raise
392 raise ConfigurationError(f"cannot read review rules {source}: {error}") from error
393 return rules
396# ----- merge ------------------------------------------------------------------------------------
397def merge(
398 layers: Iterable[Rule],
399 *,
400 available_tools: set[str],
401 reviewer_domains: set[str],
402 workspace: Path | None = None,
403) -> Catalog:
404 """Merge the layers by id (C > B > A), apply ``supersedes`` and set aside the rules whose
405 input is missing."""
406 by_id: dict[str, Rule] = {}
407 for rule in layers:
408 current = by_id.get(rule.rule_id)
409 if current is None or _LAYER_RANK[rule.layer] >= _LAYER_RANK[current.layer]: 409 ↛ 407line 409 didn't jump to line 407 because the condition on line 409 was always true
410 by_id[rule.rule_id] = rule
411 superseded = _superseded(by_id)
412 inactive: list[InactiveRule] = []
413 active: list[Rule] = []
414 for rule_id in sorted(by_id):
415 rule = by_id[rule_id]
416 reason = _missing_input(rule, available_tools, reviewer_domains, workspace)
417 if rule_id in superseded:
418 reason = f"superseded by {superseded[rule_id]}"
419 if reason is not None:
420 inactive.append(InactiveRule(rule, reason))
421 else:
422 active.append(rule)
423 return Catalog(tuple(active), tuple(inactive))
426def _superseded(by_id: dict[str, Rule]) -> dict[str, str]:
427 """``rule id -> the rule that supersedes it`` (the first one, in merge order)."""
428 superseded: dict[str, str] = {}
429 for rule in by_id.values():
430 for target in rule.supersedes:
431 if target in by_id and target != rule.rule_id: 431 ↛ 430line 431 didn't jump to line 430 because the condition on line 431 was always true
432 superseded.setdefault(target, rule.rule_id)
433 return superseded
436def _missing_input(
437 rule: Rule, tools: set[str], domains: set[str], workspace: Path | None
438) -> str | None:
439 for tool, _name in rule.tools:
440 if tool != HARNESS_TOOL and tool not in tools:
441 return f"input missing: the tool {tool} is not configured in the repository"
442 for item in rule.requires:
443 kind, _, value = item.partition(":")
444 if kind == "tool" and value != HARNESS_TOOL and value not in tools: 444 ↛ 445line 444 didn't jump to line 445 because the condition on line 444 was never true
445 return f"input missing: the tool {value} is not configured in the repository"
446 if kind == "file" and workspace is not None and not _file_exists(workspace, value):
447 return f"input missing: no file matches {value}"
448 if not rule.by_tool and rule.domain not in domains:
449 return f"input missing: no reviewer covers the domain {rule.domain}"
450 return None
453def _file_exists(workspace: Path, pattern: str) -> bool:
454 if not any(char in pattern for char in "*?["): 454 ↛ 456line 454 didn't jump to line 456 because the condition on line 454 was always true
455 return (workspace / pattern).exists()
456 return next(iter(workspace.glob(pattern)), None) is not None
459def configured_tools(standards: Any, workspace: Path, validator_ids: Iterable[str]) -> set[str]:
460 """The pack tools the repository runs: a profile validator covers it, or one of its
461 configuration files is present."""
462 from governed_harness.standards.selection import tool_validators
464 if standards is None: 464 ↛ 465line 464 didn't jump to line 465 because the condition on line 464 was never true
465 return set()
466 known = set(validator_ids)
467 tools = {
468 tool.tool_id
469 for pack in standards.packs
470 for tool in pack.tools
471 if any(item in known for item in tool.covered_by)
472 }
473 tools.update(
474 tool.tool_id for _pack, tool, _definition in tool_validators(standards, workspace, known)
475 )
476 return tools
479__all__ = [
480 "HARNESS_TOOL",
481 "RULES_DIRECTORY",
482 "Catalog",
483 "ConfirmationBudget",
484 "InactiveRule",
485 "Rule",
486 "builtin_rules",
487 "configured_tools",
488 "merge",
489 "pack_review",
490 "pack_rules",
491 "parse_rules_markdown",
492 "project_rules",
493]