Coverage for src/governed_harness/review/rules.py: 91%

298 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""The review rule catalog in three layers (#57). 

2 

3* **A, built-in**: language-neutral rules shipped with the harness 

4 (``resources/review/rules.yaml``): tests, resilience, concurrency, pipeline security, quality 

5 and architecture. 

6* **B, language packs**: the cards of the Wave 6 standards packs of the project, read as rules; 

7 the pack's ``review.yaml`` gives a card its domain and says which cards block. 

8* **C, project**: rules the team writes for its framework and its business in 

9 ``.harness/review/rules/DOMAIN.md`` (Markdown, one ``## rule-id`` section per rule). 

10 

11Rules merge by id with precedence C > B > A; a rule that ``supersedes`` others removes them. A 

12rule is verified either by a model (``verified_by: ai``) or by a tool (``tool:harness:CHECK`` for 

13the harness's own deterministic checks, ``tool:TOOL:RULE`` for a linter rule): a tool rule is 

14never sent to a model. A rule whose input is missing (its tool is not configured, a file it 

15``requires`` is absent, no reviewer covers its domain) is reported inactive with the reason.""" 

16 

17from __future__ import annotations 

18 

19import re 

20from collections.abc import Iterable, Mapping 

21from dataclasses import dataclass 

22from functools import lru_cache 

23from importlib import resources 

24from pathlib import Path 

25from typing import Any, Literal 

26 

27import yaml 

28from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator 

29 

30from governed_harness.domain.errors import ConfigurationError 

31from governed_harness.evidence.hashing import sha256_json 

32from governed_harness.review.diff import glob_match 

33 

34Severity = Literal["blocking", "warning"] 

35Layer = Literal["A", "B", "C"] 

36 

37RULES_DIRECTORY = ".harness/review/rules" 

38HARNESS_TOOL = "harness" 

39DEFAULT_PRIORITY = 50 

40_RULE_ID = re.compile(r"^[a-z][a-z0-9-]*(?:\.[a-z0-9][a-z0-9-]*)+$") 

41_DOMAIN = re.compile(r"^[a-z][a-z0-9-]{0,63}$") 

42_LAYER_RANK: dict[str, int] = {"A": 0, "B": 1, "C": 2} 

43 

44 

45class ConfirmationBudget(BaseModel): 

46 """How much a reviewer may read to confirm a finding of the rule. A reviewer that cannot 

47 confirm within it reports nothing: the catalog prefers false negatives.""" 

48 

49 model_config = ConfigDict(extra="forbid", populate_by_name=True, frozen=True) 

50 

51 tool: str = "read" 

52 max_reads: int = Field(default=2, alias="maxReads", ge=0, le=20) 

53 

54 

55class Rule(BaseModel): 

56 model_config = ConfigDict(extra="forbid", populate_by_name=True, frozen=True) 

57 

58 rule_id: str = Field(alias="id") 

59 domain: str 

60 title: str = "" 

61 rule: str = Field(min_length=1) 

62 severity: Severity = "warning" 

63 when: str = "" 

64 applies_to: tuple[str, ...] = Field(default=(), alias="appliesTo") 

65 exceptions: tuple[str, ...] = () 

66 good: tuple[str, ...] = () 

67 bad: tuple[str, ...] = () 

68 budget: ConfirmationBudget = Field(default_factory=ConfirmationBudget) 

69 supersedes: tuple[str, ...] = () 

70 priority: int = Field(default=DEFAULT_PRIORITY, ge=1, le=100) 

71 """Lower is reported first among findings of the same severity.""" 

72 verified_by: tuple[str, ...] = Field(default=("ai",), alias="verifiedBy") 

73 requires: tuple[str, ...] = () 

74 """Inputs the rule needs: ``tool:ID`` or ``file:GLOB``; without one it is inactive.""" 

75 layer: Layer = "A" 

76 source: str = "builtin" 

77 

78 @field_validator("rule_id") 

79 @classmethod 

80 def _id(cls, value: str) -> str: 

81 if not _RULE_ID.match(value): 

82 raise ValueError(f"rule id must be dotted lower case (domain.name): {value!r}") 

83 return value 

84 

85 @field_validator("domain") 

86 @classmethod 

87 def _domain(cls, value: str) -> str: 

88 if not _DOMAIN.match(value): 88 ↛ 89line 88 didn't jump to line 89 because the condition on line 88 was never true

89 raise ValueError(f"domain must be lower case with dashes: {value!r}") 

90 return value 

91 

92 @field_validator("verified_by", mode="before") 

93 @classmethod 

94 def _verified_by(cls, value: Any) -> Any: 

95 items = [value] if isinstance(value, str) else list(value or ["ai"]) 

96 for item in items: 

97 if item != "ai" and not ( 97 ↛ 100line 97 didn't jump to line 100 because the condition on line 97 was never true

98 isinstance(item, str) and item.startswith("tool:") and len(item.split(":")) >= 2 

99 ): 

100 raise ValueError(f"verified_by is ai or tool:TOOL[:RULE], got {item!r}") 

101 if "ai" in items and len(items) > 1: 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true

102 raise ValueError("verified_by is either ai or a list of tool ids") 

103 return tuple(items) 

104 

105 @property 

106 def by_tool(self) -> bool: 

107 return self.verified_by != ("ai",) 

108 

109 @property 

110 def tools(self) -> tuple[tuple[str, str], ...]: 

111 """``(tool, rule)`` of every ``tool:TOOL:RULE`` entry (rule empty for a whole tool).""" 

112 found: list[tuple[str, str]] = [] 

113 for item in self.verified_by: 

114 if item.startswith("tool:"): 

115 _, tool, *rest = item.split(":", 2) 

116 found.append((tool, rest[0] if rest else "")) 

117 return tuple(found) 

118 

119 @property 

120 def blocking(self) -> bool: 

121 return self.severity == "blocking" 

122 

123 def applies(self, path: str) -> bool: 

124 return not self.applies_to or any(glob_match(path, item) for item in self.applies_to) 

125 

126 

127@dataclass(frozen=True) 

128class InactiveRule: 

129 rule: Rule 

130 reason: str 

131 

132 def as_dict(self) -> dict[str, Any]: 

133 return { 

134 "id": self.rule.rule_id, 

135 "domain": self.rule.domain, 

136 "layer": self.rule.layer, 

137 "reason": self.reason, 

138 } 

139 

140 

141@dataclass(frozen=True) 

142class Catalog: 

143 rules: tuple[Rule, ...] 

144 inactive: tuple[InactiveRule, ...] = () 

145 

146 @property 

147 def digest(self) -> str: 

148 return sha256_json([rule.model_dump(mode="json", by_alias=True) for rule in self.rules]) 

149 

150 def get(self, rule_id: str) -> Rule | None: 

151 return next((rule for rule in self.rules if rule.rule_id == rule_id), None) 

152 

153 def for_domain(self, domain: str, *, ai_only: bool = False) -> tuple[Rule, ...]: 

154 return tuple( 

155 rule 

156 for rule in self.rules 

157 if rule.domain == domain and (not ai_only or not rule.by_tool) 

158 ) 

159 

160 def tool_rules(self) -> tuple[Rule, ...]: 

161 return tuple(rule for rule in self.rules if rule.by_tool) 

162 

163 def summary(self) -> dict[str, Any]: 

164 layers: dict[str, int] = {"A": 0, "B": 0, "C": 0} 

165 for rule in self.rules: 

166 layers[rule.layer] += 1 

167 return { 

168 "digest": self.digest, 

169 "active": len(self.rules), 

170 "byLayer": layers, 

171 "byTool": len(self.tool_rules()), 

172 "inactive": [item.as_dict() for item in self.inactive], 

173 } 

174 

175 

176# ----- layer A ---------------------------------------------------------------------------------- 

177@lru_cache(maxsize=1) 

178def builtin_rules() -> tuple[Rule, ...]: 

179 target = resources.files("governed_harness.resources").joinpath("review", "rules.yaml") 

180 data = yaml.safe_load(target.read_text(encoding="utf-8")) or {} 

181 return tuple( 

182 Rule.model_validate({**item, "layer": "A", "source": "builtin"}) 

183 for item in data.get("rules") or [] 

184 ) 

185 

186 

187# ----- layer B ---------------------------------------------------------------------------------- 

188@lru_cache(maxsize=64) 

189def pack_review(pack: str) -> dict[str, Any]: 

190 """The review metadata of a built-in pack (``standards/PACK/review.yaml``): card domains, 

191 blocking cards, review signals and pipeline files.""" 

192 target = resources.files("governed_harness.resources").joinpath( 

193 "standards", pack, "review.yaml" 

194 ) 

195 if not target.is_file(): 195 ↛ 196line 195 didn't jump to line 196 because the condition on line 195 was never true

196 return {} 

197 value = yaml.safe_load(target.read_text(encoding="utf-8")) 

198 return value if isinstance(value, dict) else {} 

199 

200 

201def pack_rules(packs: Iterable[Any]) -> list[Rule]: 

202 """The cards of the project's standards packs as rules (layer B).""" 

203 rules: list[Rule] = [] 

204 for pack in packs: 

205 meta = pack_review(pack.pack_id) 

206 domains: Mapping[str, str] = meta.get("domains") or {} 

207 blocking = set(meta.get("blocking") or []) 

208 for card in pack.cards: 

209 verified = [ 

210 f"tool:{item}" for item in card.verified_by if item != "review" and ":" in item 

211 ] 

212 rules.append( 

213 Rule.model_validate( 

214 { 

215 "id": card.card_id, 

216 "domain": domains.get(card.card_id, "quality"), 

217 "title": card.title, 

218 "rule": card.rule, 

219 "severity": "blocking" if card.card_id in blocking else "warning", 

220 "appliesTo": list(card.applies_to), 

221 "exceptions": list(card.exceptions), 

222 "verifiedBy": verified or ["ai"], 

223 "layer": "B", 

224 "source": f"pack:{pack.pack_id}", 

225 } 

226 ) 

227 ) 

228 for item in meta.get("rules") or []: 228 ↛ 229line 228 didn't jump to line 229 because the loop on line 228 never started

229 rules.append( 

230 Rule.model_validate({**item, "layer": "B", "source": f"pack:{pack.pack_id}"}) 

231 ) 

232 return rules 

233 

234 

235# ----- layer C ---------------------------------------------------------------------------------- 

236_FIELD = re.compile(r"^[-*]\s+([A-Za-z_]+)\s*:(.*)$") # the value is stripped by the reader 

237_LIST_FIELDS = {"exceptions", "appliesTo", "applies_to", "supersedes", "requires", "verifiedBy"} 

238_ALIASES = { 

239 "verified_by": "verifiedBy", 

240 "applies_to": "appliesTo", 

241 "max_reads": "maxReads", 

242} 

243 

244 

245def _frontmatter(text: str) -> tuple[dict[str, Any], str]: 

246 if not text.startswith("---"): 

247 return {}, text 

248 parts = text.split("\n") 

249 for index in range(1, len(parts)): 249 ↛ 255line 249 didn't jump to line 255 because the loop on line 249 didn't complete

250 if parts[index].strip() == "---": 

251 head = yaml.safe_load("\n".join(parts[1:index])) or {} 

252 if not isinstance(head, dict): 252 ↛ 253line 252 didn't jump to line 253 because the condition on line 252 was never true

253 raise ValueError("the frontmatter must be a mapping") 

254 return head, "\n".join(parts[index + 1 :]) 

255 raise ValueError("the frontmatter is not closed with ---") 

256 

257 

258def _split_list(value: str) -> list[str]: 

259 value = value.strip() 

260 if value.startswith("["): 260 ↛ 261line 260 didn't jump to line 261 because the condition on line 260 was never true

261 parsed = yaml.safe_load(value) 

262 return [str(item) for item in parsed or []] 

263 return [item.strip() for item in value.split(";") if item.strip()] 

264 

265 

266def _budget(value: str) -> dict[str, Any]: 

267 match = re.search(r"(\d+)", value) 

268 tool = re.search(r"\b(read|grep|search|open)\w*", value) 

269 return { 

270 "maxReads": int(match.group(1)) if match else 2, 

271 "tool": tool.group(1) if tool else "read", 

272 } 

273 

274 

275class _Section: 

276 """One ``## rule-id: Title`` section read line by line: the leading ``- key: value`` 

277 fields, the rule's paragraph and the code blocks under ``Good:`` and ``Bad:``.""" 

278 

279 def __init__(self, rule_id: str, title: str, domain: str) -> None: 

280 self.data: dict[str, Any] = {"id": rule_id, "domain": domain, "title": title} 

281 self.paragraph: list[str] = [] 

282 self.examples: dict[str, list[str]] = {"good": [], "bad": []} 

283 self.target: str | None = None 

284 self.block: list[str] | None = None 

285 

286 def feed(self, line: str) -> None: 

287 stripped = line.strip() 

288 if self.block is not None: 

289 self._block_line(self.block, line, stripped) 

290 return 

291 if stripped.startswith("```"): 

292 self.block = [] 

293 return 

294 lowered = stripped.lower().rstrip(":") 

295 if lowered in {"good", "bad"} and stripped.endswith(":"): 

296 self.target = lowered 

297 return 

298 field = _FIELD.match(stripped) 

299 if field and not self.paragraph: 

300 self._field(field.group(1), field.group(2).strip()) 

301 return 

302 if stripped and self.target is None: 

303 self.paragraph.append(stripped) 

304 

305 def _block_line(self, block: list[str], line: str, stripped: str) -> None: 

306 if not stripped.startswith("```"): 

307 block.append(line) 

308 return 

309 if self.target is not None: 309 ↛ 311line 309 didn't jump to line 311 because the condition on line 309 was always true

310 self.examples[self.target].append("\n".join(block)) 

311 self.block = None 

312 

313 def _field(self, name: str, value: str) -> None: 

314 key = _ALIASES.get(name, name) 

315 if key in _LIST_FIELDS: 

316 self.data[key] = _split_list(value) 

317 elif key == "budget": 

318 self.data["budget"] = _budget(value) 

319 elif key == "priority": 

320 self.data["priority"] = int(value) 

321 else: 

322 self.data[key] = value 

323 

324 def as_data(self) -> dict[str, Any]: 

325 return { 

326 **self.data, 

327 "rule": " ".join(self.paragraph), 

328 "good": self.examples["good"], 

329 "bad": self.examples["bad"], 

330 } 

331 

332 

333def _heading(line: str) -> tuple[str, str | None] | None: 

334 """A ``## rule-id: Title`` line as ``(rule id, title or None)``, read in one pass: what 

335 ``^##\\s+([^:]+?)\\s*(?::\\s*(.*))?$`` captures on a line without line breaks. The id runs to 

336 the first colon without its trailing whitespace; an id made only of whitespace is the last 

337 space after ``##`` when there are at least two.""" 

338 rest = line[2:] if line.startswith("##") else "" 

339 name = rest.lstrip() 

340 spaces = len(rest) - len(name) 

341 if not spaces: 

342 return None 

343 rule_id, colon, title = name.partition(":") 

344 rule_id = rule_id.rstrip() 

345 if not rule_id: 345 ↛ 346line 345 didn't jump to line 346 because the condition on line 345 was never true

346 if spaces < 2: 

347 return None 

348 rule_id = rest[spaces - 1] 

349 return rule_id, (title.lstrip() if colon else None) 

350 

351 

352def parse_rules_markdown(text: str, *, domain: str, source: str) -> list[Rule]: 

353 """The rules of one project file: a ``## rule-id: Title`` section per rule, ``- key: value`` 

354 fields (severity, priority, when, exceptions, appliesTo, verifiedBy, supersedes, requires, 

355 budget), the rule as the first paragraph, and ``Good:``/``Bad:`` code blocks.""" 

356 head, body = _frontmatter(text) 

357 domain = str(head.get("domain") or domain) 

358 sections: list[tuple[str, str, list[str]]] = [] 

359 for line in body.split("\n"): 

360 heading = _heading(line) 

361 if heading: 

362 sections.append((heading[0], (heading[1] or "").strip(), [])) 

363 elif sections: 

364 sections[-1][2].append(line) 

365 rules: list[Rule] = [] 

366 for rule_id, title, lines in sections: 

367 section = _Section(rule_id, title, domain) 

368 for line in lines: 

369 section.feed(line) 

370 try: 

371 rules.append(Rule.model_validate({**section.as_data(), "layer": "C", "source": source})) 

372 except ValidationError as error: 

373 raise ConfigurationError( 

374 f"invalid review rule {rule_id} in {source}: {error}" 

375 ) from error 

376 return rules 

377 

378 

379def project_rules(workspace: Path, directory: str = RULES_DIRECTORY) -> list[Rule]: 

380 root = workspace / directory 

381 if not root.is_dir(): 

382 return [] 

383 rules: list[Rule] = [] 

384 for path in sorted(root.glob("*.md")): 

385 source = path.relative_to(workspace).as_posix() 

386 try: 

387 text = path.read_text(encoding="utf-8") 

388 rules.extend(parse_rules_markdown(text, domain=path.stem, source=source)) 

389 except (OSError, ValueError, yaml.YAMLError) as error: 

390 if isinstance(error, ConfigurationError): 

391 raise 

392 raise ConfigurationError(f"cannot read review rules {source}: {error}") from error 

393 return rules 

394 

395 

396# ----- merge ------------------------------------------------------------------------------------ 

397def merge( 

398 layers: Iterable[Rule], 

399 *, 

400 available_tools: set[str], 

401 reviewer_domains: set[str], 

402 workspace: Path | None = None, 

403) -> Catalog: 

404 """Merge the layers by id (C > B > A), apply ``supersedes`` and set aside the rules whose 

405 input is missing.""" 

406 by_id: dict[str, Rule] = {} 

407 for rule in layers: 

408 current = by_id.get(rule.rule_id) 

409 if current is None or _LAYER_RANK[rule.layer] >= _LAYER_RANK[current.layer]: 409 ↛ 407line 409 didn't jump to line 407 because the condition on line 409 was always true

410 by_id[rule.rule_id] = rule 

411 superseded = _superseded(by_id) 

412 inactive: list[InactiveRule] = [] 

413 active: list[Rule] = [] 

414 for rule_id in sorted(by_id): 

415 rule = by_id[rule_id] 

416 reason = _missing_input(rule, available_tools, reviewer_domains, workspace) 

417 if rule_id in superseded: 

418 reason = f"superseded by {superseded[rule_id]}" 

419 if reason is not None: 

420 inactive.append(InactiveRule(rule, reason)) 

421 else: 

422 active.append(rule) 

423 return Catalog(tuple(active), tuple(inactive)) 

424 

425 

426def _superseded(by_id: dict[str, Rule]) -> dict[str, str]: 

427 """``rule id -> the rule that supersedes it`` (the first one, in merge order).""" 

428 superseded: dict[str, str] = {} 

429 for rule in by_id.values(): 

430 for target in rule.supersedes: 

431 if target in by_id and target != rule.rule_id: 431 ↛ 430line 431 didn't jump to line 430 because the condition on line 431 was always true

432 superseded.setdefault(target, rule.rule_id) 

433 return superseded 

434 

435 

436def _missing_input( 

437 rule: Rule, tools: set[str], domains: set[str], workspace: Path | None 

438) -> str | None: 

439 for tool, _name in rule.tools: 

440 if tool != HARNESS_TOOL and tool not in tools: 

441 return f"input missing: the tool {tool} is not configured in the repository" 

442 for item in rule.requires: 

443 kind, _, value = item.partition(":") 

444 if kind == "tool" and value != HARNESS_TOOL and value not in tools: 444 ↛ 445line 444 didn't jump to line 445 because the condition on line 444 was never true

445 return f"input missing: the tool {value} is not configured in the repository" 

446 if kind == "file" and workspace is not None and not _file_exists(workspace, value): 

447 return f"input missing: no file matches {value}" 

448 if not rule.by_tool and rule.domain not in domains: 

449 return f"input missing: no reviewer covers the domain {rule.domain}" 

450 return None 

451 

452 

453def _file_exists(workspace: Path, pattern: str) -> bool: 

454 if not any(char in pattern for char in "*?["): 454 ↛ 456line 454 didn't jump to line 456 because the condition on line 454 was always true

455 return (workspace / pattern).exists() 

456 return next(iter(workspace.glob(pattern)), None) is not None 

457 

458 

459def configured_tools(standards: Any, workspace: Path, validator_ids: Iterable[str]) -> set[str]: 

460 """The pack tools the repository runs: a profile validator covers it, or one of its 

461 configuration files is present.""" 

462 from governed_harness.standards.selection import tool_validators 

463 

464 if standards is None: 464 ↛ 465line 464 didn't jump to line 465 because the condition on line 464 was never true

465 return set() 

466 known = set(validator_ids) 

467 tools = { 

468 tool.tool_id 

469 for pack in standards.packs 

470 for tool in pack.tools 

471 if any(item in known for item in tool.covered_by) 

472 } 

473 tools.update( 

474 tool.tool_id for _pack, tool, _definition in tool_validators(standards, workspace, known) 

475 ) 

476 return tools 

477 

478 

479__all__ = [ 

480 "HARNESS_TOOL", 

481 "RULES_DIRECTORY", 

482 "Catalog", 

483 "ConfirmationBudget", 

484 "InactiveRule", 

485 "Rule", 

486 "builtin_rules", 

487 "configured_tools", 

488 "merge", 

489 "pack_review", 

490 "pack_rules", 

491 "parse_rules_markdown", 

492 "project_rules", 

493]