Coverage for src/governed_harness/review/reviewers.py: 86%
219 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Reviewer definitions (#57): Markdown files with a YAML frontmatter.
3The frontmatter says who reviews and how: the domain, the model per provider family or provider
4id, the effort, the timeout, the token budget, the modes it runs in (``run``, ``hook``,
5``manual``, ``staged``), its diff slice, what activates it, the tools it may use and the MCP
6servers of the project's allowlist it may reach. The body is the reviewer's prompt; between the
7``BEGIN``/``END`` markers it holds the rules of its domain, a block written only by
8``harness review rules sync`` (``--check`` reports drift). A fixed output contract is added by
9the harness to every request; it is not part of the file.
11Six reviewers ship with the harness (``resources/review/agents``): quality, architecture,
12resilience, tests, concurrency and pipeline-security. A project adds or replaces reviewers in
13``.harness/review/agents/ID.md``. A built-in reviewer that the project did not copy renders its
14block from the catalog when it is loaded, so it cannot drift."""
16from __future__ import annotations
18import re
19from dataclasses import dataclass
20from importlib import resources
21from pathlib import Path
22from typing import Any, Literal
24import yaml
25from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
27from governed_harness.domain.errors import ConfigurationError
28from governed_harness.evidence.hashing import sha256_bytes, sha256_json
29from governed_harness.review.rules import Catalog, Rule, _frontmatter
31AGENTS_DIRECTORY = ".harness/review/agents"
32BUILTIN_REVIEWERS: tuple[str, ...] = (
33 "quality",
34 "architecture",
35 "resilience",
36 "tests",
37 "concurrency",
38 "pipeline-security",
39)
40BEGIN_MARKER = "<!-- BEGIN HARNESS REVIEW RULES (harness review rules sync; do not edit) -->"
41END_MARKER = "<!-- END HARNESS REVIEW RULES -->"
42Mode = Literal["run", "hook", "manual", "staged"]
43MODES: tuple[Mode, ...] = ("run", "hook", "manual", "staged")
44SLICES: tuple[str, ...] = ("all", "sources", "tests", "pipeline")
45DEFAULT_TOOLS: tuple[str, ...] = ("Read", "Grep", "Glob")
46_ID = re.compile(r"^[a-z][a-z0-9-]{0,63}$")
49class ReviewerSpec(BaseModel):
50 model_config = ConfigDict(extra="forbid", populate_by_name=True, frozen=True)
52 reviewer_id: str = Field(alias="id")
53 domain: str
54 title: str = ""
55 models: dict[str, str] = Field(default_factory=dict)
56 """Model per provider family (``claude-code``, ``codex``) or per provider id."""
57 effort: str | None = None
58 timeout_seconds: int | None = Field(default=None, alias="timeoutSeconds", ge=10, le=7200)
59 max_budget: int | None = Field(default=None, alias="maxBudget", ge=1)
60 modes: tuple[Mode, ...] = MODES
61 diff_slice: str | tuple[str, ...] = Field(default="sources", alias="diffSlice")
62 """``all``, ``sources``, ``tests``, ``pipeline`` or a list of path globs."""
63 activation: str = "changed"
64 """``changed`` (the slice is not empty), ``always``, or ``signal:NAME`` (a pack signal such as
65 ``concurrency`` matches a changed line of the slice)."""
66 patterns: tuple[str, ...] = ()
67 """Regular expressions that also activate the reviewer when a changed line matches one."""
68 tools: tuple[str, ...] = DEFAULT_TOOLS
69 mcp_servers: tuple[str, ...] = Field(default=(), alias="mcpServers")
70 provider: str | None = None
71 max_findings: int | None = Field(default=None, alias="maxFindings", ge=1, le=200)
73 @field_validator("reviewer_id", "domain")
74 @classmethod
75 def _id(cls, value: str) -> str:
76 if not _ID.match(value): 76 ↛ 77line 76 didn't jump to line 77 because the condition on line 76 was never true
77 raise ValueError(f"must be lower case with dashes: {value!r}")
78 return value
80 @field_validator("diff_slice", mode="before")
81 @classmethod
82 def _slice(cls, value: Any) -> Any:
83 if isinstance(value, str):
84 if value not in SLICES: 84 ↛ 85line 84 didn't jump to line 85 because the condition on line 84 was never true
85 raise ValueError(f"diffSlice is one of {', '.join(SLICES)} or a list of globs")
86 return value
87 return tuple(str(item) for item in value or ())
89 @field_validator("activation")
90 @classmethod
91 def _activation(cls, value: str) -> str:
92 if value not in {"changed", "always"} and not value.startswith("signal:"): 92 ↛ 93line 92 didn't jump to line 93 because the condition on line 92 was never true
93 raise ValueError("activation is changed, always or signal:NAME")
94 return value
96 @field_validator("patterns")
97 @classmethod
98 def _patterns(cls, value: tuple[str, ...]) -> tuple[str, ...]:
99 for item in value:
100 try:
101 re.compile(item)
102 except re.error as error:
103 raise ValueError(f"invalid pattern {item!r}: {error}") from error
104 return value
107@dataclass(frozen=True)
108class Reviewer:
109 spec: ReviewerSpec
110 body: str
111 """The prompt with its rules block."""
112 source: str
113 """``builtin`` or the project file it came from."""
114 raw: str
115 """The file as written (its digest is the definition hash)."""
117 @property
118 def reviewer_id(self) -> str:
119 return self.spec.reviewer_id
121 @property
122 def domain(self) -> str:
123 return self.spec.domain
125 @property
126 def definition_hash(self) -> str:
127 return sha256_bytes(self.raw.encode("utf-8"))
129 @property
130 def builtin(self) -> bool:
131 return self.source == "builtin"
133 def block(self) -> str | None:
134 return extract_block(self.body)
136 def model_for(self, provider_id: str, family: str) -> str | None:
137 return self.spec.models.get(provider_id) or self.spec.models.get(family)
140# ----- the generated rules block ----------------------------------------------------------------
141def _sentence(text: str) -> str:
142 text = " ".join(text.split())
143 return text if not text or text.endswith((".", "!", "?")) else text + "."
146def render_rule(rule: Rule) -> str:
147 parts = [
148 f"- `{rule.rule_id}` ({rule.severity}, priority {rule.priority}): {_sentence(rule.rule)}"
149 ]
150 if rule.when:
151 parts.append(f"When: {_sentence(rule.when)}")
152 if rule.applies_to:
153 parts.append(f"Files: {', '.join(rule.applies_to)}.")
154 if rule.exceptions:
155 parts.append("Exceptions: " + " ".join(_sentence(item) for item in rule.exceptions))
156 if rule.bad: 156 ↛ 157line 156 didn't jump to line 157 because the condition on line 156 was never true
157 parts.append("Bad: " + "; ".join(f"`{' '.join(item.split())}`" for item in rule.bad) + ".")
158 if rule.good: 158 ↛ 159line 158 didn't jump to line 159 because the condition on line 158 was never true
159 parts.append(
160 "Good: " + "; ".join(f"`{' '.join(item.split())}`" for item in rule.good) + "."
161 )
162 budget = rule.budget
163 parts.append(
164 f"Confirm with at most {budget.max_reads} {budget.tool}(s) of the code under review; "
165 "if you cannot confirm it, report nothing."
166 )
167 return " ".join(parts)
170def render_block(rules: tuple[Rule, ...]) -> str:
171 """The rules a reviewer receives: the model-verified rules of its domain, blocking first,
172 then by priority and id. Rules verified by a tool are never listed."""
173 ordered = sorted(
174 (rule for rule in rules if not rule.by_tool),
175 key=lambda rule: (not rule.blocking, rule.priority, rule.rule_id),
176 )
177 lines = [BEGIN_MARKER, "## Rules", ""]
178 if ordered: 178 ↛ 181line 178 didn't jump to line 181 because the condition on line 178 was always true
179 lines.extend(render_rule(rule) for rule in ordered)
180 else:
181 lines.append("No rule of this domain is active; report only defects you can show.")
182 lines.extend(("", END_MARKER))
183 return "\n".join(lines)
186def extract_block(body: str) -> str | None:
187 start = body.find(BEGIN_MARKER)
188 end = body.find(END_MARKER)
189 if start == -1 or end == -1 or end < start: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true
190 return None
191 return body[start : end + len(END_MARKER)]
194def with_block(body: str, block: str) -> str:
195 current = extract_block(body)
196 if current is None: 196 ↛ 197line 196 didn't jump to line 197 because the condition on line 196 was never true
197 return body.rstrip("\n") + "\n\n" + block + "\n"
198 return body.replace(current, block, 1)
201# ----- loading ----------------------------------------------------------------------------------
202def parse_reviewer(text: str, *, source: str, fallback_id: str | None = None) -> Reviewer:
203 try:
204 head, body = _frontmatter(text)
205 except (ValueError, yaml.YAMLError) as error:
206 raise ConfigurationError(f"invalid reviewer {source}: {error}") from error
207 if fallback_id and "id" not in head:
208 head = {**head, "id": fallback_id}
209 try:
210 spec = ReviewerSpec.model_validate(head)
211 except ValidationError as error:
212 raise ConfigurationError(f"invalid reviewer {source}: {error}") from error
213 return Reviewer(spec, body.strip("\n") + "\n", source, text)
216def builtin_reviewer(reviewer_id: str) -> Reviewer:
217 target = resources.files("governed_harness.resources").joinpath(
218 "review", "agents", f"{reviewer_id}.md"
219 )
220 return parse_reviewer(target.read_text(encoding="utf-8"), source="builtin")
223def load_reviewers(
224 workspace: Path,
225 catalog: Catalog,
226 *,
227 enabled: tuple[str, ...] | None = None,
228 directory: str = AGENTS_DIRECTORY,
229) -> list[Reviewer]:
230 """The built-in reviewers and the project's (a project file replaces the built-in reviewer
231 with its id), filtered by ``review.panel.reviewers``, in id order. A built-in reviewer gets
232 its rules block from the catalog; a project reviewer keeps the block in its file."""
233 found: dict[str, Reviewer] = {}
234 for reviewer_id in BUILTIN_REVIEWERS:
235 reviewer = builtin_reviewer(reviewer_id)
236 block = render_block(catalog.for_domain(reviewer.domain))
237 found[reviewer_id] = Reviewer(
238 reviewer.spec, with_block(reviewer.body, block), "builtin", reviewer.raw
239 )
240 root = workspace / directory
241 if root.is_dir():
242 for path in sorted(root.glob("*.md")):
243 source = path.relative_to(workspace).as_posix()
244 try:
245 text = path.read_text(encoding="utf-8")
246 except OSError as error:
247 raise ConfigurationError(f"cannot read reviewer {source}: {error}") from error
248 reviewer = parse_reviewer(text, source=source, fallback_id=path.stem)
249 found[reviewer.reviewer_id] = reviewer
250 selected = [found[key] for key in sorted(found) if enabled is None or key in set(enabled)]
251 return selected
254def reviewer_domains(workspace: Path, enabled: tuple[str, ...] | None = None) -> set[str]:
255 """The domains the reviewers cover, read without the catalog (to decide which rules are
256 inactive because no reviewer covers them)."""
257 domains: set[str] = set()
258 for reviewer_id in BUILTIN_REVIEWERS:
259 if enabled is None or reviewer_id in enabled: 259 ↛ 258line 259 didn't jump to line 258 because the condition on line 259 was always true
260 domains.add(builtin_reviewer(reviewer_id).domain)
261 root = workspace / AGENTS_DIRECTORY
262 if root.is_dir():
263 for path in sorted(root.glob("*.md")):
264 try:
265 reviewer = parse_reviewer(
266 path.read_text(encoding="utf-8"), source=path.name, fallback_id=path.stem
267 )
268 except OSError:
269 continue
270 if enabled is None or reviewer.reviewer_id in enabled: 270 ↛ 263line 270 didn't jump to line 263 because the condition on line 270 was always true
271 domains.add(reviewer.domain)
272 return domains
275# ----- sync -------------------------------------------------------------------------------------
276@dataclass(frozen=True)
277class SyncResult:
278 reviewer_id: str
279 path: str
280 status: Literal["unchanged", "updated", "drift"]
282 def as_dict(self) -> dict[str, str]:
283 return {"reviewer": self.reviewer_id, "path": self.path, "status": self.status}
286def sync_blocks(
287 workspace: Path, catalog: Catalog, *, check: bool, directory: str = AGENTS_DIRECTORY
288) -> list[SyncResult]:
289 """Write the rules block of every project reviewer (``check``: only report the ones whose
290 block differs from the catalog). The only writer of the block."""
291 root = workspace / directory
292 results: list[SyncResult] = []
293 if not root.is_dir(): 293 ↛ 294line 293 didn't jump to line 294 because the condition on line 293 was never true
294 return results
295 for path in sorted(root.glob("*.md")):
296 source = path.relative_to(workspace).as_posix()
297 text = path.read_text(encoding="utf-8")
298 reviewer = parse_reviewer(text, source=source, fallback_id=path.stem)
299 block = render_block(catalog.for_domain(reviewer.domain))
300 if reviewer.block() == block:
301 results.append(SyncResult(reviewer.reviewer_id, source, "unchanged"))
302 continue
303 if check:
304 results.append(SyncResult(reviewer.reviewer_id, source, "drift"))
305 continue
306 path.write_text(_raw_head(text) + with_block(reviewer.body, block), encoding="utf-8")
307 results.append(SyncResult(reviewer.reviewer_id, source, "updated"))
308 return results
311def _raw_head(text: str) -> str:
312 """The frontmatter of a reviewer file as written, with its closing ``---`` line."""
313 if not text.startswith("---"): 313 ↛ 314line 313 didn't jump to line 314 because the condition on line 313 was never true
314 return ""
315 lines = text.split("\n")
316 for index in range(1, len(lines)): 316 ↛ 319line 316 didn't jump to line 319 because the loop on line 316 didn't complete
317 if lines[index].strip() == "---":
318 return "\n".join(lines[: index + 1]) + "\n"
319 return ""
322def drift(reviewers: list[Reviewer], catalog: Catalog) -> list[str]:
323 """Project reviewers whose rules block is not the catalog's."""
324 return [
325 reviewer.reviewer_id
326 for reviewer in reviewers
327 if not reviewer.builtin
328 and reviewer.block() != render_block(catalog.for_domain(reviewer.domain))
329 ]
332def definitions_hash(reviewers: list[Reviewer]) -> str:
333 return sha256_json(
334 [
335 {
336 "id": item.reviewer_id,
337 "hash": item.definition_hash,
338 "body": sha256_bytes(item.body.encode()),
339 }
340 for item in reviewers
341 ]
342 )
345__all__ = [
346 "AGENTS_DIRECTORY",
347 "BEGIN_MARKER",
348 "BUILTIN_REVIEWERS",
349 "END_MARKER",
350 "MODES",
351 "SLICES",
352 "Reviewer",
353 "ReviewerSpec",
354 "SyncResult",
355 "builtin_reviewer",
356 "definitions_hash",
357 "drift",
358 "extract_block",
359 "load_reviewers",
360 "parse_reviewer",
361 "render_block",
362 "render_rule",
363 "reviewer_domains",
364 "sync_blocks",
365 "with_block",
366]