Coverage for src/governed_harness/review/reviewers.py: 86%

219 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Reviewer definitions (#57): Markdown files with a YAML frontmatter. 

2 

3The frontmatter says who reviews and how: the domain, the model per provider family or provider 

4id, the effort, the timeout, the token budget, the modes it runs in (``run``, ``hook``, 

5``manual``, ``staged``), its diff slice, what activates it, the tools it may use and the MCP 

6servers of the project's allowlist it may reach. The body is the reviewer's prompt; between the 

7``BEGIN``/``END`` markers it holds the rules of its domain, a block written only by 

8``harness review rules sync`` (``--check`` reports drift). A fixed output contract is added by 

9the harness to every request; it is not part of the file. 

10 

11Six reviewers ship with the harness (``resources/review/agents``): quality, architecture, 

12resilience, tests, concurrency and pipeline-security. A project adds or replaces reviewers in 

13``.harness/review/agents/ID.md``. A built-in reviewer that the project did not copy renders its 

14block from the catalog when it is loaded, so it cannot drift.""" 

15 

16from __future__ import annotations 

17 

18import re 

19from dataclasses import dataclass 

20from importlib import resources 

21from pathlib import Path 

22from typing import Any, Literal 

23 

24import yaml 

25from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator 

26 

27from governed_harness.domain.errors import ConfigurationError 

28from governed_harness.evidence.hashing import sha256_bytes, sha256_json 

29from governed_harness.review.rules import Catalog, Rule, _frontmatter 

30 

31AGENTS_DIRECTORY = ".harness/review/agents" 

32BUILTIN_REVIEWERS: tuple[str, ...] = ( 

33 "quality", 

34 "architecture", 

35 "resilience", 

36 "tests", 

37 "concurrency", 

38 "pipeline-security", 

39) 

40BEGIN_MARKER = "<!-- BEGIN HARNESS REVIEW RULES (harness review rules sync; do not edit) -->" 

41END_MARKER = "<!-- END HARNESS REVIEW RULES -->" 

42Mode = Literal["run", "hook", "manual", "staged"] 

43MODES: tuple[Mode, ...] = ("run", "hook", "manual", "staged") 

44SLICES: tuple[str, ...] = ("all", "sources", "tests", "pipeline") 

45DEFAULT_TOOLS: tuple[str, ...] = ("Read", "Grep", "Glob") 

46_ID = re.compile(r"^[a-z][a-z0-9-]{0,63}$") 

47 

48 

49class ReviewerSpec(BaseModel): 

50 model_config = ConfigDict(extra="forbid", populate_by_name=True, frozen=True) 

51 

52 reviewer_id: str = Field(alias="id") 

53 domain: str 

54 title: str = "" 

55 models: dict[str, str] = Field(default_factory=dict) 

56 """Model per provider family (``claude-code``, ``codex``) or per provider id.""" 

57 effort: str | None = None 

58 timeout_seconds: int | None = Field(default=None, alias="timeoutSeconds", ge=10, le=7200) 

59 max_budget: int | None = Field(default=None, alias="maxBudget", ge=1) 

60 modes: tuple[Mode, ...] = MODES 

61 diff_slice: str | tuple[str, ...] = Field(default="sources", alias="diffSlice") 

62 """``all``, ``sources``, ``tests``, ``pipeline`` or a list of path globs.""" 

63 activation: str = "changed" 

64 """``changed`` (the slice is not empty), ``always``, or ``signal:NAME`` (a pack signal such as 

65 ``concurrency`` matches a changed line of the slice).""" 

66 patterns: tuple[str, ...] = () 

67 """Regular expressions that also activate the reviewer when a changed line matches one.""" 

68 tools: tuple[str, ...] = DEFAULT_TOOLS 

69 mcp_servers: tuple[str, ...] = Field(default=(), alias="mcpServers") 

70 provider: str | None = None 

71 max_findings: int | None = Field(default=None, alias="maxFindings", ge=1, le=200) 

72 

73 @field_validator("reviewer_id", "domain") 

74 @classmethod 

75 def _id(cls, value: str) -> str: 

76 if not _ID.match(value): 76 ↛ 77line 76 didn't jump to line 77 because the condition on line 76 was never true

77 raise ValueError(f"must be lower case with dashes: {value!r}") 

78 return value 

79 

80 @field_validator("diff_slice", mode="before") 

81 @classmethod 

82 def _slice(cls, value: Any) -> Any: 

83 if isinstance(value, str): 

84 if value not in SLICES: 84 ↛ 85line 84 didn't jump to line 85 because the condition on line 84 was never true

85 raise ValueError(f"diffSlice is one of {', '.join(SLICES)} or a list of globs") 

86 return value 

87 return tuple(str(item) for item in value or ()) 

88 

89 @field_validator("activation") 

90 @classmethod 

91 def _activation(cls, value: str) -> str: 

92 if value not in {"changed", "always"} and not value.startswith("signal:"): 92 ↛ 93line 92 didn't jump to line 93 because the condition on line 92 was never true

93 raise ValueError("activation is changed, always or signal:NAME") 

94 return value 

95 

96 @field_validator("patterns") 

97 @classmethod 

98 def _patterns(cls, value: tuple[str, ...]) -> tuple[str, ...]: 

99 for item in value: 

100 try: 

101 re.compile(item) 

102 except re.error as error: 

103 raise ValueError(f"invalid pattern {item!r}: {error}") from error 

104 return value 

105 

106 

107@dataclass(frozen=True) 

108class Reviewer: 

109 spec: ReviewerSpec 

110 body: str 

111 """The prompt with its rules block.""" 

112 source: str 

113 """``builtin`` or the project file it came from.""" 

114 raw: str 

115 """The file as written (its digest is the definition hash).""" 

116 

117 @property 

118 def reviewer_id(self) -> str: 

119 return self.spec.reviewer_id 

120 

121 @property 

122 def domain(self) -> str: 

123 return self.spec.domain 

124 

125 @property 

126 def definition_hash(self) -> str: 

127 return sha256_bytes(self.raw.encode("utf-8")) 

128 

129 @property 

130 def builtin(self) -> bool: 

131 return self.source == "builtin" 

132 

133 def block(self) -> str | None: 

134 return extract_block(self.body) 

135 

136 def model_for(self, provider_id: str, family: str) -> str | None: 

137 return self.spec.models.get(provider_id) or self.spec.models.get(family) 

138 

139 

140# ----- the generated rules block ---------------------------------------------------------------- 

141def _sentence(text: str) -> str: 

142 text = " ".join(text.split()) 

143 return text if not text or text.endswith((".", "!", "?")) else text + "." 

144 

145 

146def render_rule(rule: Rule) -> str: 

147 parts = [ 

148 f"- `{rule.rule_id}` ({rule.severity}, priority {rule.priority}): {_sentence(rule.rule)}" 

149 ] 

150 if rule.when: 

151 parts.append(f"When: {_sentence(rule.when)}") 

152 if rule.applies_to: 

153 parts.append(f"Files: {', '.join(rule.applies_to)}.") 

154 if rule.exceptions: 

155 parts.append("Exceptions: " + " ".join(_sentence(item) for item in rule.exceptions)) 

156 if rule.bad: 156 ↛ 157line 156 didn't jump to line 157 because the condition on line 156 was never true

157 parts.append("Bad: " + "; ".join(f"`{' '.join(item.split())}`" for item in rule.bad) + ".") 

158 if rule.good: 158 ↛ 159line 158 didn't jump to line 159 because the condition on line 158 was never true

159 parts.append( 

160 "Good: " + "; ".join(f"`{' '.join(item.split())}`" for item in rule.good) + "." 

161 ) 

162 budget = rule.budget 

163 parts.append( 

164 f"Confirm with at most {budget.max_reads} {budget.tool}(s) of the code under review; " 

165 "if you cannot confirm it, report nothing." 

166 ) 

167 return " ".join(parts) 

168 

169 

170def render_block(rules: tuple[Rule, ...]) -> str: 

171 """The rules a reviewer receives: the model-verified rules of its domain, blocking first, 

172 then by priority and id. Rules verified by a tool are never listed.""" 

173 ordered = sorted( 

174 (rule for rule in rules if not rule.by_tool), 

175 key=lambda rule: (not rule.blocking, rule.priority, rule.rule_id), 

176 ) 

177 lines = [BEGIN_MARKER, "## Rules", ""] 

178 if ordered: 178 ↛ 181line 178 didn't jump to line 181 because the condition on line 178 was always true

179 lines.extend(render_rule(rule) for rule in ordered) 

180 else: 

181 lines.append("No rule of this domain is active; report only defects you can show.") 

182 lines.extend(("", END_MARKER)) 

183 return "\n".join(lines) 

184 

185 

186def extract_block(body: str) -> str | None: 

187 start = body.find(BEGIN_MARKER) 

188 end = body.find(END_MARKER) 

189 if start == -1 or end == -1 or end < start: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true

190 return None 

191 return body[start : end + len(END_MARKER)] 

192 

193 

194def with_block(body: str, block: str) -> str: 

195 current = extract_block(body) 

196 if current is None: 196 ↛ 197line 196 didn't jump to line 197 because the condition on line 196 was never true

197 return body.rstrip("\n") + "\n\n" + block + "\n" 

198 return body.replace(current, block, 1) 

199 

200 

201# ----- loading ---------------------------------------------------------------------------------- 

202def parse_reviewer(text: str, *, source: str, fallback_id: str | None = None) -> Reviewer: 

203 try: 

204 head, body = _frontmatter(text) 

205 except (ValueError, yaml.YAMLError) as error: 

206 raise ConfigurationError(f"invalid reviewer {source}: {error}") from error 

207 if fallback_id and "id" not in head: 

208 head = {**head, "id": fallback_id} 

209 try: 

210 spec = ReviewerSpec.model_validate(head) 

211 except ValidationError as error: 

212 raise ConfigurationError(f"invalid reviewer {source}: {error}") from error 

213 return Reviewer(spec, body.strip("\n") + "\n", source, text) 

214 

215 

216def builtin_reviewer(reviewer_id: str) -> Reviewer: 

217 target = resources.files("governed_harness.resources").joinpath( 

218 "review", "agents", f"{reviewer_id}.md" 

219 ) 

220 return parse_reviewer(target.read_text(encoding="utf-8"), source="builtin") 

221 

222 

223def load_reviewers( 

224 workspace: Path, 

225 catalog: Catalog, 

226 *, 

227 enabled: tuple[str, ...] | None = None, 

228 directory: str = AGENTS_DIRECTORY, 

229) -> list[Reviewer]: 

230 """The built-in reviewers and the project's (a project file replaces the built-in reviewer 

231 with its id), filtered by ``review.panel.reviewers``, in id order. A built-in reviewer gets 

232 its rules block from the catalog; a project reviewer keeps the block in its file.""" 

233 found: dict[str, Reviewer] = {} 

234 for reviewer_id in BUILTIN_REVIEWERS: 

235 reviewer = builtin_reviewer(reviewer_id) 

236 block = render_block(catalog.for_domain(reviewer.domain)) 

237 found[reviewer_id] = Reviewer( 

238 reviewer.spec, with_block(reviewer.body, block), "builtin", reviewer.raw 

239 ) 

240 root = workspace / directory 

241 if root.is_dir(): 

242 for path in sorted(root.glob("*.md")): 

243 source = path.relative_to(workspace).as_posix() 

244 try: 

245 text = path.read_text(encoding="utf-8") 

246 except OSError as error: 

247 raise ConfigurationError(f"cannot read reviewer {source}: {error}") from error 

248 reviewer = parse_reviewer(text, source=source, fallback_id=path.stem) 

249 found[reviewer.reviewer_id] = reviewer 

250 selected = [found[key] for key in sorted(found) if enabled is None or key in set(enabled)] 

251 return selected 

252 

253 

254def reviewer_domains(workspace: Path, enabled: tuple[str, ...] | None = None) -> set[str]: 

255 """The domains the reviewers cover, read without the catalog (to decide which rules are 

256 inactive because no reviewer covers them).""" 

257 domains: set[str] = set() 

258 for reviewer_id in BUILTIN_REVIEWERS: 

259 if enabled is None or reviewer_id in enabled: 259 ↛ 258line 259 didn't jump to line 258 because the condition on line 259 was always true

260 domains.add(builtin_reviewer(reviewer_id).domain) 

261 root = workspace / AGENTS_DIRECTORY 

262 if root.is_dir(): 

263 for path in sorted(root.glob("*.md")): 

264 try: 

265 reviewer = parse_reviewer( 

266 path.read_text(encoding="utf-8"), source=path.name, fallback_id=path.stem 

267 ) 

268 except OSError: 

269 continue 

270 if enabled is None or reviewer.reviewer_id in enabled: 270 ↛ 263line 270 didn't jump to line 263 because the condition on line 270 was always true

271 domains.add(reviewer.domain) 

272 return domains 

273 

274 

275# ----- sync ------------------------------------------------------------------------------------- 

276@dataclass(frozen=True) 

277class SyncResult: 

278 reviewer_id: str 

279 path: str 

280 status: Literal["unchanged", "updated", "drift"] 

281 

282 def as_dict(self) -> dict[str, str]: 

283 return {"reviewer": self.reviewer_id, "path": self.path, "status": self.status} 

284 

285 

286def sync_blocks( 

287 workspace: Path, catalog: Catalog, *, check: bool, directory: str = AGENTS_DIRECTORY 

288) -> list[SyncResult]: 

289 """Write the rules block of every project reviewer (``check``: only report the ones whose 

290 block differs from the catalog). The only writer of the block.""" 

291 root = workspace / directory 

292 results: list[SyncResult] = [] 

293 if not root.is_dir(): 293 ↛ 294line 293 didn't jump to line 294 because the condition on line 293 was never true

294 return results 

295 for path in sorted(root.glob("*.md")): 

296 source = path.relative_to(workspace).as_posix() 

297 text = path.read_text(encoding="utf-8") 

298 reviewer = parse_reviewer(text, source=source, fallback_id=path.stem) 

299 block = render_block(catalog.for_domain(reviewer.domain)) 

300 if reviewer.block() == block: 

301 results.append(SyncResult(reviewer.reviewer_id, source, "unchanged")) 

302 continue 

303 if check: 

304 results.append(SyncResult(reviewer.reviewer_id, source, "drift")) 

305 continue 

306 path.write_text(_raw_head(text) + with_block(reviewer.body, block), encoding="utf-8") 

307 results.append(SyncResult(reviewer.reviewer_id, source, "updated")) 

308 return results 

309 

310 

311def _raw_head(text: str) -> str: 

312 """The frontmatter of a reviewer file as written, with its closing ``---`` line.""" 

313 if not text.startswith("---"): 313 ↛ 314line 313 didn't jump to line 314 because the condition on line 313 was never true

314 return "" 

315 lines = text.split("\n") 

316 for index in range(1, len(lines)): 316 ↛ 319line 316 didn't jump to line 319 because the loop on line 316 didn't complete

317 if lines[index].strip() == "---": 

318 return "\n".join(lines[: index + 1]) + "\n" 

319 return "" 

320 

321 

322def drift(reviewers: list[Reviewer], catalog: Catalog) -> list[str]: 

323 """Project reviewers whose rules block is not the catalog's.""" 

324 return [ 

325 reviewer.reviewer_id 

326 for reviewer in reviewers 

327 if not reviewer.builtin 

328 and reviewer.block() != render_block(catalog.for_domain(reviewer.domain)) 

329 ] 

330 

331 

332def definitions_hash(reviewers: list[Reviewer]) -> str: 

333 return sha256_json( 

334 [ 

335 { 

336 "id": item.reviewer_id, 

337 "hash": item.definition_hash, 

338 "body": sha256_bytes(item.body.encode()), 

339 } 

340 for item in reviewers 

341 ] 

342 ) 

343 

344 

345__all__ = [ 

346 "AGENTS_DIRECTORY", 

347 "BEGIN_MARKER", 

348 "BUILTIN_REVIEWERS", 

349 "END_MARKER", 

350 "MODES", 

351 "SLICES", 

352 "Reviewer", 

353 "ReviewerSpec", 

354 "SyncResult", 

355 "builtin_reviewer", 

356 "definitions_hash", 

357 "drift", 

358 "extract_block", 

359 "load_reviewers", 

360 "parse_reviewer", 

361 "render_block", 

362 "render_rule", 

363 "reviewer_domains", 

364 "sync_blocks", 

365 "with_block", 

366]