Coverage for src/governed_harness/orchestration/engineering.py: 87%

226 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Standards, principles and testing strategy in the run (#56). 

2 

3``AgentResults`` delegates to this class; a project without the ``standards``, ``testing``, 

4``architecture`` or ``verification.principles`` settings never reaches it. 

5 

6Token cost rules, all deterministic: 

7 

8* the implement request carries only the standards cards that apply to the files the call works 

9 on, compact, selected once per digest of (packs, paths) and reused from the cache; 

10* the review call carries the cards no tool verifies and the principles checklist in its own 

11 request: no extra call; 

12* the TDD evidence (red, green, refactor) is measured by running the tests, not by asking.""" 

13 

14from __future__ import annotations 

15 

16from datetime import UTC, datetime 

17from pathlib import Path 

18from typing import TYPE_CHECKING, Any 

19 

20from governed_harness.capabilities import grants_from_rules 

21from governed_harness.checks import DiffFile, Issue, is_test_path 

22from governed_harness.checks.principles import PRINCIPLES_CHECKLIST 

23from governed_harness.configuration.engineering import ( 

24 PrinciplesConfig, 

25 StandardsConfig, 

26 TestingConfig, 

27) 

28from governed_harness.domain.enums import ActorType, FindingSeverity, PhaseId, ResultStatus 

29from governed_harness.domain.models import Actor, ChangeSet, Execution, PhaseExecution, Task 

30from governed_harness.orchestration.engine_types import Strategy as Strategy 

31from governed_harness.orchestration.workspace_ops import materialized 

32from governed_harness.runtime import CancellationToken, SafeProcessRunner 

33from governed_harness.runtime.process_runner import CommandSpec 

34from governed_harness.standards import ( 

35 ProjectStandards, 

36 WorkspaceFacts, 

37 bdd_frameworks, 

38 builtin_pack, 

39 cards_payload, 

40 checklist_payload, 

41 detect_from_facts, 

42 project_standards, 

43 select_cards, 

44 selection_digest, 

45) 

46from governed_harness.validators import ValidatorOutput 

47 

48if TYPE_CHECKING: 

49 from governed_harness.orchestration.hosts import ResultsHost 

50 

51TDD_ID = "harness.tdd" 

52SETUP_FLAG = "projectsetup" 

53 

54 

55def _architecture_notes(architecture: dict[str, Any]) -> list[str]: 

56 """The sentences the instructions gain for each part of the architecture block.""" 

57 notes: list[str] = [] 

58 if "layers" in architecture: 

59 notes.append( 

60 "Respect the architecture block: a file of one layer may import only the " 

61 "layers listed as allowed for it." 

62 ) 

63 if "expert" in architecture: 

64 notes.append( 

65 "Act as the expert of the chosen architecture style: follow the expert prompt of the " 

66 "architecture block, the organization it carries and the style document it points " 

67 "to, and keep the implementation aligned with the style." 

68 ) 

69 if "reverseEngineering" in architecture: 

70 notes.append( 

71 "The architecture block points to the reverse engineering documents of this " 

72 "project: read them before searching the code and follow their working agreement; " 

73 "do not analyse the whole project again." 

74 ) 

75 return notes 

76 

77 

78def detect_testing(workspace: Path, technologies: tuple[str, ...] = ()) -> Strategy: 

79 """``bdd`` when the repository has feature files or a BDD framework, ``conventional`` 

80 when it has tests, else ``unknown`` (asked in INTENT under ``intake.projectSetup``).""" 

81 facts = WorkspaceFacts.read(workspace) 

82 packs = [builtin_pack(item) for item in detect_from_facts(facts, technologies)] 

83 frameworks = bdd_frameworks(packs, facts) 

84 if frameworks: 84 ↛ 85line 84 didn't jump to line 85 because the condition on line 84 was never true

85 return Strategy("bdd", "detected", tuple(frameworks)) 

86 if any(is_test_path(item) for item in facts.files): 

87 return Strategy("conventional", "detected") 

88 return Strategy("unknown", "unknown") 

89 

90 

91class Engineering: 

92 def __init__(self, results: ResultsHost) -> None: 

93 self.results = results 

94 self._standards: ProjectStandards | None = None 

95 self._detected: Strategy | None = None 

96 """The detected strategy, read once per command (the workspace is walked once).""" 

97 

98 # ----- settings ----------------------------------------------------------------------------- 

99 @property 

100 def standards_config(self) -> StandardsConfig | None: 

101 return self.results.project.standards 

102 

103 @property 

104 def principles_config(self) -> PrinciplesConfig | None: 

105 verification = self.results.project.verification 

106 return verification.principles if verification else None 

107 

108 @property 

109 def testing_config(self) -> TestingConfig | None: 

110 return self.results.project.testing 

111 

112 @property 

113 def configured(self) -> bool: 

114 project = self.results.project 

115 return any( 

116 ( 

117 project.standards is not None, 

118 project.testing is not None, 

119 project.architecture is not None, 

120 self.principles_config is not None, 

121 bool(project.intake and project.intake.project_setup is not None), 

122 ) 

123 ) 

124 

125 def technologies(self) -> tuple[str, ...]: 

126 return tuple(item.technology for item in self.results.s.resolved.profiles) 

127 

128 def standards(self) -> ProjectStandards | None: 

129 config = self.standards_config 

130 if config is None: 

131 return None 

132 if self._standards is None: 

133 packs = config.packs 

134 answered = self.setup_record(self.results.project.project_id).get("standards") 

135 if (packs is None or "auto" in packs) and isinstance(answered, list) and answered: 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true

136 packs = tuple(str(item) for item in answered) 

137 self._standards = project_standards( 

138 self.results.s.paths.workspace, 

139 packs=packs, 

140 overrides=config.overrides_path, 

141 disabled=config.disabled or (), 

142 technologies=self.technologies(), 

143 ) 

144 return self._standards 

145 

146 # ----- project setup answers ------------------------------------------------------------- 

147 def setup_record(self, project_id: str) -> dict[str, Any]: 

148 value = self.results.flag_json(f"{SETUP_FLAG}:{project_id}") 

149 return value if isinstance(value, dict) else {} 

150 

151 # ----- testing strategy ------------------------------------------------------------------ 

152 def strategy(self, project_id: str) -> Strategy: 

153 """The testing strategy: the configuration, else a person's answer (rule ``P1``), else, 

154 only under ``testing.strategy: auto``, what the repository shows. Without the 

155 ``testing`` section nothing is detected: the earlier behaviour (``conventional``).""" 

156 config = self.testing_config 

157 if config is not None and config.strategy not in {None, "auto"}: 

158 return Strategy(str(config.strategy), "configuration") 

159 answered = self.setup_record(project_id).get("testing") 

160 if isinstance(answered, str) and answered in {"tdd", "bdd", "conventional"}: 

161 return Strategy(answered, "project-setup") 

162 if config is None: 

163 return Strategy("conventional", "off") 

164 if self._detected is None: 

165 self._detected = detect_testing(self.results.s.paths.workspace, self.technologies()) 

166 return self._detected 

167 

168 def features_directory(self) -> str: 

169 config = self.testing_config 

170 return config.features_path if config is not None else "features" 

171 

172 def bdd_command(self) -> tuple[str, ...]: 

173 config = self.testing_config 

174 if config is not None and config.bdd_command: 174 ↛ 176line 174 didn't jump to line 176 because the condition on line 174 was always true

175 return config.bdd_command 

176 for technology in self._languages(): 

177 runner = builtin_pack(technology).runner 

178 if runner.bdd: 

179 return runner.bdd 

180 return ("python", "-m", "behave") 

181 

182 def test_command(self, files: list[str]) -> tuple[str, ...]: 

183 """The command that runs ``files`` (TDD red check): ``testing.testCommand`` with the 

184 files appended, else the pack runner of the files' language.""" 

185 config = self.testing_config 

186 if config is not None and config.test_command: 186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true

187 return (*config.test_command, *files) 

188 for technology in self._languages(files): 188 ↛ 192line 188 didn't jump to line 192 because the loop on line 188 didn't complete

189 runner = builtin_pack(technology).runner 

190 if runner.test: 190 ↛ 188line 190 didn't jump to line 188 because the condition on line 190 was always true

191 return (*runner.test, *files) if runner.test_files else runner.test 

192 return ("python", "-m", "pytest", "-q", *files) 

193 

194 def _languages(self, files: list[str] | None = None) -> list[str]: 

195 standards = self.standards() 

196 ids = [item.pack_id for item in standards.packs] if standards else [] 

197 if not ids: 197 ↛ 199line 197 didn't jump to line 199 because the condition on line 197 was always true

198 ids = [item for item in self.technologies() if item in {"python", "node"}] 

199 if files: 199 ↛ 215line 199 didn't jump to line 215 because the condition on line 199 was always true

200 by_suffix = { 

201 ".py": "python", 

202 ".go": "go", 

203 ".rs": "rust", 

204 ".java": "java", 

205 ".kt": "kotlin", 

206 ".cs": "csharp", 

207 ".php": "php", 

208 ".rb": "ruby", 

209 ".swift": "swift", 

210 } 

211 first = [ 

212 by_suffix[Path(item).suffix] for item in files if Path(item).suffix in by_suffix 

213 ] 

214 ids = [*first, *ids] 

215 return list(dict.fromkeys(item for item in ids if item)) 

216 

217 # ----- the implement request -------------------------------------------------------------- 

218 def candidate_paths(self, task: Task, changed: list[str], extra: list[str]) -> list[str]: 

219 from governed_harness.agents.context_manifest import referenced_paths 

220 

221 workspace = self.results.s.paths.workspace 

222 paths = [ 

223 *[str(item) for item in task.metadata.get("ownedPaths") or []], 

224 *referenced_paths(task, workspace), 

225 *(patch.path for patch in task.implementation.patches), 

226 *changed, 

227 *extra, 

228 ] 

229 return list(dict.fromkeys(item.replace("\\", "/") for item in paths if item)) 

230 

231 def implement_extra( 

232 self, 

233 execution: Execution, 

234 phase: PhaseExecution, 

235 paths: list[str], 

236 ) -> tuple[dict[str, Any], str]: 

237 """The ``standards`` and ``testing`` blocks of the implement request and the sentence 

238 the instructions gain for them.""" 

239 extra: dict[str, Any] = {} 

240 notes: list[str] = [] 

241 standards = self.standards() 

242 config = self.standards_config 

243 if standards is not None and config is not None and config.cards_enabled: 

244 payload = self._cards(execution, phase, standards, paths, config.card_limit) 

245 if payload["cards"]: 245 ↛ 251line 245 didn't jump to line 251 because the condition on line 245 was always true

246 extra["standards"] = payload 

247 notes.append( 

248 "Follow the standards cards for the files they apply to; listed exceptions " 

249 "are allowed." 

250 ) 

251 if self.testing_config is not None or self.setup_record(execution.project_id): 

252 strategy = self.strategy(execution.project_id) 

253 if strategy.strategy in {"tdd", "bdd"}: 

254 extra["testing"] = strategy.as_dict() 

255 if strategy.strategy == "tdd": 

256 notes.append( 

257 "Work test-first (TDD): write or change the tests for each criterion first, " 

258 "make them pass with the smallest change, then refactor with every test " 

259 "green. The harness runs your new tests on the code before your change and " 

260 "they must fail there." 

261 ) 

262 elif strategy.strategy == "bdd": 

263 notes.append( 

264 "Behaviour-driven (BDD): the approved feature files are frozen; write the " 

265 "step definitions and the code that make their scenarios pass." 

266 ) 

267 architecture = self.results.architecture.request_extra() 

268 if architecture is not None: 

269 extra["architecture"] = architecture 

270 notes.extend(_architecture_notes(architecture)) 

271 return extra, " ".join(notes) 

272 

273 def _cards( 

274 self, 

275 execution: Execution, 

276 phase: PhaseExecution, 

277 standards: ProjectStandards, 

278 paths: list[str], 

279 limit: int, 

280 ) -> dict[str, Any]: 

281 digest = selection_digest(standards, paths, limit) 

282 key = f"standardscards:{execution.project_id}:{digest}" 

283 cached = self.results.flag_json(key) 

284 reused = isinstance(cached, dict) and isinstance(cached.get("cards"), list) 

285 if reused: 

286 payload: dict[str, Any] = cached 

287 else: 

288 cards = select_cards(standards, paths, limit) 

289 payload = cards_payload(cards, digest, packs=[item.pack_id for item in standards.packs]) 

290 self.results.set_flag_json(key, payload) 

291 self.results.s.events.append( 

292 execution.execution_id, 

293 "standards.cards.selected", 

294 { 

295 "digest": digest, 

296 "packs": payload["packs"], 

297 "cards": [item["id"] for item in payload["cards"]], 

298 "paths": len(paths), 

299 "reused": reused, 

300 }, 

301 phase_execution_id=phase.phase_execution_id, 

302 ) 

303 return payload 

304 

305 # ----- the review call ------------------------------------------------------------------- 

306 def review_extra(self, change_set: ChangeSet) -> tuple[dict[str, Any], str]: 

307 """The checklist the existing review call gains: the cards no tool verifies, for the 

308 files the ChangeSet changed, and the principles checklist. No extra call.""" 

309 checklist: dict[str, Any] = {} 

310 standards = self.standards() 

311 config = self.standards_config 

312 paths = [item.path for item in change_set.files] 

313 if standards is not None and config is not None and config.cards_enabled: 313 ↛ 317line 313 didn't jump to line 317 because the condition on line 313 was always true

314 cards = select_cards(standards, paths, config.card_limit, review_only=True) 

315 if cards: 315 ↛ 317line 315 didn't jump to line 317 because the condition on line 315 was always true

316 checklist["standards"] = checklist_payload(cards) 

317 principles = self.principles_config 

318 if principles is not None and principles.enabled and principles.checklist is not False: 318 ↛ 322line 318 didn't jump to line 322 because the condition on line 318 was always true

319 checklist["principles"] = [ 

320 {"id": rule, "check": text} for rule, text in PRINCIPLES_CHECKLIST 

321 ] 

322 if not checklist: 322 ↛ 323line 322 didn't jump to line 323 because the condition on line 322 was never true

323 return {}, "" 

324 return {"checklist": checklist}, ( 

325 "Also check each checklist item against the ChangeSet; report a violation as a " 

326 "finding whose rule is the item id." 

327 ) 

328 

329 # ----- TDD evidence (VERIFICATION) --------------------------------------------------------- 

330 def tdd_validation( 

331 self, 

332 execution: Execution, 

333 change_set: ChangeSet, 

334 diff: list[DiffFile], 

335 outputs_so_far: list[ValidatorOutput], 

336 ) -> ValidatorOutput | None: 

337 if not self._tdd_applies(execution, change_set): 

338 return None 

339 results = self.results 

340 started = datetime.now(UTC) 

341 tests = [item.path for item in diff if not item.is_deleted and is_test_path(item.path)] 

342 sources = [item.path for item in diff if not is_test_path(item.path)] 

343 record: dict[str, Any] = {"tests": tests, "sources": sources} 

344 record["red"], issues = self._red_step(execution, tests, sources) 

345 latest = results.engine._latest_validations(execution.execution_id, change_set.digest) 

346 green = [ 

347 item.validator_id 

348 for item in latest 

349 if item.mandatory and item.status is not ResultStatus.PASSED 

350 ] 

351 record["green"] = {"status": "PASSED" if not green else "FAILED", "failing": green} 

352 refactor_checks = {"harness.principles", "harness.architecture", "harness.layers"} 

353 refactor_blocking = [ 

354 output.result.validator_id 

355 for output in outputs_so_far 

356 if output.result.validator_id in refactor_checks 

357 and output.result.status is not ResultStatus.PASSED 

358 ] 

359 record["refactor"] = { 

360 "status": "PASSED" if not refactor_blocking and not green else "FAILED", 

361 "checks": sorted(refactor_checks), 

362 "failing": refactor_blocking, 

363 } 

364 red = record["red"] 

365 ref = results.record_json( 

366 execution, 

367 PhaseId.VERIFICATION, 

368 record, 

369 kind="tdd-evidence", 

370 summary=( 

371 f"TDD: red {red if isinstance(red, str) else red['status']}, " 

372 f"green {record['green']['status']}, refactor {record['refactor']['status']}" 

373 ), 

374 ) 

375 output = results.verification._output( 

376 execution, 

377 change_set, 

378 TDD_ID, 

379 "enforce", 

380 issues, 

381 "test-driven development (red, green, refactor)", 

382 started=started, 

383 extra_evidence=(ref,), 

384 report={"tdd": record}, 

385 ) 

386 return output 

387 

388 def _tdd_applies(self, execution: Execution, change_set: ChangeSet) -> bool: 

389 """Whether the TDD evidence is checked: the strategy is ``tdd`` and the ChangeSet is 

390 not exempt (``friction.changeTypes``, #58: a documentation or configuration change).""" 

391 if self.testing_config is None and not self.setup_record(execution.project_id): 

392 return False 

393 if self.strategy(execution.project_id).strategy != "tdd": 

394 return False 

395 friction = self.results.engine.friction 

396 return not ( 

397 friction.active and friction.tests_exempt(execution, change_set, "TDD evidence") 

398 ) 

399 

400 def _red_step( 

401 self, execution: Execution, tests: list[str], sources: list[str] 

402 ) -> tuple[str | dict[str, Any], list[Issue]]: 

403 """The red step: the tests the change adds must fail on the code before it.""" 

404 if sources and not tests: 404 ↛ 405line 404 didn't jump to line 405 because the condition on line 404 was never true

405 no_tests = Issue( 

406 rule_id="tdd.no-tests", 

407 severity=FindingSeverity.HIGH, 

408 message="The change modifies code but adds or changes no test (TDD)", 

409 category="tests", 

410 recommendation="Write the failing test first, then the code.", 

411 ) 

412 return "NO_TESTS", [no_tests] 

413 if not (tests and sources): 413 ↛ 414line 413 didn't jump to line 414 because the condition on line 413 was never true

414 return "NOT_APPLICABLE", [] 

415 red = self._red(execution, tests, sources) 

416 if red["status"] == ResultStatus.PASSED.value: 

417 not_red = Issue( 

418 rule_id="tdd.not-red", 

419 severity=FindingSeverity.HIGH, 

420 message=( 

421 f"The tests the change adds ({', '.join(tests[:5])}) already pass on " 

422 "the code before the change: they were not written first" 

423 ), 

424 category="tests", 

425 recommendation="Write a test that fails without the change.", 

426 ) 

427 return red, [not_red] 

428 if red["status"] != ResultStatus.FAILED.value: 428 ↛ 429line 428 didn't jump to line 429 because the condition on line 428 was never true

429 unavailable = Issue( 

430 rule_id="tdd.red-unavailable", 

431 severity=FindingSeverity.INFO, 

432 message=f"The red step could not be measured ({red['status']})", 

433 category="tests", 

434 ) 

435 return red, [unavailable] 

436 return red, [] 

437 

438 def _red(self, execution: Execution, tests: list[str], sources: list[str]) -> dict[str, Any]: 

439 """Run the changed tests on the workspace with the changed sources reverted to the 

440 baseline: they must fail there.""" 

441 results = self.results 

442 engine = results.engine 

443 contents = results.baseline_contents(execution) 

444 if contents is None: 444 ↛ 445line 444 didn't jump to line 445 because the condition on line 444 was never true

445 return {"status": ResultStatus.NOT_APPLICABLE.value, "reason": "no baseline"} 

446 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{TDD_ID}", version="1") 

447 grants = grants_from_rules( 

448 execution.execution_id, actor, results.s.resolved.effective_capabilities 

449 ) 

450 argv = self.test_command(tests) 

451 workspace = results.s.paths.workspace 

452 with materialized( 

453 workspace, results.s.paths.harness_dir / "tmp", contents, sources 

454 ) as copy: 

455 if copy is None: 455 ↛ 456line 455 didn't jump to line 456 because the condition on line 455 was never true

456 return {"status": ResultStatus.NOT_APPLICABLE.value, "reason": "not revertible"} 

457 outcome = SafeProcessRunner(copy).run( 

458 CommandSpec( 

459 argv=argv, 

460 cwd=copy, 

461 timeout_seconds=float(engine._bounded_timeout(900)), 

462 max_output_bytes=results.project.runtime.max_output_bytes, 

463 ), 

464 actor=actor, 

465 grants=grants, 

466 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)), 

467 ) 

468 return { 

469 "status": outcome.status.value, 

470 "exitCode": outcome.exit_code, 

471 "command": list(argv), 

472 "stdoutRef": results.s.artifacts.put(outcome.stdout, media_type="text/plain").uri, 

473 } 

474 

475 

476__all__ = ["SETUP_FLAG", "TDD_ID", "Engineering", "Strategy", "detect_testing"]