Coverage for src/governed_harness/orchestration/engineering.py: 87%
226 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Standards, principles and testing strategy in the run (#56).
3``AgentResults`` delegates to this class; a project without the ``standards``, ``testing``,
4``architecture`` or ``verification.principles`` settings never reaches it.
6Token cost rules, all deterministic:
8* the implement request carries only the standards cards that apply to the files the call works
9 on, compact, selected once per digest of (packs, paths) and reused from the cache;
10* the review call carries the cards no tool verifies and the principles checklist in its own
11 request: no extra call;
12* the TDD evidence (red, green, refactor) is measured by running the tests, not by asking."""
14from __future__ import annotations
16from datetime import UTC, datetime
17from pathlib import Path
18from typing import TYPE_CHECKING, Any
20from governed_harness.capabilities import grants_from_rules
21from governed_harness.checks import DiffFile, Issue, is_test_path
22from governed_harness.checks.principles import PRINCIPLES_CHECKLIST
23from governed_harness.configuration.engineering import (
24 PrinciplesConfig,
25 StandardsConfig,
26 TestingConfig,
27)
28from governed_harness.domain.enums import ActorType, FindingSeverity, PhaseId, ResultStatus
29from governed_harness.domain.models import Actor, ChangeSet, Execution, PhaseExecution, Task
30from governed_harness.orchestration.engine_types import Strategy as Strategy
31from governed_harness.orchestration.workspace_ops import materialized
32from governed_harness.runtime import CancellationToken, SafeProcessRunner
33from governed_harness.runtime.process_runner import CommandSpec
34from governed_harness.standards import (
35 ProjectStandards,
36 WorkspaceFacts,
37 bdd_frameworks,
38 builtin_pack,
39 cards_payload,
40 checklist_payload,
41 detect_from_facts,
42 project_standards,
43 select_cards,
44 selection_digest,
45)
46from governed_harness.validators import ValidatorOutput
48if TYPE_CHECKING:
49 from governed_harness.orchestration.hosts import ResultsHost
51TDD_ID = "harness.tdd"
52SETUP_FLAG = "projectsetup"
55def _architecture_notes(architecture: dict[str, Any]) -> list[str]:
56 """The sentences the instructions gain for each part of the architecture block."""
57 notes: list[str] = []
58 if "layers" in architecture:
59 notes.append(
60 "Respect the architecture block: a file of one layer may import only the "
61 "layers listed as allowed for it."
62 )
63 if "expert" in architecture:
64 notes.append(
65 "Act as the expert of the chosen architecture style: follow the expert prompt of the "
66 "architecture block, the organization it carries and the style document it points "
67 "to, and keep the implementation aligned with the style."
68 )
69 if "reverseEngineering" in architecture:
70 notes.append(
71 "The architecture block points to the reverse engineering documents of this "
72 "project: read them before searching the code and follow their working agreement; "
73 "do not analyse the whole project again."
74 )
75 return notes
78def detect_testing(workspace: Path, technologies: tuple[str, ...] = ()) -> Strategy:
79 """``bdd`` when the repository has feature files or a BDD framework, ``conventional``
80 when it has tests, else ``unknown`` (asked in INTENT under ``intake.projectSetup``)."""
81 facts = WorkspaceFacts.read(workspace)
82 packs = [builtin_pack(item) for item in detect_from_facts(facts, technologies)]
83 frameworks = bdd_frameworks(packs, facts)
84 if frameworks: 84 ↛ 85line 84 didn't jump to line 85 because the condition on line 84 was never true
85 return Strategy("bdd", "detected", tuple(frameworks))
86 if any(is_test_path(item) for item in facts.files):
87 return Strategy("conventional", "detected")
88 return Strategy("unknown", "unknown")
91class Engineering:
92 def __init__(self, results: ResultsHost) -> None:
93 self.results = results
94 self._standards: ProjectStandards | None = None
95 self._detected: Strategy | None = None
96 """The detected strategy, read once per command (the workspace is walked once)."""
98 # ----- settings -----------------------------------------------------------------------------
99 @property
100 def standards_config(self) -> StandardsConfig | None:
101 return self.results.project.standards
103 @property
104 def principles_config(self) -> PrinciplesConfig | None:
105 verification = self.results.project.verification
106 return verification.principles if verification else None
108 @property
109 def testing_config(self) -> TestingConfig | None:
110 return self.results.project.testing
112 @property
113 def configured(self) -> bool:
114 project = self.results.project
115 return any(
116 (
117 project.standards is not None,
118 project.testing is not None,
119 project.architecture is not None,
120 self.principles_config is not None,
121 bool(project.intake and project.intake.project_setup is not None),
122 )
123 )
125 def technologies(self) -> tuple[str, ...]:
126 return tuple(item.technology for item in self.results.s.resolved.profiles)
128 def standards(self) -> ProjectStandards | None:
129 config = self.standards_config
130 if config is None:
131 return None
132 if self._standards is None:
133 packs = config.packs
134 answered = self.setup_record(self.results.project.project_id).get("standards")
135 if (packs is None or "auto" in packs) and isinstance(answered, list) and answered: 135 ↛ 136line 135 didn't jump to line 136 because the condition on line 135 was never true
136 packs = tuple(str(item) for item in answered)
137 self._standards = project_standards(
138 self.results.s.paths.workspace,
139 packs=packs,
140 overrides=config.overrides_path,
141 disabled=config.disabled or (),
142 technologies=self.technologies(),
143 )
144 return self._standards
146 # ----- project setup answers -------------------------------------------------------------
147 def setup_record(self, project_id: str) -> dict[str, Any]:
148 value = self.results.flag_json(f"{SETUP_FLAG}:{project_id}")
149 return value if isinstance(value, dict) else {}
151 # ----- testing strategy ------------------------------------------------------------------
152 def strategy(self, project_id: str) -> Strategy:
153 """The testing strategy: the configuration, else a person's answer (rule ``P1``), else,
154 only under ``testing.strategy: auto``, what the repository shows. Without the
155 ``testing`` section nothing is detected: the earlier behaviour (``conventional``)."""
156 config = self.testing_config
157 if config is not None and config.strategy not in {None, "auto"}:
158 return Strategy(str(config.strategy), "configuration")
159 answered = self.setup_record(project_id).get("testing")
160 if isinstance(answered, str) and answered in {"tdd", "bdd", "conventional"}:
161 return Strategy(answered, "project-setup")
162 if config is None:
163 return Strategy("conventional", "off")
164 if self._detected is None:
165 self._detected = detect_testing(self.results.s.paths.workspace, self.technologies())
166 return self._detected
168 def features_directory(self) -> str:
169 config = self.testing_config
170 return config.features_path if config is not None else "features"
172 def bdd_command(self) -> tuple[str, ...]:
173 config = self.testing_config
174 if config is not None and config.bdd_command: 174 ↛ 176line 174 didn't jump to line 176 because the condition on line 174 was always true
175 return config.bdd_command
176 for technology in self._languages():
177 runner = builtin_pack(technology).runner
178 if runner.bdd:
179 return runner.bdd
180 return ("python", "-m", "behave")
182 def test_command(self, files: list[str]) -> tuple[str, ...]:
183 """The command that runs ``files`` (TDD red check): ``testing.testCommand`` with the
184 files appended, else the pack runner of the files' language."""
185 config = self.testing_config
186 if config is not None and config.test_command: 186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true
187 return (*config.test_command, *files)
188 for technology in self._languages(files): 188 ↛ 192line 188 didn't jump to line 192 because the loop on line 188 didn't complete
189 runner = builtin_pack(technology).runner
190 if runner.test: 190 ↛ 188line 190 didn't jump to line 188 because the condition on line 190 was always true
191 return (*runner.test, *files) if runner.test_files else runner.test
192 return ("python", "-m", "pytest", "-q", *files)
194 def _languages(self, files: list[str] | None = None) -> list[str]:
195 standards = self.standards()
196 ids = [item.pack_id for item in standards.packs] if standards else []
197 if not ids: 197 ↛ 199line 197 didn't jump to line 199 because the condition on line 197 was always true
198 ids = [item for item in self.technologies() if item in {"python", "node"}]
199 if files: 199 ↛ 215line 199 didn't jump to line 215 because the condition on line 199 was always true
200 by_suffix = {
201 ".py": "python",
202 ".go": "go",
203 ".rs": "rust",
204 ".java": "java",
205 ".kt": "kotlin",
206 ".cs": "csharp",
207 ".php": "php",
208 ".rb": "ruby",
209 ".swift": "swift",
210 }
211 first = [
212 by_suffix[Path(item).suffix] for item in files if Path(item).suffix in by_suffix
213 ]
214 ids = [*first, *ids]
215 return list(dict.fromkeys(item for item in ids if item))
217 # ----- the implement request --------------------------------------------------------------
218 def candidate_paths(self, task: Task, changed: list[str], extra: list[str]) -> list[str]:
219 from governed_harness.agents.context_manifest import referenced_paths
221 workspace = self.results.s.paths.workspace
222 paths = [
223 *[str(item) for item in task.metadata.get("ownedPaths") or []],
224 *referenced_paths(task, workspace),
225 *(patch.path for patch in task.implementation.patches),
226 *changed,
227 *extra,
228 ]
229 return list(dict.fromkeys(item.replace("\\", "/") for item in paths if item))
231 def implement_extra(
232 self,
233 execution: Execution,
234 phase: PhaseExecution,
235 paths: list[str],
236 ) -> tuple[dict[str, Any], str]:
237 """The ``standards`` and ``testing`` blocks of the implement request and the sentence
238 the instructions gain for them."""
239 extra: dict[str, Any] = {}
240 notes: list[str] = []
241 standards = self.standards()
242 config = self.standards_config
243 if standards is not None and config is not None and config.cards_enabled:
244 payload = self._cards(execution, phase, standards, paths, config.card_limit)
245 if payload["cards"]: 245 ↛ 251line 245 didn't jump to line 251 because the condition on line 245 was always true
246 extra["standards"] = payload
247 notes.append(
248 "Follow the standards cards for the files they apply to; listed exceptions "
249 "are allowed."
250 )
251 if self.testing_config is not None or self.setup_record(execution.project_id):
252 strategy = self.strategy(execution.project_id)
253 if strategy.strategy in {"tdd", "bdd"}:
254 extra["testing"] = strategy.as_dict()
255 if strategy.strategy == "tdd":
256 notes.append(
257 "Work test-first (TDD): write or change the tests for each criterion first, "
258 "make them pass with the smallest change, then refactor with every test "
259 "green. The harness runs your new tests on the code before your change and "
260 "they must fail there."
261 )
262 elif strategy.strategy == "bdd":
263 notes.append(
264 "Behaviour-driven (BDD): the approved feature files are frozen; write the "
265 "step definitions and the code that make their scenarios pass."
266 )
267 architecture = self.results.architecture.request_extra()
268 if architecture is not None:
269 extra["architecture"] = architecture
270 notes.extend(_architecture_notes(architecture))
271 return extra, " ".join(notes)
273 def _cards(
274 self,
275 execution: Execution,
276 phase: PhaseExecution,
277 standards: ProjectStandards,
278 paths: list[str],
279 limit: int,
280 ) -> dict[str, Any]:
281 digest = selection_digest(standards, paths, limit)
282 key = f"standardscards:{execution.project_id}:{digest}"
283 cached = self.results.flag_json(key)
284 reused = isinstance(cached, dict) and isinstance(cached.get("cards"), list)
285 if reused:
286 payload: dict[str, Any] = cached
287 else:
288 cards = select_cards(standards, paths, limit)
289 payload = cards_payload(cards, digest, packs=[item.pack_id for item in standards.packs])
290 self.results.set_flag_json(key, payload)
291 self.results.s.events.append(
292 execution.execution_id,
293 "standards.cards.selected",
294 {
295 "digest": digest,
296 "packs": payload["packs"],
297 "cards": [item["id"] for item in payload["cards"]],
298 "paths": len(paths),
299 "reused": reused,
300 },
301 phase_execution_id=phase.phase_execution_id,
302 )
303 return payload
305 # ----- the review call -------------------------------------------------------------------
306 def review_extra(self, change_set: ChangeSet) -> tuple[dict[str, Any], str]:
307 """The checklist the existing review call gains: the cards no tool verifies, for the
308 files the ChangeSet changed, and the principles checklist. No extra call."""
309 checklist: dict[str, Any] = {}
310 standards = self.standards()
311 config = self.standards_config
312 paths = [item.path for item in change_set.files]
313 if standards is not None and config is not None and config.cards_enabled: 313 ↛ 317line 313 didn't jump to line 317 because the condition on line 313 was always true
314 cards = select_cards(standards, paths, config.card_limit, review_only=True)
315 if cards: 315 ↛ 317line 315 didn't jump to line 317 because the condition on line 315 was always true
316 checklist["standards"] = checklist_payload(cards)
317 principles = self.principles_config
318 if principles is not None and principles.enabled and principles.checklist is not False: 318 ↛ 322line 318 didn't jump to line 322 because the condition on line 318 was always true
319 checklist["principles"] = [
320 {"id": rule, "check": text} for rule, text in PRINCIPLES_CHECKLIST
321 ]
322 if not checklist: 322 ↛ 323line 322 didn't jump to line 323 because the condition on line 322 was never true
323 return {}, ""
324 return {"checklist": checklist}, (
325 "Also check each checklist item against the ChangeSet; report a violation as a "
326 "finding whose rule is the item id."
327 )
329 # ----- TDD evidence (VERIFICATION) ---------------------------------------------------------
330 def tdd_validation(
331 self,
332 execution: Execution,
333 change_set: ChangeSet,
334 diff: list[DiffFile],
335 outputs_so_far: list[ValidatorOutput],
336 ) -> ValidatorOutput | None:
337 if not self._tdd_applies(execution, change_set):
338 return None
339 results = self.results
340 started = datetime.now(UTC)
341 tests = [item.path for item in diff if not item.is_deleted and is_test_path(item.path)]
342 sources = [item.path for item in diff if not is_test_path(item.path)]
343 record: dict[str, Any] = {"tests": tests, "sources": sources}
344 record["red"], issues = self._red_step(execution, tests, sources)
345 latest = results.engine._latest_validations(execution.execution_id, change_set.digest)
346 green = [
347 item.validator_id
348 for item in latest
349 if item.mandatory and item.status is not ResultStatus.PASSED
350 ]
351 record["green"] = {"status": "PASSED" if not green else "FAILED", "failing": green}
352 refactor_checks = {"harness.principles", "harness.architecture", "harness.layers"}
353 refactor_blocking = [
354 output.result.validator_id
355 for output in outputs_so_far
356 if output.result.validator_id in refactor_checks
357 and output.result.status is not ResultStatus.PASSED
358 ]
359 record["refactor"] = {
360 "status": "PASSED" if not refactor_blocking and not green else "FAILED",
361 "checks": sorted(refactor_checks),
362 "failing": refactor_blocking,
363 }
364 red = record["red"]
365 ref = results.record_json(
366 execution,
367 PhaseId.VERIFICATION,
368 record,
369 kind="tdd-evidence",
370 summary=(
371 f"TDD: red {red if isinstance(red, str) else red['status']}, "
372 f"green {record['green']['status']}, refactor {record['refactor']['status']}"
373 ),
374 )
375 output = results.verification._output(
376 execution,
377 change_set,
378 TDD_ID,
379 "enforce",
380 issues,
381 "test-driven development (red, green, refactor)",
382 started=started,
383 extra_evidence=(ref,),
384 report={"tdd": record},
385 )
386 return output
388 def _tdd_applies(self, execution: Execution, change_set: ChangeSet) -> bool:
389 """Whether the TDD evidence is checked: the strategy is ``tdd`` and the ChangeSet is
390 not exempt (``friction.changeTypes``, #58: a documentation or configuration change)."""
391 if self.testing_config is None and not self.setup_record(execution.project_id):
392 return False
393 if self.strategy(execution.project_id).strategy != "tdd":
394 return False
395 friction = self.results.engine.friction
396 return not (
397 friction.active and friction.tests_exempt(execution, change_set, "TDD evidence")
398 )
400 def _red_step(
401 self, execution: Execution, tests: list[str], sources: list[str]
402 ) -> tuple[str | dict[str, Any], list[Issue]]:
403 """The red step: the tests the change adds must fail on the code before it."""
404 if sources and not tests: 404 ↛ 405line 404 didn't jump to line 405 because the condition on line 404 was never true
405 no_tests = Issue(
406 rule_id="tdd.no-tests",
407 severity=FindingSeverity.HIGH,
408 message="The change modifies code but adds or changes no test (TDD)",
409 category="tests",
410 recommendation="Write the failing test first, then the code.",
411 )
412 return "NO_TESTS", [no_tests]
413 if not (tests and sources): 413 ↛ 414line 413 didn't jump to line 414 because the condition on line 413 was never true
414 return "NOT_APPLICABLE", []
415 red = self._red(execution, tests, sources)
416 if red["status"] == ResultStatus.PASSED.value:
417 not_red = Issue(
418 rule_id="tdd.not-red",
419 severity=FindingSeverity.HIGH,
420 message=(
421 f"The tests the change adds ({', '.join(tests[:5])}) already pass on "
422 "the code before the change: they were not written first"
423 ),
424 category="tests",
425 recommendation="Write a test that fails without the change.",
426 )
427 return red, [not_red]
428 if red["status"] != ResultStatus.FAILED.value: 428 ↛ 429line 428 didn't jump to line 429 because the condition on line 428 was never true
429 unavailable = Issue(
430 rule_id="tdd.red-unavailable",
431 severity=FindingSeverity.INFO,
432 message=f"The red step could not be measured ({red['status']})",
433 category="tests",
434 )
435 return red, [unavailable]
436 return red, []
438 def _red(self, execution: Execution, tests: list[str], sources: list[str]) -> dict[str, Any]:
439 """Run the changed tests on the workspace with the changed sources reverted to the
440 baseline: they must fail there."""
441 results = self.results
442 engine = results.engine
443 contents = results.baseline_contents(execution)
444 if contents is None: 444 ↛ 445line 444 didn't jump to line 445 because the condition on line 444 was never true
445 return {"status": ResultStatus.NOT_APPLICABLE.value, "reason": "no baseline"}
446 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{TDD_ID}", version="1")
447 grants = grants_from_rules(
448 execution.execution_id, actor, results.s.resolved.effective_capabilities
449 )
450 argv = self.test_command(tests)
451 workspace = results.s.paths.workspace
452 with materialized(
453 workspace, results.s.paths.harness_dir / "tmp", contents, sources
454 ) as copy:
455 if copy is None: 455 ↛ 456line 455 didn't jump to line 456 because the condition on line 455 was never true
456 return {"status": ResultStatus.NOT_APPLICABLE.value, "reason": "not revertible"}
457 outcome = SafeProcessRunner(copy).run(
458 CommandSpec(
459 argv=argv,
460 cwd=copy,
461 timeout_seconds=float(engine._bounded_timeout(900)),
462 max_output_bytes=results.project.runtime.max_output_bytes,
463 ),
464 actor=actor,
465 grants=grants,
466 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)),
467 )
468 return {
469 "status": outcome.status.value,
470 "exitCode": outcome.exit_code,
471 "command": list(argv),
472 "stdoutRef": results.s.artifacts.put(outcome.stdout, media_type="text/plain").uri,
473 }
476__all__ = ["SETUP_FLAG", "TDD_ID", "Engineering", "Strategy", "detect_testing"]