Coverage for src/governed_harness/orchestration/verification_checks.py: 87%
427 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Deterministic validators of the agent-results settings in VERIFICATION (#40, #52).
3Each enabled check becomes one validation result for the current ChangeSet, with its issues as
4findings. Under ``enforce`` a check is mandatory and fails when it finds something the gate
5blocks on, so the correction loop (``runtime.verificationCorrections``) sends the agent the
6finding with its location; under ``warn`` its findings are ``LOW`` and the check never fails.
7The checks read the unredacted diff in memory only (as the independent review does); what is
8stored is the redacted record of each finding."""
10from __future__ import annotations
12import json
13import re
14from collections.abc import Callable, Mapping, Sequence
15from dataclasses import dataclass, replace
16from datetime import UTC, datetime
17from pathlib import Path
18from typing import TYPE_CHECKING, Any
20from governed_harness.capabilities import grants_from_rules
21from governed_harness.checks import DiffFile, Issue, is_test_path, parse_unified_diff
22from governed_harness.checks.architecture import ArchitectureLimits, check_architecture
23from governed_harness.checks.constraints import check_constraints, constraints_from_task
24from governed_harness.checks.interface import check_interface, declared_names
25from governed_harness.checks.layers import LayerRules, check_layers, known_modules
26from governed_harness.checks.principles import (
27 PrincipleLimits,
28 check_principles,
29 is_source,
30 source_corpus,
31)
32from governed_harness.checks.risk import detect_risk_factors
33from governed_harness.checks.sarif import read_sarif
34from governed_harness.checks.secrets import scan_secrets
35from governed_harness.checks.security_patterns import check_security_patterns
36from governed_harness.checks.suite_quality import (
37 assertion_free_tests,
38 changed_line_coverage,
39 pytest_node_ids,
40 untested_interface_methods,
41)
42from governed_harness.checks.weakened import check_weakened_controls
43from governed_harness.configuration.agent_results import DEFAULT_RISK_ACTIONS, RISK_FACTORS
44from governed_harness.configuration.models import ValidatorDefinition, VerificationConfig
45from governed_harness.domain.enums import (
46 ActorType,
47 FindingSeverity,
48 ResultStatus,
49 ValidationKind,
50)
51from governed_harness.domain.models import (
52 Actor,
53 ChangeSet,
54 Execution,
55 Finding,
56 Task,
57)
58from governed_harness.runtime import CancellationToken
59from governed_harness.runtime.process_runner import CommandSpec
60from governed_harness.validators import CommandValidator, ValidationContext, ValidatorOutput
62if TYPE_CHECKING:
63 from governed_harness.orchestration.hosts import ResultsHost
65INTERFACE_ID = "harness.interface"
66ARCHITECTURE_ID = "harness.architecture"
67SECURITY_ID = "harness.security-patterns"
68CONSTRAINTS_ID = "harness.constraints"
69WEAKENED_ID = "harness.weakened-controls"
70TEST_QUALITY_ID = "harness.test-quality"
71SECRETS_ID = "harness.secrets"
72SARIF_ID = "harness.sarif"
73RISK_ID = "harness.risk-factors"
74CHANGE_REQUESTS_ID = "harness.change-requests"
75RISK_ACK_FLAG = "riskack"
76PRINCIPLES_ID = "harness.principles"
77LAYERS_ID = "harness.layers"
79_SARIF_SEVERITY = {
80 "error": FindingSeverity.HIGH,
81 "warning": FindingSeverity.MEDIUM,
82 "note": FindingSeverity.LOW,
83}
84_RISK_SEVERITY = {
85 "block": FindingSeverity.HIGH,
86 "acknowledge": FindingSeverity.MEDIUM,
87 "inform": FindingSeverity.LOW,
88}
89_SEVERITY_RANK = {
90 FindingSeverity.INFO: 0,
91 FindingSeverity.LOW: 1,
92 FindingSeverity.MEDIUM: 2,
93 FindingSeverity.HIGH: 3,
94 FindingSeverity.CRITICAL: 4,
95}
98def declared_interfaces(task: Task) -> list[tuple[str, str]]:
99 """``(stub, module)`` pairs a task declares in ``metadata.interface``: an object with
100 ``stub`` and ``module`` paths, or a list of them."""
101 raw = task.metadata.get("interface")
102 if isinstance(raw, list): 102 ↛ 103line 102 didn't jump to line 103 because the condition on line 102 was never true
103 entries = raw
104 elif raw:
105 entries = [raw]
106 else:
107 entries = []
108 pairs: list[tuple[str, str]] = []
109 for entry in entries:
110 if isinstance(entry, dict) and entry.get("stub") and entry.get("module"): 110 ↛ 109line 110 didn't jump to line 109 because the condition on line 110 was always true
111 pairs.append((str(entry["stub"]), str(entry["module"])))
112 return pairs
115class VerificationChecks:
116 def __init__(self, results: ResultsHost) -> None:
117 self.results = results
119 @property
120 def config(self) -> Any:
121 return self.results.project.verification
123 def blocking(self) -> set[FindingSeverity]:
124 names = self.results.s.resolved.effective_policies.get(
125 "findingBlockSeverities", ["HIGH", "CRITICAL"]
126 )
127 return {FindingSeverity(str(name)) for name in names}
129 # ----- entry point ----------------------------------------------------------------------
130 def run(self, execution: Execution, change_set: ChangeSet) -> list[ValidatorOutput]:
131 engine = self.results.engine
132 raw = engine._compute_owned_diff(execution).unified_diff.decode("utf-8", "replace")
133 diff = parse_unified_diff(raw)
134 outputs = self._run_outputs(execution, change_set, diff)
135 config = self.config
136 layers = self.results.architecture.rules() if self.results.engineering.configured else None
137 if config is None and layers is None: 137 ↛ 138line 137 didn't jump to line 138 because the condition on line 137 was never true
138 tdd = self.results.engineering.tdd_validation(execution, change_set, diff, outputs)
139 return [*outputs, tdd] if tdd is not None else outputs
140 task = engine.run_task(execution)
141 files = current_files(self.results.s.paths.workspace, diff)
142 architecture = self.results.project.architecture
143 for item in pure_checks(
144 config or _NO_CHECKS,
145 task,
146 diff,
147 files,
148 self.results.s.paths.workspace,
149 self.results.baseline_text(execution),
150 layers=layers,
151 layer_policy=architecture.policy if architecture is not None else "enforce",
152 layer_differential=bool(architecture is not None and architecture.differential),
153 ):
154 outputs.append(
155 self._output(
156 execution,
157 change_set,
158 item.validator_id,
159 item.policy,
160 item.issues,
161 item.label,
162 started=item.started,
163 )
164 )
165 if self.results.engineering.configured:
166 # testing.strategy: tdd (#56): red, green and refactor evidence of the ChangeSet.
167 tdd = self.results.engineering.tdd_validation(execution, change_set, diff, outputs)
168 if tdd is not None:
169 outputs.append(tdd)
170 if config is None: 170 ↛ 171line 170 didn't jump to line 171 because the condition on line 170 was never true
171 return outputs
172 return [*outputs, *self._configured(execution, change_set, config, task, diff, files)]
174 def _run_outputs(
175 self, execution: Execution, change_set: ChangeSet, diff: list[DiffFile]
176 ) -> list[ValidatorOutput]:
177 """The run's own checks: owned paths, the reproduced correction, the acceptance tests
178 and the structured change requests."""
179 outputs = [
180 item
181 for item in (
182 self.results.stop_line.owned_paths_output(execution, change_set),
183 self.results.corrections.validation(execution, change_set),
184 self.results.acceptance.validation(execution, change_set),
185 )
186 if item is not None
187 ]
188 requests = self.results.change_requests(execution)
189 if requests:
190 outputs.append(self._change_requests(execution, change_set, requests, diff))
191 return outputs
193 def _quality_exempt(self, execution: Execution, change_set: ChangeSet) -> bool:
194 """Whether the fast lane exempts the ChangeSet from the test quality check."""
195 friction = self.results.engine.friction
196 return bool(
197 friction.active and friction.tests_exempt(execution, change_set, "test quality")
198 )
200 def _configured(
201 self,
202 execution: Execution,
203 change_set: ChangeSet,
204 config: VerificationConfig,
205 task: Task,
206 diff: list[DiffFile],
207 files: dict[str, str],
208 ) -> list[ValidatorOutput]:
209 """Test quality, invariants, SARIF and risk factors, as ``verification`` asks."""
210 outputs: list[ValidatorOutput] = []
211 if config.test_quality is not None and not self._quality_exempt(execution, change_set):
212 outputs.append(self._test_quality(execution, change_set, task, diff, files))
213 if config.invariants:
214 outputs.extend(self._invariants(execution, change_set))
215 if config.sarif:
216 outputs.append(self._sarif(execution, change_set))
217 if config.risk_factors is not None:
218 outputs.append(self._risk(execution, change_set, task, diff))
219 return outputs
221 # ----- helpers ---------------------------------------------------------------------------
222 def _output(
223 self,
224 execution: Execution,
225 change_set: ChangeSet,
226 validator_id: str,
227 policy: str,
228 issues: list[Issue],
229 label: str,
230 *,
231 started: datetime,
232 extra_evidence: tuple[str, ...] = (),
233 mandatory: bool | None = None,
234 report: dict[str, Any] | None = None,
235 ) -> ValidatorOutput:
236 results = self.results
237 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{validator_id}", version="1")
238 warn = policy == "warn"
239 findings: list[Finding] = []
240 report_ref = results.s.artifacts.put_json(
241 {
242 "validatorId": validator_id,
243 "changeSetDigest": change_set.digest,
244 "policy": policy,
245 "issues": [
246 {
247 "ruleId": item.rule_id,
248 "severity": item.severity,
249 "path": item.path,
250 "line": item.line,
251 "message": item.message,
252 }
253 for item in issues
254 ],
255 **(report or {}),
256 },
257 metadata={"kind": "verification-check", "validatorId": validator_id},
258 )
259 refs = (report_ref.uri, *extra_evidence)
260 for issue in issues:
261 severity = FindingSeverity.LOW if warn and _rank(issue.severity) > 1 else issue.severity
262 findings.append(
263 results.record_finding(
264 execution,
265 validator_id=validator_id,
266 rule_id=issue.rule_id,
267 category=issue.category,
268 severity=severity,
269 message=issue.message,
270 path=issue.path,
271 line=issue.line,
272 evidence_refs=(report_ref.uri,),
273 recommendation=issue.recommendation,
274 introduced=True,
275 actor=actor,
276 )
277 )
278 is_mandatory = (policy == "enforce") if mandatory is None else mandatory
279 blocking = [item for item in findings if item.severity in self.blocking()]
280 failed = is_mandatory and bool(blocking)
281 result = results.record_validation(
282 execution,
283 validator_id=validator_id,
284 digest=change_set.digest,
285 status=ResultStatus.FAILED if failed else ResultStatus.PASSED,
286 kind=ValidationKind.VALIDATION_FAILURE if failed else ValidationKind.SUCCESS,
287 mandatory=is_mandatory,
288 summary=(
289 f"{label}: {len(findings)} finding(s), {len(blocking)} blocking"
290 if findings
291 else f"{label}: no finding"
292 ),
293 findings=tuple(findings),
294 evidence_refs=refs,
295 started_at=started,
296 )
297 return ValidatorOutput(result, ())
299 # ----- test quality (#52) ----------------------------------------------------------------
300 def _test_quality(
301 self,
302 execution: Execution,
303 change_set: ChangeSet,
304 task: Task,
305 diff: Sequence[DiffFile],
306 files: Mapping[str, str],
307 ) -> ValidatorOutput:
308 config = self.config.test_quality
309 severity = config.severity or FindingSeverity.MEDIUM
310 started = datetime.now(UTC)
311 issues: list[Issue] = []
312 report: dict[str, Any] = {}
313 evidence: list[str] = []
314 tests = {path: text for path, text in files.items() if is_test_path(path)}
315 if config.assertions: 315 ↛ 317line 315 didn't jump to line 317 because the condition on line 315 was always true
316 issues.extend(assertion_free_tests(tests, severity=severity))
317 if config.interface_tests and declared_interfaces(task): 317 ↛ 318line 317 didn't jump to line 318 because the condition on line 317 was never true
318 issues.extend(self._untested_interfaces(task, severity))
319 if config.diff_coverage is not None and self._python():
320 issues.extend(
321 self._diff_coverage(
322 execution, change_set, diff, config.diff_coverage, severity, report, evidence
323 )
324 )
325 if config.flaky_reruns and self._python():
326 flaky, refs = self._flaky(execution, change_set, config.flaky_reruns)
327 evidence.extend(refs)
328 issues.extend(flaky)
329 policy = "enforce" if severity in self.blocking() else "warn-keep"
330 return self._output(
331 execution,
332 change_set,
333 TEST_QUALITY_ID,
334 policy,
335 issues,
336 "test quality",
337 started=started,
338 extra_evidence=tuple(evidence),
339 mandatory=policy == "enforce",
340 report=report,
341 )
343 def _untested_interfaces(self, task: Task, severity: FindingSeverity) -> list[Issue]:
344 """The methods of the task's declared interfaces that no test exercises."""
345 names: list[str] = []
346 for stub, _module in declared_interfaces(task):
347 text = _read(self.results.s.paths.workspace, stub)
348 if text is not None:
349 names.extend(declared_names(text))
350 return untested_interface_methods(names, self._all_tests(), severity=severity)
352 def _diff_coverage(
353 self,
354 execution: Execution,
355 change_set: ChangeSet,
356 diff: Sequence[DiffFile],
357 minimum_percent: float,
358 severity: FindingSeverity,
359 report: dict[str, Any],
360 evidence: list[str],
361 ) -> list[Issue]:
362 """The coverage of the changed lines against the minimum; records the percentage in
363 ``report`` and the coverage evidence in ``evidence``."""
364 coverage, refs = self._coverage(execution, change_set)
365 evidence.extend(refs)
366 if coverage is None: 366 ↛ 367line 366 didn't jump to line 367 because the condition on line 366 was never true
367 report["diffCoverage"] = None
368 return [
369 Issue(
370 rule_id="tests.coverage-unavailable",
371 severity=FindingSeverity.INFO,
372 message="Coverage of the changed lines could not be measured (the "
373 "coverage package or the tests did not run)",
374 category="tests",
375 )
376 ]
377 percent, found = changed_line_coverage(
378 diff, coverage, minimum_percent=minimum_percent, severity=severity
379 )
380 report["diffCoverage"] = percent
381 return found
383 def _python(self) -> bool:
384 return any(item.technology == "python" for item in self.results.s.resolved.profiles)
386 def _all_tests(self) -> dict[str, str]:
387 workspace = self.results.s.paths.workspace
388 found: dict[str, str] = {}
389 for path in sorted(workspace.rglob("*.py")):
390 relative = path.relative_to(workspace).as_posix()
391 if any( 391 ↛ 395line 391 didn't jump to line 395 because the condition on line 391 was never true
392 part.startswith(".") or part in {"venv", "node_modules"}
393 for part in Path(relative).parts
394 ):
395 continue
396 if is_test_path(relative):
397 try:
398 found[relative] = path.read_text(encoding="utf-8")
399 except (OSError, UnicodeDecodeError):
400 continue
401 return found
403 def _context(
404 self,
405 execution: Execution,
406 change_set: ChangeSet,
407 validator_id: str,
408 command: tuple[str, ...],
409 ) -> ValidationContext:
410 engine = self.results.engine
411 actor = Actor(actor_type=ActorType.TOOL, actor_id=f"validator.{validator_id}", version="1")
412 return ValidationContext(
413 execution_id=execution.execution_id,
414 workspace=self.results.s.paths.workspace,
415 task=engine.run_task(execution),
416 change_set=change_set,
417 definition=engine._bounded_definition(
418 ValidatorDefinition(id=validator_id, command=command, mandatory=True)
419 ),
420 grants=grants_from_rules(
421 execution.execution_id, actor, self.results.s.resolved.effective_capabilities
422 ),
423 artifact_store=self.results.s.artifacts,
424 process_runner=engine._runner(execution),
425 provenance=engine._provenance(execution).model_copy(update={"actor": actor}),
426 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)),
427 max_output_bytes=self.results.project.runtime.max_output_bytes,
428 )
430 def _coverage(
431 self, execution: Execution, change_set: ChangeSet
432 ) -> tuple[dict[str, Any] | None, list[str]]:
433 """Run the tests under coverage.py and read its JSON report (Python projects)."""
434 data_dir = self.results.s.paths.harness_dir / "coverage"
435 data_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
436 data = data_dir / f"{execution.execution_id}.diffcov"
437 report = data_dir / f"{execution.execution_id}.diffcov.json"
438 report.unlink(missing_ok=True)
439 refs: list[str] = []
440 steps = (
441 ("python", "-m", "coverage", "run", f"--data-file={data}", "-m", "pytest", "-q"),
442 ("python", "-m", "coverage", "json", f"--data-file={data}", "-o", str(report)),
443 )
444 for step in steps:
445 context = self._context(execution, change_set, TEST_QUALITY_ID, step)
446 output = CommandValidator(TEST_QUALITY_ID).execute(context)
447 for tool in output.tool_invocations:
448 self.results.engine._save_tool(execution, tool)
449 refs.extend(output.result.evidence_refs[:1])
450 if output.result.status is not ResultStatus.PASSED and step[3] == "json": 450 ↛ 451line 450 didn't jump to line 451 because the condition on line 450 was never true
451 return None, refs
452 try:
453 value = json.loads(report.read_text(encoding="utf-8"))
454 except (OSError, ValueError):
455 return None, refs
456 return (value if isinstance(value, dict) else None), refs
458 def _flaky(
459 self, execution: Execution, change_set: ChangeSet, reruns: int
460 ) -> tuple[list[Issue], list[str]]:
461 """Run the test suite again with another hash seed and the test files in reverse order;
462 a failure the first run did not have marks the suite as flaky."""
463 engine = self.results.engine
464 latest = {
465 item.validator_id: item
466 for item in engine._latest_validations(execution.execution_id, change_set.digest)
467 }
468 first = latest.get("python.pytest")
469 if first is None or first.status is not ResultStatus.PASSED: 469 ↛ 470line 469 didn't jump to line 470 because the condition on line 469 was never true
470 return [], []
471 tests = sorted(self._all_tests(), reverse=True)
472 refs: list[str] = []
473 issues: list[Issue] = []
474 actor = Actor(
475 actor_type=ActorType.TOOL, actor_id=f"validator.{TEST_QUALITY_ID}", version="1"
476 )
477 grants = grants_from_rules(
478 execution.execution_id, actor, self.results.s.resolved.effective_capabilities
479 )
480 for attempt in range(1, reruns + 1):
481 seed = str(1000 + attempt)
482 spec = CommandSpec(
483 argv=("python", "-m", "pytest", "-q", "-p", "no:randomly", *tests),
484 cwd=self.results.s.paths.workspace,
485 timeout_seconds=float(engine._bounded_timeout(900)),
486 max_output_bytes=self.results.project.runtime.max_output_bytes,
487 allowed_environment=("PYTHONHASHSEED",),
488 )
489 process = engine._runner(execution).run(
490 spec,
491 actor=actor,
492 grants=grants,
493 extra_env={"PYTHONHASHSEED": seed},
494 cancellation=CancellationToken(lambda: engine.is_cancelled(execution.execution_id)),
495 )
496 ref = self.results.s.artifacts.put_json(
497 {
498 "attempt": attempt,
499 "pythonHashSeed": seed,
500 "order": "reversed",
501 "exitCode": process.exit_code,
502 "status": process.status,
503 "stdoutRef": self.results.s.artifacts.put(
504 process.stdout, media_type="text/plain"
505 ).uri,
506 },
507 metadata={"kind": "flaky-rerun", "executionId": execution.execution_id},
508 )
509 refs.append(ref.uri)
510 if process.status is not ResultStatus.PASSED:
511 failing = pytest_node_ids(process.stdout.decode("utf-8", "replace"))
512 shown = ", ".join(failing[:10]) or "see the stored output"
513 issues.append(
514 Issue(
515 rule_id="tests.flaky",
516 severity=FindingSeverity.HIGH,
517 message=(
518 f"The tests passed once and failed when run again with "
519 f"PYTHONHASHSEED={seed} in reverse file order: {shown}"
520 ),
521 category="tests",
522 recommendation=(
523 "Remove the dependence on order, hash seed, time or randomness."
524 ),
525 )
526 )
527 break
528 return issues, refs
530 # ----- structured change requests (#52) --------------------------------------------------
531 def _change_requests(
532 self,
533 execution: Execution,
534 change_set: ChangeSet,
535 items: list[dict[str, Any]],
536 diff: Sequence[DiffFile],
537 ) -> ValidatorOutput:
538 """Every blocking item of a REQUEST_CHANGES is a temporary criterion: ``test:`` items
539 must pass, ``absent:`` patterns must not be added, ``text`` items are reminders."""
540 started = datetime.now(UTC)
541 issues: list[Issue] = []
542 evidence: list[str] = []
543 added = [(item.path, line.number, line.text) for item in diff for line in item.added]
544 for item in items:
545 condition = str(item.get("condition", "text"))
546 kind, _, value = condition.partition(":")
547 label = f"{item.get('itemId')} ({item.get('description')})"
548 if kind == "test": 548 ↛ 549line 548 didn't jump to line 549 because the condition on line 548 was never true
549 issues.extend(self._requested_test(execution, change_set, label, value, evidence))
550 elif kind == "absent":
551 issues.extend(_requested_absence(label, value, added))
552 else:
553 issues.append(
554 Issue(
555 rule_id="change-request.unverified",
556 severity=FindingSeverity.INFO,
557 message=f"{label}: not checked by a tool; the reviewer verifies it",
558 category="change-request",
559 )
560 )
561 return self._output(
562 execution,
563 change_set,
564 CHANGE_REQUESTS_ID,
565 "enforce",
566 issues,
567 "change requests",
568 started=started,
569 extra_evidence=tuple(evidence),
570 report={"items": items},
571 )
573 def _requested_test(
574 self,
575 execution: Execution,
576 change_set: ChangeSet,
577 label: str,
578 value: str,
579 evidence: list[str],
580 ) -> list[Issue]:
581 """Run the test a ``test:`` item names; an issue unless it passes."""
582 context = self._context(
583 execution,
584 change_set,
585 CHANGE_REQUESTS_ID,
586 ("python", "-m", "pytest", "-q", value.strip()),
587 )
588 output = CommandValidator(CHANGE_REQUESTS_ID).execute(context)
589 for tool in output.tool_invocations:
590 self.results.engine._save_tool(execution, tool)
591 evidence.extend(output.result.evidence_refs[:1])
592 if output.result.status is ResultStatus.PASSED:
593 return []
594 return [
595 Issue(
596 rule_id="change-request.unmet",
597 severity=FindingSeverity.HIGH,
598 message=f"{label}: the test {value.strip()} does not pass",
599 category="change-request",
600 recommendation="Make the requested test pass.",
601 )
602 ]
604 # ----- invariants (#52) ------------------------------------------------------------------
605 def _invariants(self, execution: Execution, change_set: ChangeSet) -> list[ValidatorOutput]:
606 outputs: list[ValidatorOutput] = []
607 for invariant in self.config.invariants or ():
608 validator_id = f"invariant.{invariant.invariant_id}"
609 context = self._context(execution, change_set, validator_id, invariant.command)
610 output = CommandValidator(validator_id).execute(context)
611 self.results.engine._save_validator_output(execution, output)
612 outputs.append(output)
613 return outputs
615 # ----- SARIF (#52) -----------------------------------------------------------------------
616 def _sarif(self, execution: Execution, change_set: ChangeSet) -> ValidatorOutput:
617 workspace = self.results.s.paths.workspace
618 started = datetime.now(UTC)
619 issues: list[Issue] = []
620 tools: list[dict[str, Any]] = []
621 missing: list[str] = []
622 for source in self.config.sarif or ():
623 path = workspace / source.path
624 try:
625 text = path.read_text(encoding="utf-8")
626 except (OSError, UnicodeDecodeError):
627 if source.required: 627 ↛ 629line 627 didn't jump to line 629 because the condition on line 627 was always true
628 missing.append(source.path)
629 continue
630 results = read_sarif(text, workspace)
631 versions = _sarif_versions(results)
632 tools.append({"path": source.path, "results": len(results), "versions": versions})
633 for item in results:
634 issues.append(_sarif_issue(source, item))
635 for path in missing:
636 issues.append(
637 Issue(
638 rule_id="sarif.missing-report",
639 severity=FindingSeverity.HIGH,
640 message=f"The required SARIF report {path} was not found in the workspace",
641 path=path,
642 category="external-scanner",
643 recommendation="Run the scanner before verification or mark it optional.",
644 )
645 )
646 return self._output(
647 execution,
648 change_set,
649 SARIF_ID,
650 "enforce",
651 issues,
652 "external scanners (SARIF)",
653 started=started,
654 report={"reports": tools, "missing": missing},
655 )
657 # ----- risk factors (#52) ----------------------------------------------------------------
658 def risk_actions(self) -> dict[str, str]:
659 configured = self.config.risk_factors if self.config else None
660 actions: dict[str, str] = dict(DEFAULT_RISK_ACTIONS)
661 if configured: 661 ↛ 663line 661 didn't jump to line 663 because the condition on line 661 was always true
662 actions.update(configured)
663 return actions
665 def _risk(
666 self, execution: Execution, change_set: ChangeSet, task: Task, diff: Sequence[DiffFile]
667 ) -> ValidatorOutput:
668 started = datetime.now(UTC)
669 actions = self.risk_actions()
670 interfaces = [module for _stub, module in declared_interfaces(task)] + [
671 stub for stub, _module in declared_interfaces(task)
672 ]
673 signals = detect_risk_factors(diff, interface_paths=interfaces)
674 issues: list[Issue] = []
675 acknowledge: list[str] = []
676 for signal in signals:
677 action = actions.get(signal.factor, "inform")
678 if action == "off": 678 ↛ 679line 678 didn't jump to line 679 because the condition on line 678 was never true
679 continue
680 if action == "acknowledge" and signal.factor not in acknowledge: 680 ↛ 682line 680 didn't jump to line 682 because the condition on line 680 was always true
681 acknowledge.append(signal.factor)
682 issues.append(
683 Issue(
684 rule_id=f"risk.{signal.factor}",
685 severity=_RISK_SEVERITY[action],
686 message=f"{signal.message} (risk factor {signal.factor}: {action})",
687 path=signal.path,
688 line=signal.line,
689 category="risk",
690 recommendation={
691 "block": "A person must approve an exception for this change.",
692 "acknowledge": (
693 "A person must acknowledge this risk factor in the decision "
694 f"(--acknowledge-risk {signal.factor})."
695 ),
696 "inform": "Shown in the decision brief.",
697 }[action],
698 )
699 )
700 self.results.set_flag_json(
701 f"{RISK_ACK_FLAG}:{execution.execution_id}:{change_set.digest}",
702 [item for item in RISK_FACTORS if item in acknowledge],
703 )
704 return self._output(
705 execution,
706 change_set,
707 RISK_ID,
708 "enforce",
709 issues,
710 "risk factors",
711 started=started,
712 mandatory=False,
713 report={"actions": actions, "acknowledgementRequired": acknowledge},
714 )
717_NO_CHECKS = VerificationConfig()
718"""A verification section with every check off: the layer rules of ``architecture`` run on
719their own."""
722def _rank(severity: FindingSeverity) -> int:
723 return _SEVERITY_RANK[severity]
726def _subject(message: str) -> str:
727 """What a limit message is about: the module or function (and, for a forbidden import,
728 the module it imports), without the measured value."""
729 words = message.split()
730 if " imports " in message: 730 ↛ 731line 730 didn't jump to line 731 because the condition on line 730 was never true
731 return " ".join(words[:3])
732 return words[0] if words else ""
735def _read(workspace: Path, relative: str) -> str | None:
736 target = (workspace / relative).resolve()
737 try:
738 target.relative_to(workspace.resolve())
739 return target.read_text(encoding="utf-8")
740 except (ValueError, OSError): # UnicodeDecodeError is a ValueError
741 return None
744def _task_text(task: Task) -> str:
745 return "\n".join(
746 [
747 task.title,
748 task.intent,
749 *task.constraints,
750 *(item.text for item in task.requirements),
751 *(item.text for item in task.acceptance_criteria),
752 *(item.verification_hint or "" for item in task.acceptance_criteria),
753 ]
754 )
757@dataclass(frozen=True)
758class CheckResult:
759 validator_id: str
760 policy: str
761 issues: list[Issue]
762 label: str
763 started: datetime
766def current_files(workspace: Path, diff: Sequence[DiffFile]) -> dict[str, str]:
767 """The current text of every file the diff adds or changes."""
768 files: dict[str, str] = {}
769 for item in diff:
770 if item.is_deleted: 770 ↛ 771line 770 didn't jump to line 771 because the condition on line 770 was never true
771 continue
772 try:
773 files[item.path] = (workspace / item.path).read_text(encoding="utf-8")
774 except (OSError, UnicodeDecodeError):
775 continue
776 return files
779def pure_checks(
780 config: Any,
781 task: Task,
782 diff: Sequence[DiffFile],
783 files: Mapping[str, str],
784 workspace: Path,
785 baseline_text: Callable[[str], str | None] | None,
786 *,
787 layers: LayerRules | None = None,
788 layer_policy: str = "enforce",
789 layer_differential: bool = False,
790) -> list[CheckResult]:
791 """The checks that read only the diff and the files (VERIFICATION and ``harness check``):
792 interface, architecture, security patterns, constraints, weakened controls and secrets;
793 since #56 the engineering principles and the architecture's layer rules."""
794 planned = [
795 *_planned_design_checks(
796 config,
797 task,
798 diff,
799 files,
800 workspace,
801 baseline_text,
802 layers=layers,
803 layer_policy=layer_policy,
804 layer_differential=layer_differential,
805 ),
806 *_planned_diff_checks(config, task, diff, files),
807 ]
808 results: list[CheckResult] = []
809 for validator_id, policy, check in planned:
810 started = datetime.now(UTC)
811 issues, label = check()
812 results.append(CheckResult(validator_id, policy, issues, label, started))
813 return results
816_PlannedCheck = tuple[str, str, Callable[[], tuple[list[Issue], str]]]
819def _planned_design_checks(
820 config: Any,
821 task: Task,
822 diff: Sequence[DiffFile],
823 files: Mapping[str, str],
824 workspace: Path,
825 baseline_text: Callable[[str], str | None] | None,
826 *,
827 layers: LayerRules | None,
828 layer_policy: str,
829 layer_differential: bool,
830) -> list[_PlannedCheck]:
831 """The configured design checks of pure_checks: engineering principles, layer rules,
832 declared interfaces and the architecture rules, in that order."""
833 planned: list[_PlannedCheck] = []
834 principles = getattr(config, "principles", None)
835 if principles is not None and principles.enabled:
836 planned.append(
837 (
838 PRINCIPLES_ID,
839 principles.mode,
840 lambda: principle_issues(config, diff, files, workspace, baseline_text),
841 )
842 )
843 if layers is not None and layers.layers:
844 layer_severity = FindingSeverity.HIGH if layer_policy == "enforce" else FindingSeverity.LOW
845 planned.append(
846 (
847 LAYERS_ID,
848 layer_policy,
849 lambda: (
850 check_layers(
851 files,
852 layers,
853 severity=layer_severity,
854 baseline=baseline_text if layer_differential else None,
855 module_exists=known_modules(files, workspace),
856 ),
857 "architecture layer rules",
858 ),
859 )
860 )
861 if config.interface not in {None, "off"} and declared_interfaces(task):
862 planned.append(
863 (
864 INTERFACE_ID,
865 config.interface,
866 lambda: interface_issues(workspace, task, config.interface),
867 )
868 )
869 if config.architecture is not None:
870 planned.append(
871 (
872 ARCHITECTURE_ID,
873 "enforce",
874 lambda: architecture_issues(config.architecture, files, baseline_text),
875 )
876 )
877 return planned
880def _planned_diff_checks(
881 config: Any, task: Task, diff: Sequence[DiffFile], files: Mapping[str, str]
882) -> list[_PlannedCheck]:
883 """The configured checks of the diff in pure_checks: security patterns, constraints,
884 weakened controls and secrets in context, in that order."""
885 planned: list[_PlannedCheck] = []
886 if config.security_patterns:
887 planned.append(
888 (
889 SECURITY_ID,
890 "enforce",
891 lambda: (check_security_patterns(diff, files), "security patterns"),
892 )
893 )
894 if config.constraints not in {None, "off"}:
895 planned.append(
896 (
897 CONSTRAINTS_ID,
898 config.constraints,
899 lambda: constraint_issues(task, files, config.constraints),
900 )
901 )
902 if config.weakened_controls not in {None, "off"}:
903 planned.append(
904 (
905 WEAKENED_ID,
906 config.weakened_controls,
907 lambda: (
908 check_weakened_controls(diff, config.weakened_controls),
909 "weakened controls",
910 ),
911 )
912 )
913 if config.secrets == "context":
914 planned.append(
915 (
916 SECRETS_ID,
917 "enforce",
918 lambda: (scan_secrets(diff, task_text=_task_text(task)), "secrets in context"),
919 )
920 )
921 return planned
924def _requested_absence(label: str, value: str, added: list[tuple[str, int, str]]) -> list[Issue]:
925 """An issue when the ChangeSet still adds what an ``absent:`` item forbids."""
926 try:
927 pattern = re.compile(value)
928 except re.error:
929 pattern = re.compile(re.escape(value))
930 hit = next(
931 ((path, number) for path, number, text in added if pattern.search(text)),
932 None,
933 )
934 if hit is None: 934 ↛ 935line 934 didn't jump to line 935 because the condition on line 934 was never true
935 return []
936 return [
937 Issue(
938 rule_id="change-request.unmet",
939 severity=FindingSeverity.HIGH,
940 message=f"{label}: {value!r} is still added by the ChangeSet",
941 path=hit[0],
942 line=hit[1],
943 category="change-request",
944 recommendation="Remove what the reviewer asked to remove.",
945 )
946 ]
949def _sarif_versions(results: Sequence[Any]) -> list[str]:
950 """The tool versions (and rule versions) of the results of one SARIF report."""
951 return sorted(
952 {
953 f"{item.tool}@{item.tool_version or '?'}"
954 + (f" rule {item.rule_id}@{item.rule_version}" if item.rule_version else "")
955 for item in results
956 }
957 )
960def _sarif_issue(source: Any, item: Any) -> Issue:
961 """The issue of one result of an external scanner's SARIF report."""
962 return Issue(
963 rule_id=f"sarif.{source.tool or item.tool}.{item.rule_id}",
964 severity=_SARIF_SEVERITY.get(item.level, FindingSeverity.MEDIUM),
965 message=(
966 f"{item.message} ({item.tool} {item.tool_version or ''}"
967 f"{', rule ' + item.rule_version if item.rule_version else ''})"
968 ).replace(" )", ")"),
969 path=item.path,
970 line=item.line,
971 category="external-scanner",
972 recommendation=f"Reported by {item.tool}; fix it at the location shown.",
973 )
976def interface_issues(workspace: Path, task: Task, policy: str) -> tuple[list[Issue], str]:
977 severity = FindingSeverity.HIGH if policy == "enforce" else FindingSeverity.LOW
978 issues: list[Issue] = []
979 for stub, module in declared_interfaces(task):
980 declared = _read(workspace, stub)
981 implementation = _read(workspace, module)
982 if declared is None or implementation is None: 982 ↛ 983line 982 didn't jump to line 983 because the condition on line 982 was never true
983 missing = stub if declared is None else module
984 issues.append(
985 Issue(
986 rule_id="interface.missing",
987 severity=severity,
988 message=f"The declared interface file {missing} is not in the workspace",
989 path=missing,
990 category="interface",
991 recommendation="Keep the declared interface and its module in place.",
992 )
993 )
994 continue
995 issues.extend(
996 check_interface(
997 declared,
998 implementation,
999 declared_path=stub,
1000 implementation_path=module,
1001 severity=severity,
1002 )
1003 )
1004 return issues, "declared interface conformance"
1007def architecture_issues(
1008 config: Any, files: Mapping[str, str], baseline_text: Callable[[str], str | None] | None
1009) -> tuple[list[Issue], str]:
1010 severity = config.severity or FindingSeverity.MEDIUM
1011 limits = ArchitectureLimits(
1012 max_module_lines=config.max_module_lines,
1013 max_function_lines=config.max_function_lines,
1014 max_complexity=config.max_complexity,
1015 forbidden_imports=tuple(
1016 (item.source, item.target) for item in config.forbidden_imports or ()
1017 ),
1018 )
1019 kept: list[Issue] = []
1020 for issue in check_architecture(files, limits, severity=severity):
1021 # A limit the file already exceeded on the baseline (same rule, same subject) is
1022 # pre-existing: reported as LOW so the agent is not blamed for it.
1023 before = baseline_text(issue.path) if baseline_text and issue.path else None
1024 previous = (
1025 check_architecture({issue.path or "": before}, limits, severity=severity)
1026 if before is not None
1027 else []
1028 )
1029 if any(
1030 item.rule_id == issue.rule_id and _subject(item.message) == _subject(issue.message)
1031 for item in previous
1032 ):
1033 kept.append(
1034 replace(
1035 issue, severity=FindingSeverity.LOW, message="Pre-existing: " + issue.message
1036 )
1037 )
1038 else:
1039 kept.append(issue)
1040 return kept, "architecture limits"
1043def principle_issues(
1044 config: Any,
1045 diff: Sequence[DiffFile],
1046 files: Mapping[str, str],
1047 workspace: Path,
1048 baseline_text: Callable[[str], str | None] | None,
1049) -> tuple[list[Issue], str]:
1050 """The deterministic proxies of the engineering principles (#56) on the ChangeSet."""
1051 principles = config.principles
1052 severity = principles.severity or FindingSeverity.MEDIUM
1053 families = {_family_of(path) for path in files if is_source(path)}
1054 corpus = source_corpus(workspace, families) if families else {}
1055 architecture = config.architecture
1056 limit = (
1057 architecture.max_module_lines
1058 if architecture is not None and architecture.max_module_lines
1059 else 800
1060 )
1061 baseline_lines: dict[str, int] = {}
1062 if baseline_text is not None: 1062 ↛ 1067line 1062 didn't jump to line 1067 because the condition on line 1062 was always true
1063 for path in files:
1064 before = baseline_text(path)
1065 if before is not None: 1065 ↛ 1063line 1065 didn't jump to line 1063 because the condition on line 1065 was always true
1066 baseline_lines[path] = len(before.splitlines())
1067 issues = check_principles(
1068 diff,
1069 files,
1070 corpus,
1071 PrincipleLimits(
1072 window=principles.window,
1073 inheritance_depth=principles.inheritance_depth,
1074 max_module_lines=limit,
1075 unused_public=principles.unused_public is not False,
1076 boy_scout=principles.boy_scout is not False,
1077 ),
1078 severity=severity,
1079 baseline_lines=baseline_lines,
1080 )
1081 return issues, "engineering principles"
1084def _family_of(path: str) -> str:
1085 from governed_harness.checks.principles import _family
1087 return _family(path)
1090def constraint_issues(task: Task, files: Mapping[str, str], policy: str) -> tuple[list[Issue], str]:
1091 declared = task.metadata.get("checks") or []
1092 checks = constraints_from_task(
1093 task.constraints, [str(item) for item in declared] if isinstance(declared, list) else []
1094 )
1095 if not checks:
1096 return [], "task constraints (none verifiable)"
1097 severity = FindingSeverity.HIGH if policy == "enforce" else FindingSeverity.LOW
1098 return check_constraints(checks, files, severity=severity), (
1099 f"task constraints ({', '.join(checks)})"
1100 )