Coverage for src/governed_harness/validators/traceability.py: 95%
239 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Requirement traceability: relate each identified requirement of a task to the tests that name it.
3The check is deterministic and reads the workspace without executing anything. A requirement is
4identified by the token that starts its text (``A1. ...``, ``[B12] ...``, ``X8: ...``) or, when the
5text has none, by a ``requirementId`` that the task file set explicitly. Generated ids
6(``req_<32 hex digits>``) do not identify a requirement, and requirements without an identifier are
7skipped and counted.
9A requirement is traced when a test names it: a test file, class or function whose name contains the
10identifier as a token in any case (``test_a1_rounding``, ``test_A1``, ``TestA1``), or the identifier
11as a whole word in a test's source, docstring or string constants (``pytest.param(..., id="A1")``,
12``# covers A1``). Python test files are read with :mod:`ast`; a file that does not parse, and every
13Node test file, is read as text.
14"""
16from __future__ import annotations
18import ast
19import os
20import re
21from collections.abc import Iterable, Iterator
22from dataclasses import dataclass
23from datetime import UTC, datetime
24from pathlib import Path, PurePosixPath
25from typing import Literal
27from pydantic import Field
29from governed_harness.domain.enums import (
30 ActorType,
31 ErrorKind,
32 FindingSeverity,
33 ResultStatus,
34 ValidationKind,
35)
36from governed_harness.domain.ids import new_id
37from governed_harness.domain.models import (
38 Actor,
39 Finding,
40 HarnessErrorRecord,
41 Requirement,
42 StrictModel,
43 ValidationResult,
44 utc_now,
45)
46from governed_harness.evidence.artifact_store import ArtifactRef
47from governed_harness.validators.base import ValidationContext, ValidatorOutput
49TRACEABILITY_VALIDATOR_ID = "traceability.requirements"
50UNTESTED_RULE_ID = "traceability.requirement-untested"
52_IDENTIFIER = r"[A-Z]{1,3}\d{1,3}(?:\.\d+)?"
53REQUIREMENT_TOKEN = re.compile(
54 rf"^\s*(?:\[(?P<bracketed>{_IDENTIFIER})\][.:)]?|(?P<plain>{_IDENTIFIER})[.:)])\s"
55)
56"""The identifier that starts a requirement's text: ``A1. ``, ``[B12] ``, ``X8: ``, ``C3.1) ``."""
58GENERATED_REQUIREMENT_ID = re.compile(r"^req_[0-9a-f]{32}$")
59"""The shape of the id the task loader generates when the task file gives none."""
61PYTHON_TEST_FILE = re.compile(r"^(?:test_.*|.*_test)\.py$")
62NODE_TEST_FILE = re.compile(
63 r"^(?:.*[.\-_](?:test|spec)|test-.*|test)\.(?:[cm]?js|[cm]?ts|jsx|tsx)$"
64)
65NODE_SOURCE_FILE = re.compile(r"\.(?:[cm]?js|[cm]?ts|jsx|tsx)$")
66NODE_TEST_DIRECTORIES = frozenset({"test", "tests", "__tests__"})
67PRUNED_DIRECTORIES = frozenset(
68 {"node_modules", "__pycache__", "venv", "site-packages", "build", "dist"}
69)
70MAX_TEST_FILE_BYTES = 2_000_000
71EXCERPT_CHARS = 80
73_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")
74_NAME_SEPARATORS = re.compile(r"[^A-Za-z0-9]+")
75_NODE_TEST_TITLE = re.compile(
76 r"\b(?:test|it|describe)(?:\.\w+)?\s*\(\s*(?P<quote>['\"`])(?P<title>(?:\\.|(?!(?P=quote)).)*)(?P=quote)"
77)
79IdentifierSource = Literal["text", "requirementId"]
80MatchKind = Literal["name", "string", "text"]
81TraceabilityPolicy = Literal["enforce", "warn"]
84class TracedTest(StrictModel):
85 """A test that names a requirement, and how it names it."""
87 node_id: str
88 path: str
89 match: MatchKind
92class RequirementTrace(StrictModel):
93 requirement_id: str
94 identifier: str
95 identifier_source: IdentifierSource
96 excerpt: str
97 traced: bool
98 tests: tuple[TracedTest, ...] = ()
101class RequirementTraceabilityReport(StrictModel):
102 """VERIFICATION evidence: the requirement to test mapping of one ChangeSet."""
104 schema_version: Literal["1.0"] = "1.0"
105 execution_id: str
106 task_id: str
107 change_set_digest: str
108 policy: TraceabilityPolicy
109 technologies: tuple[str, ...]
110 test_files: tuple[str, ...]
111 unread_files: tuple[str, ...] = ()
112 requirements: tuple[RequirementTrace, ...]
113 skipped_requirement_ids: tuple[str, ...] = ()
114 traced_count: int = Field(ge=0)
115 untraced_count: int = Field(ge=0)
116 skipped_count: int = Field(ge=0)
117 created_at: datetime = Field(default_factory=utc_now)
120@dataclass(frozen=True)
121class RequirementIdentifier:
122 requirement: Requirement
123 identifier: str
124 source: IdentifierSource
127@dataclass(frozen=True)
128class TestNode:
129 """A unit that can name a requirement: a test file, class or function."""
131 node_id: str
132 path: str
133 names: tuple[str, ...]
134 strings: tuple[str, ...] = ()
135 text: str = ""
138@dataclass(frozen=True)
139class TestCorpus:
140 files: tuple[str, ...]
141 unread: tuple[str, ...]
142 nodes: tuple[TestNode, ...]
145@dataclass(frozen=True)
146class TraceabilityOutput(ValidatorOutput):
147 """A validator output that also carries the stored report, so it can be recorded as evidence."""
149 report: RequirementTraceabilityReport | None = None
150 report_ref: ArtifactRef | None = None
153# ----- identifiers ---------------------------------------------------------------------------
154def requirement_identifier(requirement: Requirement) -> RequirementIdentifier | None:
155 """The identifier of a requirement, or ``None`` when it has none to check."""
156 match = REQUIREMENT_TOKEN.match(requirement.text)
157 if match:
158 token = match.group("bracketed") or match.group("plain")
159 return RequirementIdentifier(requirement, token, "text")
160 explicit = requirement.requirement_id.strip()
161 if explicit and not GENERATED_REQUIREMENT_ID.fullmatch(explicit):
162 return RequirementIdentifier(requirement, explicit, "requirementId")
163 return None
166def name_tokens(name: str) -> tuple[str, ...]:
167 """Lower-case tokens of a file, class or function name: ``TestA1Rounding`` and
168 ``test_a1_rounding`` both give ``("test", "a1", "rounding")``."""
169 spaced = _CAMEL_BOUNDARY.sub("_", name)
170 return tuple(part.lower() for part in _NAME_SEPARATORS.split(spaced) if part)
173def name_mentions(name: str, identifier: str) -> bool:
174 """Whether ``identifier`` is a run of whole tokens of ``name``."""
175 wanted = name_tokens(identifier)
176 tokens = name_tokens(name)
177 if not wanted: 177 ↛ 178line 177 didn't jump to line 178 because the condition on line 177 was never true
178 return False
179 width = len(wanted)
180 return any(tokens[index : index + width] == wanted for index in range(len(tokens) - width + 1))
183def word_pattern(identifier: str) -> re.Pattern[str]:
184 """``identifier`` as a whole word: ``A1`` matches ``A1.`` and ``[A1]`` but not ``A12``,
185 ``A1.2`` or ``a1``."""
186 return re.compile(rf"(?<![A-Za-z0-9_]){re.escape(identifier)}(?![A-Za-z0-9_]|\.\d)")
189# ----- test discovery ------------------------------------------------------------------------
190def is_test_file(relative: PurePosixPath, technologies: Iterable[str]) -> bool:
191 selected = set(technologies)
192 name = relative.name
193 if "python" in selected and PYTHON_TEST_FILE.match(name):
194 return True
195 if "node" in selected and NODE_SOURCE_FILE.search(name):
196 if NODE_TEST_FILE.match(name):
197 return True
198 return any(part in NODE_TEST_DIRECTORIES for part in relative.parts[:-1])
199 return False
202def discover_test_files(workspace: Path, technologies: Iterable[str]) -> tuple[str, ...]:
203 """Workspace-relative POSIX paths of the test files, sorted. Hidden directories, dependency
204 and build directories and symbolic links are not followed."""
205 selected = tuple(technologies)
206 found: list[str] = []
207 for current, directories, files in os.walk(workspace):
208 directories[:] = sorted(
209 item
210 for item in directories
211 if not item.startswith(".")
212 and item not in PRUNED_DIRECTORIES
213 and not (Path(current) / item).is_symlink()
214 )
215 for filename in files:
216 path = Path(current) / filename
217 if path.is_symlink(): 217 ↛ 218line 217 didn't jump to line 218 because the condition on line 217 was never true
218 continue
219 relative = PurePosixPath(path.relative_to(workspace).as_posix())
220 if is_test_file(relative, selected):
221 found.append(relative.as_posix())
222 return tuple(sorted(found))
225def load_test_corpus(workspace: Path, technologies: Iterable[str]) -> TestCorpus:
226 files = discover_test_files(workspace, technologies)
227 unread: list[str] = []
228 nodes: list[TestNode] = []
229 for relative in files:
230 path = workspace / relative
231 try:
232 if path.stat().st_size > MAX_TEST_FILE_BYTES: 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true
233 unread.append(relative)
234 continue
235 source = path.read_bytes().decode("utf-8", "replace")
236 except OSError:
237 unread.append(relative)
238 continue
239 if relative.endswith(".py"):
240 nodes.extend(python_test_nodes(relative, source))
241 else:
242 nodes.extend(node_test_nodes(relative, source))
243 return TestCorpus(files=files, unread=tuple(unread), nodes=tuple(nodes))
246def python_test_nodes(relative: str, source: str) -> list[TestNode]:
247 """The file, its ``Test*`` classes and its ``test*`` functions. A file that does not parse is
248 one node whose whole text is searched."""
249 stem = PurePosixPath(relative).stem
250 try:
251 tree = ast.parse(source, filename=relative)
252 except (SyntaxError, ValueError, RecursionError, MemoryError):
253 return [TestNode(node_id=relative, path=relative, names=(stem,), text=source)]
254 lines = source.splitlines()
255 docstring = ast.get_docstring(tree, clean=False)
256 nodes = [
257 TestNode(
258 node_id=relative,
259 path=relative,
260 names=(stem,),
261 strings=(docstring,) if docstring else (),
262 )
263 ]
264 nodes.extend(_python_scope(relative, tree.body, lines, prefix=relative))
265 return nodes
268def _python_scope(
269 relative: str,
270 body: list[ast.stmt],
271 lines: list[str],
272 *,
273 prefix: str,
274) -> Iterator[TestNode]:
275 for statement in body:
276 if isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef):
277 if statement.name.startswith("test"):
278 yield TestNode(
279 node_id=f"{prefix}::{statement.name}",
280 path=relative,
281 names=(statement.name,),
282 strings=tuple(_string_constants(statement)),
283 text=_segment(statement, lines),
284 )
285 elif isinstance(statement, ast.ClassDef) and statement.name.startswith("Test"):
286 node_id = f"{prefix}::{statement.name}"
287 docstring = ast.get_docstring(statement, clean=False)
288 yield TestNode(
289 node_id=node_id,
290 path=relative,
291 names=(statement.name,),
292 strings=((docstring,) if docstring else ())
293 + tuple(
294 value
295 for decorator in statement.decorator_list
296 for value in _string_constants(decorator)
297 ),
298 )
299 yield from _python_scope(relative, statement.body, lines, prefix=node_id)
302def _string_constants(node: ast.AST) -> Iterator[str]:
303 """Docstrings, parametrize ids and every other string constant of a test and its
304 decorators."""
305 roots: list[ast.AST] = [node]
306 if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): 306 ↛ 308line 306 didn't jump to line 308 because the condition on line 306 was always true
307 roots.extend(node.decorator_list)
308 for root in roots:
309 for child in ast.walk(root):
310 if isinstance(child, ast.Constant) and isinstance(child.value, str):
311 yield child.value
314def _segment(node: ast.FunctionDef | ast.AsyncFunctionDef, lines: list[str]) -> str:
315 """The source lines of a test, decorators and comments included."""
316 first = min([node.lineno, *(item.lineno for item in node.decorator_list)])
317 last = node.end_lineno or node.lineno
318 return "\n".join(lines[first - 1 : last])
321def node_test_nodes(relative: str, source: str) -> list[TestNode]:
322 """The file (its name and whole text) and every ``test``, ``it`` or ``describe`` title."""
323 nodes = [
324 TestNode(
325 node_id=relative,
326 path=relative,
327 names=(PurePosixPath(relative).name,),
328 text=source,
329 )
330 ]
331 for match in _NODE_TEST_TITLE.finditer(source):
332 title = match.group("title")
333 nodes.append(
334 TestNode(node_id=f"{relative}::{title}", path=relative, names=(), strings=(title,))
335 )
336 return nodes
339# ----- matching ------------------------------------------------------------------------------
340def tests_naming(identifier: str, corpus: TestCorpus) -> tuple[TracedTest, ...]:
341 """The tests of ``corpus`` that name ``identifier``, in corpus order, one entry per node."""
342 pattern = word_pattern(identifier)
343 traced: list[TracedTest] = []
344 for node in corpus.nodes:
345 kind: MatchKind | None = None
346 if any(name_mentions(name, identifier) for name in node.names):
347 kind = "name"
348 elif any(pattern.search(value) for value in node.strings):
349 kind = "string"
350 elif node.text and pattern.search(node.text):
351 kind = "text"
352 if kind is not None:
353 traced.append(TracedTest(node_id=node.node_id, path=node.path, match=kind))
354 # A Node file's text contains its titles: keep the file only when no title of it matched.
355 titled = {item.path for item in traced if item.node_id != item.path and item.match == "string"}
356 return tuple(
357 item
358 for item in traced
359 if not (item.node_id == item.path and item.match == "text" and item.path in titled)
360 )
363def trace_requirements(
364 requirements: Iterable[Requirement], corpus: TestCorpus
365) -> tuple[tuple[RequirementTrace, ...], tuple[str, ...]]:
366 """The trace of every identified requirement and the ids of the skipped ones."""
367 traces: list[RequirementTrace] = []
368 skipped: list[str] = []
369 for requirement in requirements:
370 identified = requirement_identifier(requirement)
371 if identified is None:
372 skipped.append(requirement.requirement_id)
373 continue
374 tests = tests_naming(identified.identifier, corpus)
375 traces.append(
376 RequirementTrace(
377 requirement_id=requirement.requirement_id,
378 identifier=identified.identifier,
379 identifier_source=identified.source,
380 excerpt=excerpt(requirement.text),
381 traced=bool(tests),
382 tests=tests,
383 )
384 )
385 return tuple(traces), tuple(skipped)
388def excerpt(text: str) -> str:
389 return " ".join(text.split())[:EXCERPT_CHARS]
392# ----- validator -----------------------------------------------------------------------------
393class RequirementTraceabilityValidator:
394 """Report identified requirements that no test names.
396 Under ``enforce`` each untraced requirement is a ``HIGH`` finding, which the default
397 ``findingBlockSeverities`` turn into a ``FAILED`` gate; under ``warn`` it is ``LOW``. The
398 validation result itself is ``PASSED`` when the check ran, like the independent review, so
399 the run reaches the gate and the person deciding sees the findings."""
401 validator_id = TRACEABILITY_VALIDATOR_ID
403 def __init__(self, policy: TraceabilityPolicy, technologies: Iterable[str]) -> None:
404 self.policy = policy
405 self.technologies = tuple(sorted(set(technologies)))
407 def execute(self, context: ValidationContext) -> TraceabilityOutput:
408 actor = Actor(
409 actor_type=ActorType.TOOL, actor_id=f"validator.{self.validator_id}", version="1"
410 )
411 provenance = context.provenance.model_copy(update={"actor": actor})
412 started = datetime.now(UTC)
413 mandatory = self.policy == "enforce"
414 try:
415 corpus = load_test_corpus(context.workspace, self.technologies)
416 except OSError as error:
417 summary = f"Requirement traceability could not read the workspace: {error}"
418 evidence = context.artifact_store.put_json(
419 {"validatorId": self.validator_id, "status": ResultStatus.ERROR, "reason": summary},
420 metadata={"kind": "validator-availability"},
421 )
422 result = ValidationResult(
423 validation_result_id=new_id("validation"),
424 execution_id=context.execution_id,
425 validator_id=self.validator_id,
426 change_set_digest=context.change_set.digest,
427 status=ResultStatus.ERROR,
428 kind=ValidationKind.TOOL_ERROR,
429 mandatory=mandatory,
430 summary=summary,
431 evidence_refs=(evidence.uri,),
432 started_at=started,
433 finished_at=datetime.now(UTC),
434 errors=(
435 HarnessErrorRecord(
436 error_id=new_id("err"),
437 kind=ErrorKind.TOOL_ERROR,
438 message=summary,
439 actor=actor,
440 ),
441 ),
442 provenance=provenance,
443 )
444 return TraceabilityOutput(result)
445 traces, skipped = trace_requirements(context.task.requirements, corpus)
446 untraced = [item for item in traces if not item.traced]
447 report = RequirementTraceabilityReport(
448 execution_id=context.execution_id,
449 task_id=context.task.task_id,
450 change_set_digest=context.change_set.digest,
451 policy=self.policy,
452 technologies=self.technologies,
453 test_files=corpus.files,
454 unread_files=corpus.unread,
455 requirements=traces,
456 skipped_requirement_ids=skipped,
457 traced_count=len(traces) - len(untraced),
458 untraced_count=len(untraced),
459 skipped_count=len(skipped),
460 )
461 report_ref = context.artifact_store.put_json(
462 report.model_dump(mode="json", by_alias=True),
463 metadata={"kind": "requirement-traceability", "validatorId": self.validator_id},
464 )
465 severity = FindingSeverity.HIGH if self.policy == "enforce" else FindingSeverity.LOW
466 findings = tuple(
467 Finding(
468 finding_id=new_id("finding"),
469 execution_id=context.execution_id,
470 validator_id=self.validator_id,
471 rule_id=UNTESTED_RULE_ID,
472 category="traceability",
473 severity=severity,
474 message=f"No test names requirement {item.identifier}: {item.excerpt}",
475 evidence_refs=(report_ref.uri,),
476 recommendation=(
477 f"Add or rename a test so that it names {item.identifier}: a test, class or "
478 f"file name with the token {'_'.join(name_tokens(item.identifier))} "
479 f"(test_{'_'.join(name_tokens(item.identifier))}_...), or "
480 f"{item.identifier} as a word in the test's docstring or parametrize id."
481 ),
482 provenance=provenance,
483 )
484 for item in untraced
485 )
486 result = ValidationResult(
487 validation_result_id=new_id("validation"),
488 execution_id=context.execution_id,
489 validator_id=self.validator_id,
490 change_set_digest=context.change_set.digest,
491 status=ResultStatus.PASSED,
492 kind=ValidationKind.SUCCESS,
493 mandatory=mandatory,
494 summary=summarize(report),
495 finding_ids=tuple(item.finding_id for item in findings),
496 evidence_refs=(report_ref.uri,),
497 started_at=started,
498 finished_at=datetime.now(UTC),
499 provenance=provenance,
500 )
501 return TraceabilityOutput(result, findings, (), report, report_ref)
504def summarize(report: RequirementTraceabilityReport) -> str:
505 identified = report.traced_count + report.untraced_count
506 return (
507 f"{report.traced_count} of {identified} identified requirement(s) traced to tests, "
508 f"{report.untraced_count} untraced, {report.skipped_count} without an identifier skipped"
509 )