Coverage for src/governed_harness/validators/traceability.py: 95%

239 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Requirement traceability: relate each identified requirement of a task to the tests that name it. 

2 

3The check is deterministic and reads the workspace without executing anything. A requirement is 

4identified by the token that starts its text (``A1. ...``, ``[B12] ...``, ``X8: ...``) or, when the 

5text has none, by a ``requirementId`` that the task file set explicitly. Generated ids 

6(``req_<32 hex digits>``) do not identify a requirement, and requirements without an identifier are 

7skipped and counted. 

8 

9A requirement is traced when a test names it: a test file, class or function whose name contains the 

10identifier as a token in any case (``test_a1_rounding``, ``test_A1``, ``TestA1``), or the identifier 

11as a whole word in a test's source, docstring or string constants (``pytest.param(..., id="A1")``, 

12``# covers A1``). Python test files are read with :mod:`ast`; a file that does not parse, and every 

13Node test file, is read as text. 

14""" 

15 

16from __future__ import annotations 

17 

18import ast 

19import os 

20import re 

21from collections.abc import Iterable, Iterator 

22from dataclasses import dataclass 

23from datetime import UTC, datetime 

24from pathlib import Path, PurePosixPath 

25from typing import Literal 

26 

27from pydantic import Field 

28 

29from governed_harness.domain.enums import ( 

30 ActorType, 

31 ErrorKind, 

32 FindingSeverity, 

33 ResultStatus, 

34 ValidationKind, 

35) 

36from governed_harness.domain.ids import new_id 

37from governed_harness.domain.models import ( 

38 Actor, 

39 Finding, 

40 HarnessErrorRecord, 

41 Requirement, 

42 StrictModel, 

43 ValidationResult, 

44 utc_now, 

45) 

46from governed_harness.evidence.artifact_store import ArtifactRef 

47from governed_harness.validators.base import ValidationContext, ValidatorOutput 

48 

49TRACEABILITY_VALIDATOR_ID = "traceability.requirements" 

50UNTESTED_RULE_ID = "traceability.requirement-untested" 

51 

52_IDENTIFIER = r"[A-Z]{1,3}\d{1,3}(?:\.\d+)?" 

53REQUIREMENT_TOKEN = re.compile( 

54 rf"^\s*(?:\[(?P<bracketed>{_IDENTIFIER})\][.:)]?|(?P<plain>{_IDENTIFIER})[.:)])\s" 

55) 

56"""The identifier that starts a requirement's text: ``A1. ``, ``[B12] ``, ``X8: ``, ``C3.1) ``.""" 

57 

58GENERATED_REQUIREMENT_ID = re.compile(r"^req_[0-9a-f]{32}$") 

59"""The shape of the id the task loader generates when the task file gives none.""" 

60 

61PYTHON_TEST_FILE = re.compile(r"^(?:test_.*|.*_test)\.py$") 

62NODE_TEST_FILE = re.compile( 

63 r"^(?:.*[.\-_](?:test|spec)|test-.*|test)\.(?:[cm]?js|[cm]?ts|jsx|tsx)$" 

64) 

65NODE_SOURCE_FILE = re.compile(r"\.(?:[cm]?js|[cm]?ts|jsx|tsx)$") 

66NODE_TEST_DIRECTORIES = frozenset({"test", "tests", "__tests__"}) 

67PRUNED_DIRECTORIES = frozenset( 

68 {"node_modules", "__pycache__", "venv", "site-packages", "build", "dist"} 

69) 

70MAX_TEST_FILE_BYTES = 2_000_000 

71EXCERPT_CHARS = 80 

72 

73_CAMEL_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") 

74_NAME_SEPARATORS = re.compile(r"[^A-Za-z0-9]+") 

75_NODE_TEST_TITLE = re.compile( 

76 r"\b(?:test|it|describe)(?:\.\w+)?\s*\(\s*(?P<quote>['\"`])(?P<title>(?:\\.|(?!(?P=quote)).)*)(?P=quote)" 

77) 

78 

79IdentifierSource = Literal["text", "requirementId"] 

80MatchKind = Literal["name", "string", "text"] 

81TraceabilityPolicy = Literal["enforce", "warn"] 

82 

83 

84class TracedTest(StrictModel): 

85 """A test that names a requirement, and how it names it.""" 

86 

87 node_id: str 

88 path: str 

89 match: MatchKind 

90 

91 

92class RequirementTrace(StrictModel): 

93 requirement_id: str 

94 identifier: str 

95 identifier_source: IdentifierSource 

96 excerpt: str 

97 traced: bool 

98 tests: tuple[TracedTest, ...] = () 

99 

100 

101class RequirementTraceabilityReport(StrictModel): 

102 """VERIFICATION evidence: the requirement to test mapping of one ChangeSet.""" 

103 

104 schema_version: Literal["1.0"] = "1.0" 

105 execution_id: str 

106 task_id: str 

107 change_set_digest: str 

108 policy: TraceabilityPolicy 

109 technologies: tuple[str, ...] 

110 test_files: tuple[str, ...] 

111 unread_files: tuple[str, ...] = () 

112 requirements: tuple[RequirementTrace, ...] 

113 skipped_requirement_ids: tuple[str, ...] = () 

114 traced_count: int = Field(ge=0) 

115 untraced_count: int = Field(ge=0) 

116 skipped_count: int = Field(ge=0) 

117 created_at: datetime = Field(default_factory=utc_now) 

118 

119 

120@dataclass(frozen=True) 

121class RequirementIdentifier: 

122 requirement: Requirement 

123 identifier: str 

124 source: IdentifierSource 

125 

126 

127@dataclass(frozen=True) 

128class TestNode: 

129 """A unit that can name a requirement: a test file, class or function.""" 

130 

131 node_id: str 

132 path: str 

133 names: tuple[str, ...] 

134 strings: tuple[str, ...] = () 

135 text: str = "" 

136 

137 

138@dataclass(frozen=True) 

139class TestCorpus: 

140 files: tuple[str, ...] 

141 unread: tuple[str, ...] 

142 nodes: tuple[TestNode, ...] 

143 

144 

145@dataclass(frozen=True) 

146class TraceabilityOutput(ValidatorOutput): 

147 """A validator output that also carries the stored report, so it can be recorded as evidence.""" 

148 

149 report: RequirementTraceabilityReport | None = None 

150 report_ref: ArtifactRef | None = None 

151 

152 

153# ----- identifiers --------------------------------------------------------------------------- 

154def requirement_identifier(requirement: Requirement) -> RequirementIdentifier | None: 

155 """The identifier of a requirement, or ``None`` when it has none to check.""" 

156 match = REQUIREMENT_TOKEN.match(requirement.text) 

157 if match: 

158 token = match.group("bracketed") or match.group("plain") 

159 return RequirementIdentifier(requirement, token, "text") 

160 explicit = requirement.requirement_id.strip() 

161 if explicit and not GENERATED_REQUIREMENT_ID.fullmatch(explicit): 

162 return RequirementIdentifier(requirement, explicit, "requirementId") 

163 return None 

164 

165 

166def name_tokens(name: str) -> tuple[str, ...]: 

167 """Lower-case tokens of a file, class or function name: ``TestA1Rounding`` and 

168 ``test_a1_rounding`` both give ``("test", "a1", "rounding")``.""" 

169 spaced = _CAMEL_BOUNDARY.sub("_", name) 

170 return tuple(part.lower() for part in _NAME_SEPARATORS.split(spaced) if part) 

171 

172 

173def name_mentions(name: str, identifier: str) -> bool: 

174 """Whether ``identifier`` is a run of whole tokens of ``name``.""" 

175 wanted = name_tokens(identifier) 

176 tokens = name_tokens(name) 

177 if not wanted: 177 ↛ 178line 177 didn't jump to line 178 because the condition on line 177 was never true

178 return False 

179 width = len(wanted) 

180 return any(tokens[index : index + width] == wanted for index in range(len(tokens) - width + 1)) 

181 

182 

183def word_pattern(identifier: str) -> re.Pattern[str]: 

184 """``identifier`` as a whole word: ``A1`` matches ``A1.`` and ``[A1]`` but not ``A12``, 

185 ``A1.2`` or ``a1``.""" 

186 return re.compile(rf"(?<![A-Za-z0-9_]){re.escape(identifier)}(?![A-Za-z0-9_]|\.\d)") 

187 

188 

189# ----- test discovery ------------------------------------------------------------------------ 

190def is_test_file(relative: PurePosixPath, technologies: Iterable[str]) -> bool: 

191 selected = set(technologies) 

192 name = relative.name 

193 if "python" in selected and PYTHON_TEST_FILE.match(name): 

194 return True 

195 if "node" in selected and NODE_SOURCE_FILE.search(name): 

196 if NODE_TEST_FILE.match(name): 

197 return True 

198 return any(part in NODE_TEST_DIRECTORIES for part in relative.parts[:-1]) 

199 return False 

200 

201 

202def discover_test_files(workspace: Path, technologies: Iterable[str]) -> tuple[str, ...]: 

203 """Workspace-relative POSIX paths of the test files, sorted. Hidden directories, dependency 

204 and build directories and symbolic links are not followed.""" 

205 selected = tuple(technologies) 

206 found: list[str] = [] 

207 for current, directories, files in os.walk(workspace): 

208 directories[:] = sorted( 

209 item 

210 for item in directories 

211 if not item.startswith(".") 

212 and item not in PRUNED_DIRECTORIES 

213 and not (Path(current) / item).is_symlink() 

214 ) 

215 for filename in files: 

216 path = Path(current) / filename 

217 if path.is_symlink(): 217 ↛ 218line 217 didn't jump to line 218 because the condition on line 217 was never true

218 continue 

219 relative = PurePosixPath(path.relative_to(workspace).as_posix()) 

220 if is_test_file(relative, selected): 

221 found.append(relative.as_posix()) 

222 return tuple(sorted(found)) 

223 

224 

225def load_test_corpus(workspace: Path, technologies: Iterable[str]) -> TestCorpus: 

226 files = discover_test_files(workspace, technologies) 

227 unread: list[str] = [] 

228 nodes: list[TestNode] = [] 

229 for relative in files: 

230 path = workspace / relative 

231 try: 

232 if path.stat().st_size > MAX_TEST_FILE_BYTES: 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true

233 unread.append(relative) 

234 continue 

235 source = path.read_bytes().decode("utf-8", "replace") 

236 except OSError: 

237 unread.append(relative) 

238 continue 

239 if relative.endswith(".py"): 

240 nodes.extend(python_test_nodes(relative, source)) 

241 else: 

242 nodes.extend(node_test_nodes(relative, source)) 

243 return TestCorpus(files=files, unread=tuple(unread), nodes=tuple(nodes)) 

244 

245 

246def python_test_nodes(relative: str, source: str) -> list[TestNode]: 

247 """The file, its ``Test*`` classes and its ``test*`` functions. A file that does not parse is 

248 one node whose whole text is searched.""" 

249 stem = PurePosixPath(relative).stem 

250 try: 

251 tree = ast.parse(source, filename=relative) 

252 except (SyntaxError, ValueError, RecursionError, MemoryError): 

253 return [TestNode(node_id=relative, path=relative, names=(stem,), text=source)] 

254 lines = source.splitlines() 

255 docstring = ast.get_docstring(tree, clean=False) 

256 nodes = [ 

257 TestNode( 

258 node_id=relative, 

259 path=relative, 

260 names=(stem,), 

261 strings=(docstring,) if docstring else (), 

262 ) 

263 ] 

264 nodes.extend(_python_scope(relative, tree.body, lines, prefix=relative)) 

265 return nodes 

266 

267 

268def _python_scope( 

269 relative: str, 

270 body: list[ast.stmt], 

271 lines: list[str], 

272 *, 

273 prefix: str, 

274) -> Iterator[TestNode]: 

275 for statement in body: 

276 if isinstance(statement, ast.FunctionDef | ast.AsyncFunctionDef): 

277 if statement.name.startswith("test"): 

278 yield TestNode( 

279 node_id=f"{prefix}::{statement.name}", 

280 path=relative, 

281 names=(statement.name,), 

282 strings=tuple(_string_constants(statement)), 

283 text=_segment(statement, lines), 

284 ) 

285 elif isinstance(statement, ast.ClassDef) and statement.name.startswith("Test"): 

286 node_id = f"{prefix}::{statement.name}" 

287 docstring = ast.get_docstring(statement, clean=False) 

288 yield TestNode( 

289 node_id=node_id, 

290 path=relative, 

291 names=(statement.name,), 

292 strings=((docstring,) if docstring else ()) 

293 + tuple( 

294 value 

295 for decorator in statement.decorator_list 

296 for value in _string_constants(decorator) 

297 ), 

298 ) 

299 yield from _python_scope(relative, statement.body, lines, prefix=node_id) 

300 

301 

302def _string_constants(node: ast.AST) -> Iterator[str]: 

303 """Docstrings, parametrize ids and every other string constant of a test and its 

304 decorators.""" 

305 roots: list[ast.AST] = [node] 

306 if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): 306 ↛ 308line 306 didn't jump to line 308 because the condition on line 306 was always true

307 roots.extend(node.decorator_list) 

308 for root in roots: 

309 for child in ast.walk(root): 

310 if isinstance(child, ast.Constant) and isinstance(child.value, str): 

311 yield child.value 

312 

313 

314def _segment(node: ast.FunctionDef | ast.AsyncFunctionDef, lines: list[str]) -> str: 

315 """The source lines of a test, decorators and comments included.""" 

316 first = min([node.lineno, *(item.lineno for item in node.decorator_list)]) 

317 last = node.end_lineno or node.lineno 

318 return "\n".join(lines[first - 1 : last]) 

319 

320 

321def node_test_nodes(relative: str, source: str) -> list[TestNode]: 

322 """The file (its name and whole text) and every ``test``, ``it`` or ``describe`` title.""" 

323 nodes = [ 

324 TestNode( 

325 node_id=relative, 

326 path=relative, 

327 names=(PurePosixPath(relative).name,), 

328 text=source, 

329 ) 

330 ] 

331 for match in _NODE_TEST_TITLE.finditer(source): 

332 title = match.group("title") 

333 nodes.append( 

334 TestNode(node_id=f"{relative}::{title}", path=relative, names=(), strings=(title,)) 

335 ) 

336 return nodes 

337 

338 

339# ----- matching ------------------------------------------------------------------------------ 

340def tests_naming(identifier: str, corpus: TestCorpus) -> tuple[TracedTest, ...]: 

341 """The tests of ``corpus`` that name ``identifier``, in corpus order, one entry per node.""" 

342 pattern = word_pattern(identifier) 

343 traced: list[TracedTest] = [] 

344 for node in corpus.nodes: 

345 kind: MatchKind | None = None 

346 if any(name_mentions(name, identifier) for name in node.names): 

347 kind = "name" 

348 elif any(pattern.search(value) for value in node.strings): 

349 kind = "string" 

350 elif node.text and pattern.search(node.text): 

351 kind = "text" 

352 if kind is not None: 

353 traced.append(TracedTest(node_id=node.node_id, path=node.path, match=kind)) 

354 # A Node file's text contains its titles: keep the file only when no title of it matched. 

355 titled = {item.path for item in traced if item.node_id != item.path and item.match == "string"} 

356 return tuple( 

357 item 

358 for item in traced 

359 if not (item.node_id == item.path and item.match == "text" and item.path in titled) 

360 ) 

361 

362 

363def trace_requirements( 

364 requirements: Iterable[Requirement], corpus: TestCorpus 

365) -> tuple[tuple[RequirementTrace, ...], tuple[str, ...]]: 

366 """The trace of every identified requirement and the ids of the skipped ones.""" 

367 traces: list[RequirementTrace] = [] 

368 skipped: list[str] = [] 

369 for requirement in requirements: 

370 identified = requirement_identifier(requirement) 

371 if identified is None: 

372 skipped.append(requirement.requirement_id) 

373 continue 

374 tests = tests_naming(identified.identifier, corpus) 

375 traces.append( 

376 RequirementTrace( 

377 requirement_id=requirement.requirement_id, 

378 identifier=identified.identifier, 

379 identifier_source=identified.source, 

380 excerpt=excerpt(requirement.text), 

381 traced=bool(tests), 

382 tests=tests, 

383 ) 

384 ) 

385 return tuple(traces), tuple(skipped) 

386 

387 

388def excerpt(text: str) -> str: 

389 return " ".join(text.split())[:EXCERPT_CHARS] 

390 

391 

392# ----- validator ----------------------------------------------------------------------------- 

393class RequirementTraceabilityValidator: 

394 """Report identified requirements that no test names. 

395 

396 Under ``enforce`` each untraced requirement is a ``HIGH`` finding, which the default 

397 ``findingBlockSeverities`` turn into a ``FAILED`` gate; under ``warn`` it is ``LOW``. The 

398 validation result itself is ``PASSED`` when the check ran, like the independent review, so 

399 the run reaches the gate and the person deciding sees the findings.""" 

400 

401 validator_id = TRACEABILITY_VALIDATOR_ID 

402 

403 def __init__(self, policy: TraceabilityPolicy, technologies: Iterable[str]) -> None: 

404 self.policy = policy 

405 self.technologies = tuple(sorted(set(technologies))) 

406 

407 def execute(self, context: ValidationContext) -> TraceabilityOutput: 

408 actor = Actor( 

409 actor_type=ActorType.TOOL, actor_id=f"validator.{self.validator_id}", version="1" 

410 ) 

411 provenance = context.provenance.model_copy(update={"actor": actor}) 

412 started = datetime.now(UTC) 

413 mandatory = self.policy == "enforce" 

414 try: 

415 corpus = load_test_corpus(context.workspace, self.technologies) 

416 except OSError as error: 

417 summary = f"Requirement traceability could not read the workspace: {error}" 

418 evidence = context.artifact_store.put_json( 

419 {"validatorId": self.validator_id, "status": ResultStatus.ERROR, "reason": summary}, 

420 metadata={"kind": "validator-availability"}, 

421 ) 

422 result = ValidationResult( 

423 validation_result_id=new_id("validation"), 

424 execution_id=context.execution_id, 

425 validator_id=self.validator_id, 

426 change_set_digest=context.change_set.digest, 

427 status=ResultStatus.ERROR, 

428 kind=ValidationKind.TOOL_ERROR, 

429 mandatory=mandatory, 

430 summary=summary, 

431 evidence_refs=(evidence.uri,), 

432 started_at=started, 

433 finished_at=datetime.now(UTC), 

434 errors=( 

435 HarnessErrorRecord( 

436 error_id=new_id("err"), 

437 kind=ErrorKind.TOOL_ERROR, 

438 message=summary, 

439 actor=actor, 

440 ), 

441 ), 

442 provenance=provenance, 

443 ) 

444 return TraceabilityOutput(result) 

445 traces, skipped = trace_requirements(context.task.requirements, corpus) 

446 untraced = [item for item in traces if not item.traced] 

447 report = RequirementTraceabilityReport( 

448 execution_id=context.execution_id, 

449 task_id=context.task.task_id, 

450 change_set_digest=context.change_set.digest, 

451 policy=self.policy, 

452 technologies=self.technologies, 

453 test_files=corpus.files, 

454 unread_files=corpus.unread, 

455 requirements=traces, 

456 skipped_requirement_ids=skipped, 

457 traced_count=len(traces) - len(untraced), 

458 untraced_count=len(untraced), 

459 skipped_count=len(skipped), 

460 ) 

461 report_ref = context.artifact_store.put_json( 

462 report.model_dump(mode="json", by_alias=True), 

463 metadata={"kind": "requirement-traceability", "validatorId": self.validator_id}, 

464 ) 

465 severity = FindingSeverity.HIGH if self.policy == "enforce" else FindingSeverity.LOW 

466 findings = tuple( 

467 Finding( 

468 finding_id=new_id("finding"), 

469 execution_id=context.execution_id, 

470 validator_id=self.validator_id, 

471 rule_id=UNTESTED_RULE_ID, 

472 category="traceability", 

473 severity=severity, 

474 message=f"No test names requirement {item.identifier}: {item.excerpt}", 

475 evidence_refs=(report_ref.uri,), 

476 recommendation=( 

477 f"Add or rename a test so that it names {item.identifier}: a test, class or " 

478 f"file name with the token {'_'.join(name_tokens(item.identifier))} " 

479 f"(test_{'_'.join(name_tokens(item.identifier))}_...), or " 

480 f"{item.identifier} as a word in the test's docstring or parametrize id." 

481 ), 

482 provenance=provenance, 

483 ) 

484 for item in untraced 

485 ) 

486 result = ValidationResult( 

487 validation_result_id=new_id("validation"), 

488 execution_id=context.execution_id, 

489 validator_id=self.validator_id, 

490 change_set_digest=context.change_set.digest, 

491 status=ResultStatus.PASSED, 

492 kind=ValidationKind.SUCCESS, 

493 mandatory=mandatory, 

494 summary=summarize(report), 

495 finding_ids=tuple(item.finding_id for item in findings), 

496 evidence_refs=(report_ref.uri,), 

497 started_at=started, 

498 finished_at=datetime.now(UTC), 

499 provenance=provenance, 

500 ) 

501 return TraceabilityOutput(result, findings, (), report, report_ref) 

502 

503 

504def summarize(report: RequirementTraceabilityReport) -> str: 

505 identified = report.traced_count + report.untraced_count 

506 return ( 

507 f"{report.traced_count} of {identified} identified requirement(s) traced to tests, " 

508 f"{report.untraced_count} untraced, {report.skipped_count} without an identifier skipped" 

509 )