Coverage for src/governed_harness/ladder/capabilities.py: 93%

88 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Verification capabilities of the selected profiles and their detection (#55, item 6). 

2 

3Each technology offers some rungs of the ladder: static checks, unit tests, integration tests 

4with the repository's own doubles, executable behaviour through probes, sometimes a simulator 

5or an emulator. The built-in profiles' capabilities are in 

6``resources/verification/capabilities.yaml``; a project profile declares its own under 

7``verification``. Detection is read-only: a command that must exit 0 (``adb devices``, 

8``xcrun simctl list``, ``docker info``) or a path that must exist. Nothing is ever installed. 

9""" 

10 

11from __future__ import annotations 

12 

13import os 

14import re 

15import shutil 

16import subprocess # nosec B404 - catalog probes with fixed argv, no shell 

17from collections.abc import Callable, Iterable 

18from dataclasses import dataclass, field 

19from pathlib import Path 

20from typing import TYPE_CHECKING, Any 

21 

22import yaml 

23 

24from governed_harness.runtime.process_runner import resolve_executable 

25 

26if TYPE_CHECKING: 

27 from governed_harness.configuration.ladder import ( 

28 CapabilityDetection, 

29 ProfileVerification, 

30 VerificationCapability, 

31 ) 

32 from governed_harness.configuration.models import TechnologyProfileDefinition 

33 

34ANY_PROFILE = "*" 

35_OUTPUT_CHARS = 500 

36 

37 

38def load_catalog() -> dict[str, ProfileVerification]: 

39 """The verification capabilities of the built-in profiles, by profile id.""" 

40 from governed_harness.configuration.ladder import ProfileVerification 

41 from governed_harness.configuration.loader import resource_text 

42 

43 raw = yaml.safe_load(resource_text("verification", "capabilities.yaml")) or {} 

44 return {str(key): ProfileVerification.model_validate(value) for key, value in raw.items()} 

45 

46 

47def profile_verification( 

48 profiles: Iterable[TechnologyProfileDefinition], catalog: dict[str, ProfileVerification] 

49) -> list[tuple[str, ProfileVerification]]: 

50 """The verification of each selected profile (its own declaration, else the catalog's), 

51 followed by the capabilities every profile shares.""" 

52 found: list[tuple[str, ProfileVerification]] = [] 

53 for profile in profiles: 

54 declared = profile.verification or catalog.get(profile.profile_id) 

55 if declared is not None: 55 ↛ 53line 55 didn't jump to line 53 because the condition on line 55 was always true

56 found.append((profile.profile_id, declared)) 

57 shared = catalog.get(ANY_PROFILE) 

58 if shared is not None: 58 ↛ 60line 58 didn't jump to line 60 because the condition on line 58 was always true

59 found.append((ANY_PROFILE, shared)) 

60 return found 

61 

62 

63@dataclass(frozen=True) 

64class DetectionResult: 

65 description: str 

66 available: bool 

67 output: str = "" 

68 checked: bool = True 

69 """False when the command was not run (``capabilityDetection`` off): the capability is 

70 planned as available but nothing is credited without the validators' results.""" 

71 

72 def as_dict(self) -> dict[str, Any]: 

73 return { 

74 "check": self.description, 

75 "available": self.available, 

76 "checked": self.checked, 

77 "output": self.output, 

78 } 

79 

80 

81@dataclass(frozen=True) 

82class CapabilityStatus: 

83 profile_id: str 

84 level: str 

85 provides: str 

86 validators: tuple[str, ...] 

87 probes: bool 

88 available: bool 

89 detections: tuple[DetectionResult, ...] = field(default_factory=tuple) 

90 fallback: str | None = None 

91 

92 def as_dict(self) -> dict[str, Any]: 

93 return { 

94 "profileId": self.profile_id, 

95 "level": self.level, 

96 "provides": self.provides, 

97 "validators": list(self.validators), 

98 "probes": self.probes, 

99 "available": self.available, 

100 "detections": [item.as_dict() for item in self.detections], 

101 "fallback": self.fallback, 

102 } 

103 

104 

105Runner = Callable[[tuple[str, ...], Path, int], tuple[int | None, str]] 

106 

107 

108def run_detection(argv: tuple[str, ...], cwd: Path, timeout: int) -> tuple[int | None, str]: 

109 """Run a detection command without a shell; ``(None, reason)`` when it cannot start.""" 

110 if shutil.which(argv[0]) is None and not (cwd / argv[0]).exists(): 110 ↛ 111line 110 didn't jump to line 111 because the condition on line 110 was never true

111 return None, f"{argv[0]} not found" 

112 try: 

113 result = subprocess.run( # nosec B603 - argv from the trusted catalog or project profile 

114 [resolve_executable(argv[0]), *argv[1:]], 

115 cwd=cwd, 

116 capture_output=True, 

117 check=False, 

118 timeout=timeout, 

119 stdin=subprocess.DEVNULL, 

120 env={**os.environ, "GIT_TERMINAL_PROMPT": "0"}, 

121 ) 

122 except subprocess.TimeoutExpired: 

123 return None, f"timed out after {timeout} s" 

124 except OSError as error: 

125 return None, str(error) 

126 text = (result.stdout + b"\n" + result.stderr).decode("utf-8", "replace").strip() 

127 return result.returncode, text 

128 

129 

130def detect( 

131 item: CapabilityDetection, workspace: Path, timeout: int, runner: Runner = run_detection 

132) -> DetectionResult: 

133 if item.path is not None: 

134 target = workspace / item.path 

135 return DetectionResult(f"path {item.path}", target.exists()) 

136 argv = tuple(item.command or ()) 

137 code, output = runner(argv, workspace, timeout) 

138 available = code == 0 

139 if available and item.expect is not None: 

140 available = re.search(item.expect, output, re.MULTILINE) is not None 

141 return DetectionResult(" ".join(argv), available, output[-_OUTPUT_CHARS:]) 

142 

143 

144def capability_statuses( 

145 declared: list[tuple[str, ProfileVerification]], 

146 workspace: Path, 

147 validator_ids: set[str], 

148 *, 

149 run_detections: bool, 

150 timeout: int, 

151 runner: Runner = run_detection, 

152) -> list[CapabilityStatus]: 

153 """Every capability of the selected profiles with its availability: its validators are 

154 configured (any of them) and every detection passes. Without ``run_detections`` a 

155 detection command is not run: the capability is planned as available and marked not 

156 checked. The certification never credits a rung from a plan, only from the validators' 

157 results.""" 

158 statuses: list[CapabilityStatus] = [] 

159 for profile_id, verification in declared: 

160 for capability in verification.capabilities or (): 

161 statuses.append( 

162 _status( 

163 profile_id, 

164 capability, 

165 workspace, 

166 validator_ids, 

167 run_detections, 

168 timeout, 

169 runner, 

170 ) 

171 ) 

172 return statuses 

173 

174 

175def _status( 

176 profile_id: str, 

177 capability: VerificationCapability, 

178 workspace: Path, 

179 validator_ids: set[str], 

180 run_detections: bool, 

181 timeout: int, 

182 runner: Runner, 

183) -> CapabilityStatus: 

184 detections: list[DetectionResult] = [] 

185 for item in capability.detect or (): 

186 if item.command is not None and not run_detections: 

187 detections.append( 

188 DetectionResult( 

189 " ".join(item.command), 

190 True, 

191 "not run (capabilityDetection is off)", 

192 checked=False, 

193 ) 

194 ) 

195 else: 

196 detections.append(detect(item, workspace, timeout, runner)) 

197 validators = tuple(capability.validators or ()) 

198 configured = not validators or any(item in validator_ids for item in validators) 

199 available = configured and all(item.available for item in detections) 

200 return CapabilityStatus( 

201 profile_id, 

202 capability.level.value, 

203 capability.provides, 

204 validators, 

205 bool(capability.probes), 

206 available, 

207 tuple(detections), 

208 capability.fallback, 

209 ) 

210 

211 

212__all__ = [ 

213 "ANY_PROFILE", 

214 "CapabilityStatus", 

215 "DetectionResult", 

216 "capability_statuses", 

217 "detect", 

218 "load_catalog", 

219 "profile_verification", 

220 "run_detection", 

221]