Coverage for src/governed_harness/validators/command.py: 79%

111 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1from __future__ import annotations 

2 

3import json 

4from datetime import UTC, datetime 

5 

6from governed_harness.domain.enums import ( 

7 ActorType, 

8 ErrorKind, 

9 FindingSeverity, 

10 PhaseId, 

11 ResultStatus, 

12 ValidationKind, 

13) 

14from governed_harness.domain.errors import ConfigurationError 

15from governed_harness.domain.ids import new_id 

16from governed_harness.domain.models import ( 

17 Actor, 

18 Finding, 

19 FindingLocation, 

20 HarnessErrorRecord, 

21 ToolInvocation, 

22 ValidationResult, 

23) 

24from governed_harness.runtime.process_runner import CommandSpec, ProcessResult, find_executable 

25from governed_harness.validators.base import ValidationContext, ValidatorOutput 

26from governed_harness.validators.parsers import MAX_ISSUES, parse_output, report_files 

27 

28# Exits with _MODULE_MISSING_EXIT_CODE when the module named in argv[1] cannot be found. 

29_MODULE_MISSING_EXIT_CODE = 3 

30_MODULE_PROBE = ( 

31 "import importlib.util, sys\n" 

32 "try:\n" 

33 " found = importlib.util.find_spec(sys.argv[1]) is not None\n" 

34 "except Exception:\n" 

35 " found = False\n" 

36 "sys.exit(0 if found else 3)\n" 

37) 

38 

39 

40class CommandValidator: 

41 def __init__(self, validator_id: str) -> None: 

42 self.validator_id = validator_id 

43 

44 def execute(self, context: ValidationContext) -> ValidatorOutput: 

45 actor = Actor( 

46 actor_type=ActorType.TOOL, actor_id=f"validator.{self.validator_id}", version="1" 

47 ) 

48 started = datetime.now(UTC) 

49 availability = self._availability(context, actor) 

50 if availability is not None: 

51 status, reason = availability 

52 evidence = context.artifact_store.put_json( 

53 {"validatorId": self.validator_id, "status": status, "reason": reason}, 

54 metadata={"kind": "validator-availability"}, 

55 ) 

56 result = ValidationResult( 

57 validation_result_id=new_id("validation"), 

58 execution_id=context.execution_id, 

59 validator_id=self.validator_id, 

60 change_set_digest=context.change_set.digest, 

61 status=status, 

62 kind=( 

63 ValidationKind.CONFIGURATION_ERROR 

64 if status in {ResultStatus.BLOCKED, ResultStatus.FAILED} 

65 else ValidationKind.INCONCLUSIVE 

66 ), 

67 mandatory=context.definition.mandatory, 

68 summary=reason, 

69 evidence_refs=(evidence.uri,), 

70 started_at=started, 

71 finished_at=datetime.now(UTC), 

72 provenance=context.provenance.model_copy(update={"actor": actor}), 

73 ) 

74 return ValidatorOutput(result) 

75 command = context.definition.command 

76 if command is None: # not reached: _availability blocks a validator without a command 76 ↛ 77line 76 didn't jump to line 77 because the condition on line 76 was never true

77 raise ConfigurationError(f"validator {self.validator_id} has no configured command") 

78 process = context.process_runner.run( 

79 CommandSpec( 

80 argv=command, 

81 cwd=context.workspace, 

82 timeout_seconds=float(context.definition.timeout_seconds or 900), 

83 allowed_environment=context.definition.pass_env or (), 

84 max_output_bytes=context.max_output_bytes, 

85 ), 

86 actor=actor, 

87 grants=context.grants, 

88 cancellation=context.cancellation, 

89 ) 

90 stdout = context.artifact_store.put( 

91 process.stdout, 

92 media_type="text/plain", 

93 metadata={"validatorId": self.validator_id, "stream": "stdout"}, 

94 ) 

95 stderr = context.artifact_store.put( 

96 process.stderr, 

97 media_type="text/plain", 

98 metadata={"validatorId": self.validator_id, "stream": "stderr"}, 

99 ) 

100 report = context.artifact_store.put_json( 

101 { 

102 "validatorId": self.validator_id, 

103 "argv": list(command), 

104 # The absolute program a bare name resolved to (Windows, F12). 

105 **( 

106 {"executable": process.executable} 

107 if process.executable not in (None, command[0]) 

108 else {} 

109 ), 

110 "exitCode": process.exit_code, 

111 "status": process.status, 

112 "durationMs": process.duration_ms, 

113 "timedOut": process.timed_out, 

114 "cancelled": process.cancelled, 

115 "stdoutRef": stdout.uri, 

116 "stderrRef": stderr.uri, 

117 "stdoutTruncated": process.stdout_truncated, 

118 "stderrTruncated": process.stderr_truncated, 

119 }, 

120 metadata={"kind": "validation-report", "validatorId": self.validator_id}, 

121 ) 

122 tool = ToolInvocation( 

123 invocation_id=new_id("tool"), 

124 execution_id=context.execution_id, 

125 phase_id=PhaseId.VERIFICATION, 

126 actor=actor, 

127 tool_id=self.validator_id, 

128 argv=command, 

129 cwd=".", 

130 started_at=started, 

131 finished_at=datetime.now(UTC), 

132 status=process.status, 

133 exit_code=process.exit_code, 

134 timed_out=process.timed_out, 

135 cancelled=process.cancelled, 

136 stdout_ref=stdout.uri, 

137 stderr_ref=stderr.uri, 

138 provenance=context.provenance.model_copy(update={"actor": actor}), 

139 ) 

140 errors: tuple[HarnessErrorRecord, ...] = () 

141 findings: tuple[Finding, ...] = () 

142 if process.status is ResultStatus.PASSED: 

143 kind = ValidationKind.SUCCESS 

144 summary = f"{self.validator_id} passed" 

145 elif process.status is ResultStatus.TIMED_OUT: 145 ↛ 146line 145 didn't jump to line 146 because the condition on line 145 was never true

146 kind = ValidationKind.TOOL_ERROR 

147 summary = f"{self.validator_id} timed out" 

148 errors = ( 

149 HarnessErrorRecord( 

150 error_id=new_id("err"), 

151 kind=ErrorKind.TOOL_ERROR, 

152 message=summary, 

153 actor=actor, 

154 ), 

155 ) 

156 elif process.status is ResultStatus.CANCELLED: 156 ↛ 157line 156 didn't jump to line 157 because the condition on line 156 was never true

157 kind = ValidationKind.INCONCLUSIVE 

158 summary = f"{self.validator_id} was cancelled" 

159 else: 

160 kind = ValidationKind.VALIDATION_FAILURE 

161 summary = f"{self.validator_id} failed with exit code {process.exit_code}" 

162 finding = Finding( 

163 finding_id=new_id("finding"), 

164 execution_id=context.execution_id, 

165 validator_id=self.validator_id, 

166 rule_id=f"{self.validator_id}.failed", 

167 category="validation", 

168 severity=context.definition.failure_severity 

169 or ( 

170 FindingSeverity.HIGH if context.definition.mandatory else FindingSeverity.MEDIUM 

171 ), 

172 message=summary, 

173 location=FindingLocation(), 

174 evidence_refs=(report.uri, stdout.uri, stderr.uri), 

175 recommendation="Inspect the validator output and correct the project before approval.", 

176 introduced=None, 

177 provenance=context.provenance.model_copy(update={"actor": actor}), 

178 ) 

179 findings = (finding, *self._parsed_findings(context, process, finding, actor)) 

180 result = ValidationResult( 

181 validation_result_id=new_id("validation"), 

182 execution_id=context.execution_id, 

183 validator_id=self.validator_id, 

184 change_set_digest=context.change_set.digest, 

185 status=process.status, 

186 kind=kind, 

187 mandatory=context.definition.mandatory, 

188 summary=summary, 

189 finding_ids=tuple(item.finding_id for item in findings), 

190 evidence_refs=(report.uri, stdout.uri, stderr.uri), 

191 tool_invocation_ids=(tool.invocation_id,), 

192 started_at=started, 

193 finished_at=datetime.now(UTC), 

194 errors=errors, 

195 provenance=context.provenance.model_copy(update={"actor": actor}), 

196 ) 

197 return ValidatorOutput(result, findings, (tool,)) 

198 

199 def _parsed_findings( 

200 self, context: ValidationContext, process: ProcessResult, summary: Finding, actor: Actor 

201 ) -> list[Finding]: 

202 """One finding per problem the tool reported, when ``verification.outputParsers`` is on. 

203 

204 Errors keep the severity of the summary finding (so the gate outcome is the one the 

205 summary already decides) and warnings are ``LOW``; the rule is 

206 ``<validator>.<tool rule>``.""" 

207 parser = context.definition.parser 

208 if parser is None: 

209 if not context.parse_output: 

210 return [] 

211 parser = "auto" 

212 if parser == "none": 212 ↛ 213line 212 didn't jump to line 213 because the condition on line 212 was never true

213 return [] 

214 reports = [] 

215 for path in report_files(context.definition.command or (), context.workspace): 215 ↛ 216line 215 didn't jump to line 216 because the loop on line 215 never started

216 try: 

217 reports.append(path.read_text(encoding="utf-8", errors="replace")) 

218 except OSError: 

219 continue 

220 issues = parse_output( 

221 process.stdout.decode("utf-8", "replace"), 

222 process.stderr.decode("utf-8", "replace"), 

223 context.workspace, 

224 tuple(reports), 

225 parser, 

226 ) 

227 severities = { 

228 "error": summary.severity, 

229 "warning": FindingSeverity.LOW, 

230 "note": FindingSeverity.INFO, 

231 **(context.definition.severity or {}), 

232 } 

233 findings = [ 

234 Finding( 

235 finding_id=new_id("finding"), 

236 execution_id=context.execution_id, 

237 validator_id=self.validator_id, 

238 rule_id=f"{self.validator_id}.{issue.rule}", 

239 category="validation", 

240 severity=severities[issue.level], 

241 message=context.artifact_store.redactor.redact_configured_text(issue.message), 

242 location=FindingLocation( 

243 path=issue.path, 

244 start_line=issue.line, 

245 end_line=issue.end_line or issue.line, 

246 ), 

247 evidence_refs=summary.evidence_refs, 

248 recommendation=f"Reported by {issue.tool or self.validator_id}; fix it at the " 

249 "location shown.", 

250 introduced=None, 

251 provenance=context.provenance.model_copy(update={"actor": actor}), 

252 ) 

253 for issue in issues[:MAX_ISSUES] 

254 ] 

255 if len(issues) > MAX_ISSUES: 255 ↛ 256line 255 didn't jump to line 256 because the condition on line 255 was never true

256 findings.append( 

257 Finding( 

258 finding_id=new_id("finding"), 

259 execution_id=context.execution_id, 

260 validator_id=self.validator_id, 

261 rule_id=f"{self.validator_id}.more-issues", 

262 category="validation", 

263 severity=FindingSeverity.INFO, 

264 message=f"{len(issues) - MAX_ISSUES} more reported problem(s) were not " 

265 "recorded as findings; see the stored output", 

266 location=FindingLocation(), 

267 evidence_refs=summary.evidence_refs, 

268 provenance=context.provenance.model_copy(update={"actor": actor}), 

269 ) 

270 ) 

271 return findings 

272 

273 def _availability( 

274 self, context: ValidationContext, actor: Actor 

275 ) -> tuple[ResultStatus, str] | None: 

276 definition = context.definition 

277 missing_command = context.missing_command_status or ResultStatus.BLOCKED 

278 missing_script = context.missing_script_status or ResultStatus.BLOCKED 

279 if definition.command is None: 279 ↛ 280line 279 didn't jump to line 280 because the condition on line 279 was never true

280 status = ResultStatus.BLOCKED if definition.mandatory else ResultStatus.NOT_APPLICABLE 

281 return status, f"validator {self.validator_id} has no configured command" 

282 if definition.script: 

283 undefined = _undefined_script(context, missing_script) 

284 if undefined is not None: 284 ↛ 285line 284 didn't jump to line 285 because the condition on line 284 was never true

285 return undefined 

286 argv0 = definition.command[0] 

287 if find_executable(argv0) is None: 287 ↛ 288line 287 didn't jump to line 288 because the condition on line 287 was never true

288 status = missing_command if definition.mandatory else ResultStatus.NOT_APPLICABLE 

289 return status, f"executable {argv0!r} is not available" 

290 if len(definition.command) >= 3 and definition.command[1] == "-m": 

291 return self._missing_module( 

292 context, actor, argv0, definition.command[2], missing_command 

293 ) 

294 return None 

295 

296 def _missing_module( 

297 self, 

298 context: ValidationContext, 

299 actor: Actor, 

300 argv0: str, 

301 module: str, 

302 missing_command: ResultStatus, 

303 ) -> tuple[ResultStatus, str] | None: 

304 """The status of a ``python -m module`` command whose module is not installed.""" 

305 definition = context.definition 

306 if self._module_available(context, actor, argv0, module): 

307 return None 

308 status = missing_command if definition.mandatory else ResultStatus.NOT_APPLICABLE 

309 kind = "mandatory" if definition.mandatory else "optional" 

310 return status, f"{kind} Python module {module!r} is not installed for {argv0!r}" 

311 

312 @staticmethod 

313 def _module_available( 

314 context: ValidationContext, actor: Actor, interpreter: str, module: str 

315 ) -> bool: 

316 """Ask the interpreter that will run the command, not the harness's own, whether the 

317 module can be imported. The probe goes through the governed runner, so it needs the same 

318 process capability as the validator itself.""" 

319 probe = context.process_runner.run( 

320 CommandSpec( 

321 argv=(interpreter, "-c", _MODULE_PROBE, module), 

322 cwd=context.workspace, 

323 timeout_seconds=60.0, 

324 max_output_bytes=4096, 

325 ), 

326 actor=actor, 

327 grants=context.grants, 

328 cancellation=context.cancellation, 

329 ) 

330 return probe.exit_code != _MODULE_MISSING_EXIT_CODE 

331 

332 

333def _undefined_script( 

334 context: ValidationContext, missing_script: ResultStatus 

335) -> tuple[ResultStatus, str] | None: 

336 """The status of a validator whose package script ``package.json`` does not define.""" 

337 definition = context.definition 

338 package_json = context.workspace / "package.json" 

339 try: 

340 package = json.loads(package_json.read_text(encoding="utf-8")) 

341 scripts = package.get("scripts", {}) 

342 except (OSError, json.JSONDecodeError): 

343 scripts = {} 

344 if definition.script not in scripts: 344 ↛ 345line 344 didn't jump to line 345 because the condition on line 344 was never true

345 status = missing_script if definition.mandatory else ResultStatus.NOT_APPLICABLE 

346 return status, f"package script {definition.script!r} is not defined" 

347 return None