Coverage for src/governed_harness/checks/secrets.py: 99%

146 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Context-aware secret detection over the added lines of a ChangeSet diff. 

2 

3A plain pattern scanner either drowns a reviewer in test fixtures (``password="test"``) or misses 

4the secret an agent copied from the task statement into a test. This check weighs where the line 

5lives (test or production path), whether the value is an obvious dummy, and whether the value was 

6disclosed in the task text: a declared value is critical wherever it appears. 

7 

8The value itself is never reported: messages show its first two characters and its length.""" 

9 

10from __future__ import annotations 

11 

12import re 

13from collections.abc import Iterator, Sequence 

14 

15from governed_harness.domain.enums import FindingSeverity 

16 

17from .model import DiffFile, Issue, is_test_path 

18 

19CATEGORY = "security" 

20_CREDENTIAL_RULE = "secrets.credential" 

21 

22# ----- values -------------------------------------------------------------------------------- 

23_DUMMY_WORDS = frozenset( 

24 { 

25 "test", 

26 "testing", 

27 "dummy", 

28 "example", 

29 "changeme", 

30 "secret", 

31 "password", 

32 "pass", 

33 "pwd", 

34 "fake", 

35 "placeholder", 

36 "sample", 

37 "foo", 

38 "bar", 

39 "baz", 

40 "xxx", 

41 "redacted", 

42 "none", 

43 "null", 

44 } 

45) 

46_DUMMY_PREFIX = re.compile(r"(?:test|dummy|fake|example|sample|mock)[\w.-]*") 

47_PLACEHOLDER = ( 

48 re.compile(r"\$\{[^}]*\}|\{\{[^}]*\}\}|\{\}|%(?:\([^)]*\))?[sdr]"), 

49 re.compile(r"\{[A-Za-z_][\w.]*(?:![rsa])?(?::[^{}]*)?\}"), 

50) 

51_MIN_SECRET_LENGTH = 6 

52 

53 

54def _literal(tag: str = "") -> str: 

55 """A quoted string literal pattern; ``tag`` prefixes its group names so the pattern can be 

56 used several times in one expression.""" 

57 return ( 

58 rf"(?P<{tag}prefix>[rRbBuUfF]{{0,2}})(?P<{tag}quote>[\"'`])" 

59 rf"(?P<{tag}value>(?:\\.|(?!(?P={tag}quote)).)*)(?P={tag}quote)" 

60 ) 

61 

62 

63_LITERAL = _literal() 

64_LITERAL_RE = re.compile(_LITERAL) 

65 

66 

67def is_dummy_value(value: str) -> bool: 

68 """Whether a string literal is an obvious placeholder rather than a usable secret: a dummy 

69 word, a ``test…``/``fake…`` style value, a mask (``xxx``, ``***``), a template (``<token>``, 

70 ``${VAR}``, ``{name}``) or a value too short to be a credential.""" 

71 stripped = value.strip() 

72 lowered = stripped.lower() 

73 return ( 

74 lowered in _DUMMY_WORDS 

75 or _DUMMY_PREFIX.fullmatch(lowered) is not None 

76 or re.fullmatch(r"x+", lowered) is not None 

77 or re.fullmatch(r"\*+", stripped) is not None 

78 or re.fullmatch(r"<[^<>]*>", stripped) is not None 

79 or any(pattern.search(stripped) for pattern in _PLACEHOLDER) 

80 or len(stripped) < _MIN_SECRET_LENGTH 

81 ) 

82 

83 

84def _is_template(value: str, prefix: str) -> bool: 

85 """A value built at run time (f-string, ``${VAR}``, ``<token>``) holds no secret itself.""" 

86 stripped = value.strip() 

87 return ( 

88 ("f" in prefix.lower() and "{" in stripped) 

89 or any(pattern.search(stripped) for pattern in _PLACEHOLDER) 

90 or re.fullmatch(r"<[^<>]*>", stripped) is not None 

91 ) 

92 

93 

94def _mask(value: str) -> str: 

95 return f"'{value[:2]}…' ({len(value)} chars)" 

96 

97 

98# ----- declared values ----------------------------------------------------------------------- 

99_SECRET_WORDING = re.compile( 

100 r"pass(?:word|wd|phrase)?|pwd|secret|token|api[ _-]?key|\bkey\b|credential", re.IGNORECASE 

101) 

102_QUOTED_IN_TEXT = re.compile(r"\"([^\"\n]+)\"|'([^'\n]+)'|`([^`\n]+)`|“([^”\n]+)”") 

103_WORDED_IN_TEXT = re.compile( 

104 r"(?:password|passwd|passphrase|pwd|secret|token|api[ _-]?key|key|credential)s?\b" 

105 r"(?:\s+(?:is|=|:)|\s*[:=])\s*(?P<value>[^\s\"'`“”,;]+)", 

106 re.IGNORECASE, 

107) 

108_CONTEXT_CHARS = 80 

109 

110 

111def _declared_values(task_text: str) -> set[str]: 

112 """Values the task statement discloses as secrets. 

113 

114 A quoted or backticked string counts only when secret wording precedes it nearby: a task 

115 quotes endpoints and identifiers too, and those must not become critical findings. An 

116 unquoted value after the wording counts unless it is a plain lowercase word ("the token is 

117 stored ...").""" 

118 values: set[str] = set() 

119 for match in _QUOTED_IN_TEXT.finditer(task_text): 

120 value = next(group for group in match.groups() if group is not None).strip() 

121 context = task_text[max(0, match.start() - _CONTEXT_CHARS) : match.start()] 

122 context = context.rsplit("\n", 1)[-1] 

123 if len(value) >= _MIN_SECRET_LENGTH and _SECRET_WORDING.search(context): 

124 values.add(value) 

125 for match in _WORDED_IN_TEXT.finditer(task_text): 

126 value = match.group("value").rstrip(".)]") 

127 if len(value) >= _MIN_SECRET_LENGTH and not (value.isalpha() and value.islower()): 

128 values.add(value) 

129 return values 

130 

131 

132# ----- line patterns ------------------------------------------------------------------------- 

133_CREDENTIAL_FORMAT = re.compile( 

134 r"\bgh[oprsu]_[A-Za-z0-9]{30,}|\bgithub_pat_[A-Za-z0-9_]{20,}|\bAKIA[0-9A-Z]{16}\b" 

135 r"|\bsk-[A-Za-z0-9]{20,}|\bxox[baprs]-[A-Za-z0-9-]+" 

136 r"|-----BEGIN (?:RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----" 

137) 

138_SECRET_NAME = re.compile( 

139 r"password|passwd|pwd|secret|token|api_key|apikey|credential|private_key", re.IGNORECASE 

140) 

141_NOT_A_SECRET_NAME = re.compile( 

142 r"(?:^|_)(?:url|uri|name|field|type|env|var|path|file|header|prefix|label|message|msg" 

143 r"|endpoint|length|len|policy|pattern|regex|format)s?$|tokeniz", 

144 re.IGNORECASE, 

145) 

146_ASSIGNMENT = re.compile( 

147 r"(?<![\w.\"'])(?P<name>[A-Za-z_][\w]*)\s*(?::\s*[\w\[\], |.]+?)?\s*=(?!=)\s*" + _LITERAL 

148) 

149_KEYED = re.compile(r"(?P<q>[\"']?)(?P<name>[A-Za-z_][\w-]*)(?P=q)\s*:\s*" + _LITERAL) 

150_AUTH_CALL = re.compile( 

151 r"(?<![\w])(?P<function>[\w.]*?(?:login|register|authenticate|set_password|hash_password" 

152 r"|verify_password|create_user|signup|sign_up)\w*)\s*\(", 

153 re.IGNORECASE, 

154) 

155_ENVIRONMENT = re.compile( 

156 r"os\.environ\[\s*[\"'](?P<k1>[^\"']+)[\"']\s*\]\s*=(?!=)\s*" 

157 + _literal("a") 

158 + r"|os\.environ\.setdefault\(\s*[\"'](?P<k2>[^\"']+)[\"']\s*,\s*" 

159 + _literal("b") 

160 + r"|\b(?:os\.putenv|\w+\.setenv)\(\s*[\"'](?P<k3>[^\"']+)[\"']\s*,\s*" 

161 + _literal("c") 

162 + r"|process\.env(?:\.(?P<k4>\w+)|\[\s*[\"'](?P<k5>[^\"']+)[\"']\s*\])\s*=(?!=)\s*" 

163 + _literal("d") 

164) 

165_ENVIRONMENT_KEY = re.compile(r"PASS|SECRET|TOKEN|KEY|PWD|CREDENTIAL") 

166_CONSTANT = re.compile( 

167 r"^\s*(?:export\s+)?(?:(?:const|let|var|final)\s+)?(?P<name>[A-Z][A-Z0-9_]*)" 

168 r"\s*(?::\s*[\w\[\], |.]+?)?\s*=(?!=)\s*" + _LITERAL 

169) 

170_CONSTANT_WORD = re.compile(r"PASSWORD|PASSWD|PASSPHRASE|SECRET|TOKEN|CREDENTIAL") 

171_CONSTANT_SEGMENTS = frozenset( 

172 {"PW", "PWD", "PASS", "KEY", "KEYS", "APIKEY", "PRIVATEKEY", "CRED", "CREDS"} 

173) 

174_EMAIL = re.compile(r"[^@\s]+@[^@\s]+\.\w+") 

175_IDENTIFIER_PATH = re.compile(r"[a-z_]\w*(?:\.[a-z_]\w*)+") 

176_IDENTIFIER = re.compile(r"[a-z]+(?:_[a-z0-9]+)+|[a-z]+(?:[A-Z][a-z0-9]*)+") 

177 

178 

179_ENVIRONMENT_GROUPS = (("k1", "a"), ("k2", "b"), ("k3", "c"), ("k4", "d"), ("k5", "d")) 

180 

181 

182def _secret_name(name: str) -> bool: 

183 return _SECRET_NAME.search(name) is not None and _NOT_A_SECRET_NAME.search(name) is None 

184 

185 

186def _constant_name(name: str) -> bool: 

187 if _NOT_A_SECRET_NAME.search(name): 

188 return False 

189 return any( 

190 _CONSTANT_WORD.search(segment) or segment in _CONSTANT_SEGMENTS 

191 for segment in name.split("_") 

192 ) 

193 

194 

195def _not_a_value(value: str) -> bool: 

196 """URLs without user info, absolute paths, e-mail addresses and dotted identifiers are 

197 configuration, not credentials.""" 

198 return ( 

199 (("://" in value) and "@" not in value) 

200 or value.startswith("/") 

201 or _EMAIL.fullmatch(value) is not None 

202 or _IDENTIFIER_PATH.fullmatch(value) is not None 

203 ) 

204 

205 

206def _field_reference(value: str) -> bool: 

207 """An identifier that itself names a secret (``"input_tokens"``, ``"userPassword"``) maps 

208 one field name to another; it is not a credential.""" 

209 return _IDENTIFIER.fullmatch(value) is not None and _SECRET_NAME.search(value) is not None 

210 

211 

212def _credential_literals(text: str) -> Iterator[tuple[str, str, str]]: 

213 """``(name, prefix, value)`` for every credential-named assignment, keyword or key.""" 

214 for pattern in (_ASSIGNMENT, _KEYED): 

215 for match in pattern.finditer(text): 

216 if _secret_name(match.group("name")) and not _field_reference(match.group("value")): 

217 yield match.group("name"), match.group("prefix"), match.group("value") 

218 

219 

220def _auth_call_literals(text: str) -> Iterator[tuple[str, str, str]]: 

221 """``(function, prefix, value)`` for positional string literals of an authentication call.""" 

222 if re.match(r"^\s*(?:async\s+)?(?:def|function)\b", text): 

223 return 

224 for call in _AUTH_CALL.finditer(text): 

225 for literal in _LITERAL_RE.finditer(text, call.end()): 

226 before = text[call.end() : literal.start()].rstrip() 

227 if before.endswith(("=", ":")): 

228 continue 

229 value = literal.group("value") 

230 if not _not_a_value(value): 

231 yield call.group("function"), literal.group("prefix"), value 

232 

233 

234def _environment_literals(text: str) -> Iterator[tuple[str, str, str]]: 

235 for match in _ENVIRONMENT.finditer(text): 

236 for key_group, letter in _ENVIRONMENT_GROUPS: 236 ↛ 235line 236 didn't jump to line 235 because the loop on line 236 didn't complete

237 key = match.group(key_group) 

238 if key is not None: 

239 yield key, match.group(f"{letter}prefix"), match.group(f"{letter}value") 

240 break 

241 

242 

243class _Collector: 

244 """Issues keyed by (path, line, rule): one per rule and line, first wins.""" 

245 

246 def __init__(self) -> None: 

247 self.issues: dict[tuple[str, int, str], Issue] = {} 

248 

249 def add( 

250 self, 

251 rule: str, 

252 severity: FindingSeverity, 

253 message: str, 

254 path: str, 

255 line: int, 

256 recommendation: str, 

257 ) -> None: 

258 self.issues.setdefault( 

259 (path, line, rule), 

260 Issue( 

261 rule_id=rule, 

262 severity=severity, 

263 message=message, 

264 path=path, 

265 line=line, 

266 category=CATEGORY, 

267 recommendation=recommendation, 

268 ), 

269 ) 

270 

271 

272_MOVE_TO_SECRET_STORE = "Read the value from the environment or a secret store; never commit it." # nosec B105 - remediation text, not a secret 

273_TEST_RECOMMENDATION = "Use an obvious dummy value or generate the credential in a fixture." 

274 

275 

276def _graded( 

277 collector: _Collector, 

278 rule: str, 

279 what: str, 

280 value: str, 

281 prefix: str, 

282 path: str, 

283 line: int, 

284 *, 

285 in_tests: bool, 

286) -> None: 

287 """Record a credential under the path context: critical in production, medium in tests, 

288 informational for a dummy value in tests.""" 

289 if not value.strip() or _is_template(value, prefix): 

290 return 

291 message = f"{what} with a literal value {_mask(value)}" 

292 if not in_tests: 

293 collector.add(rule, FindingSeverity.CRITICAL, message, path, line, _MOVE_TO_SECRET_STORE) 

294 elif is_dummy_value(value): 

295 collector.add( 

296 "secrets.test-dummy", 

297 FindingSeverity.INFO, 

298 message + " (dummy test value)", 

299 path, 

300 line, 

301 _TEST_RECOMMENDATION, 

302 ) 

303 else: 

304 test_rule = "secrets.test-credential" if rule == _CREDENTIAL_RULE else rule 

305 collector.add(test_rule, FindingSeverity.MEDIUM, message, path, line, _TEST_RECOMMENDATION) 

306 

307 

308def scan_secrets(diff: Sequence[DiffFile], *, task_text: str = "") -> list[Issue]: 

309 """Secret issues on the added lines of ``diff``, in diff and line order.""" 

310 declared = _declared_values(task_text) 

311 collector = _Collector() 

312 for file in diff: 

313 if file.is_deleted: 

314 continue 

315 path = file.path 

316 in_tests = is_test_path(path) 

317 for diff_line in file.added: 

318 _scan_line(collector, declared, diff_line.text, path, diff_line.number, in_tests) 

319 return list(collector.issues.values()) 

320 

321 

322def _scan_line( 

323 collector: _Collector, 

324 declared: set[str], 

325 text: str, 

326 path: str, 

327 number: int, 

328 in_tests: bool, 

329) -> None: 

330 """Record the secret issues of one added line, in the order the rules run.""" 

331 _declared_value(collector, declared, text, path, number) 

332 credential_format = _CREDENTIAL_FORMAT.search(text) 

333 if credential_format: 

334 collector.add( 

335 "secrets.credential-format", 

336 FindingSeverity.CRITICAL, 

337 f"A known credential format appears {_mask(credential_format.group(0))}", 

338 path, 

339 number, 

340 "Revoke the credential and remove it from the change.", 

341 ) 

342 named = _named_credentials(collector, text, path, number, in_tests) 

343 constant = None if named else _CONSTANT.match(text) 

344 if constant and _secret_constant(constant): 

345 collector.add( 

346 "secrets.constant", 

347 FindingSeverity.MEDIUM if in_tests else FindingSeverity.CRITICAL, 

348 f"constant {constant.group('name')!r} holds a literal {_mask(constant.group('value'))}", 

349 path, 

350 number, 

351 _TEST_RECOMMENDATION if in_tests else _MOVE_TO_SECRET_STORE, 

352 ) 

353 

354 

355def _declared_value( 

356 collector: _Collector, declared: set[str], text: str, path: str, number: int 

357) -> None: 

358 """Record the first value the task discloses as a secret that the line contains.""" 

359 for value in sorted(declared): 

360 if value in text: 360 ↛ 359line 360 didn't jump to line 359 because the condition on line 360 was always true

361 collector.add( 

362 "secrets.declared-value", 

363 FindingSeverity.CRITICAL, 

364 f"A value the task discloses as a secret is written into the code {_mask(value)}", 

365 path, 

366 number, 

367 "Never copy a secret from the task; read it from the environment.", 

368 ) 

369 return 

370 

371 

372def _named_credentials( 

373 collector: _Collector, text: str, path: str, number: int, in_tests: bool 

374) -> bool: 

375 """Grade the credentials the line assigns to a name; whether any name was found.""" 

376 named = False 

377 for name, prefix, value in _credential_literals(text): 

378 named = True 

379 _graded( 

380 collector, 

381 _CREDENTIAL_RULE, 

382 f"{name!r} is assigned", 

383 value, 

384 prefix, 

385 path, 

386 number, 

387 in_tests=in_tests, 

388 ) 

389 for function, prefix, value in _auth_call_literals(text): 

390 named = True 

391 _graded( 

392 collector, 

393 _CREDENTIAL_RULE, 

394 f"{function}() receives a positional string", 

395 value, 

396 prefix, 

397 path, 

398 number, 

399 in_tests=in_tests, 

400 ) 

401 for key, prefix, value in _environment_literals(text): 

402 if _ENVIRONMENT_KEY.search(key.upper()): 

403 named = True 

404 _graded( 

405 collector, 

406 "secrets.environment-assignment", 

407 f"environment variable {key!r} is set", 

408 value, 

409 prefix, 

410 path, 

411 number, 

412 in_tests=in_tests, 

413 ) 

414 return named 

415 

416 

417def _secret_constant(constant: re.Match[str]) -> bool: 

418 """Whether a constant assignment holds a literal that looks like a real secret.""" 

419 return bool( 

420 _constant_name(constant.group("name")) 

421 and constant.group("value").strip() 

422 and not is_dummy_value(constant.group("value")) 

423 and not _is_template(constant.group("value"), constant.group("prefix")) 

424 and not _not_a_value(constant.group("value")) 

425 )