Coverage for src/governed_harness/checks/secrets.py: 99%
146 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Context-aware secret detection over the added lines of a ChangeSet diff.
3A plain pattern scanner either drowns a reviewer in test fixtures (``password="test"``) or misses
4the secret an agent copied from the task statement into a test. This check weighs where the line
5lives (test or production path), whether the value is an obvious dummy, and whether the value was
6disclosed in the task text: a declared value is critical wherever it appears.
8The value itself is never reported: messages show its first two characters and its length."""
10from __future__ import annotations
12import re
13from collections.abc import Iterator, Sequence
15from governed_harness.domain.enums import FindingSeverity
17from .model import DiffFile, Issue, is_test_path
19CATEGORY = "security"
20_CREDENTIAL_RULE = "secrets.credential"
22# ----- values --------------------------------------------------------------------------------
23_DUMMY_WORDS = frozenset(
24 {
25 "test",
26 "testing",
27 "dummy",
28 "example",
29 "changeme",
30 "secret",
31 "password",
32 "pass",
33 "pwd",
34 "fake",
35 "placeholder",
36 "sample",
37 "foo",
38 "bar",
39 "baz",
40 "xxx",
41 "redacted",
42 "none",
43 "null",
44 }
45)
46_DUMMY_PREFIX = re.compile(r"(?:test|dummy|fake|example|sample|mock)[\w.-]*")
47_PLACEHOLDER = (
48 re.compile(r"\$\{[^}]*\}|\{\{[^}]*\}\}|\{\}|%(?:\([^)]*\))?[sdr]"),
49 re.compile(r"\{[A-Za-z_][\w.]*(?:![rsa])?(?::[^{}]*)?\}"),
50)
51_MIN_SECRET_LENGTH = 6
54def _literal(tag: str = "") -> str:
55 """A quoted string literal pattern; ``tag`` prefixes its group names so the pattern can be
56 used several times in one expression."""
57 return (
58 rf"(?P<{tag}prefix>[rRbBuUfF]{{0,2}})(?P<{tag}quote>[\"'`])"
59 rf"(?P<{tag}value>(?:\\.|(?!(?P={tag}quote)).)*)(?P={tag}quote)"
60 )
63_LITERAL = _literal()
64_LITERAL_RE = re.compile(_LITERAL)
67def is_dummy_value(value: str) -> bool:
68 """Whether a string literal is an obvious placeholder rather than a usable secret: a dummy
69 word, a ``test…``/``fake…`` style value, a mask (``xxx``, ``***``), a template (``<token>``,
70 ``${VAR}``, ``{name}``) or a value too short to be a credential."""
71 stripped = value.strip()
72 lowered = stripped.lower()
73 return (
74 lowered in _DUMMY_WORDS
75 or _DUMMY_PREFIX.fullmatch(lowered) is not None
76 or re.fullmatch(r"x+", lowered) is not None
77 or re.fullmatch(r"\*+", stripped) is not None
78 or re.fullmatch(r"<[^<>]*>", stripped) is not None
79 or any(pattern.search(stripped) for pattern in _PLACEHOLDER)
80 or len(stripped) < _MIN_SECRET_LENGTH
81 )
84def _is_template(value: str, prefix: str) -> bool:
85 """A value built at run time (f-string, ``${VAR}``, ``<token>``) holds no secret itself."""
86 stripped = value.strip()
87 return (
88 ("f" in prefix.lower() and "{" in stripped)
89 or any(pattern.search(stripped) for pattern in _PLACEHOLDER)
90 or re.fullmatch(r"<[^<>]*>", stripped) is not None
91 )
94def _mask(value: str) -> str:
95 return f"'{value[:2]}…' ({len(value)} chars)"
98# ----- declared values -----------------------------------------------------------------------
99_SECRET_WORDING = re.compile(
100 r"pass(?:word|wd|phrase)?|pwd|secret|token|api[ _-]?key|\bkey\b|credential", re.IGNORECASE
101)
102_QUOTED_IN_TEXT = re.compile(r"\"([^\"\n]+)\"|'([^'\n]+)'|`([^`\n]+)`|“([^”\n]+)”")
103_WORDED_IN_TEXT = re.compile(
104 r"(?:password|passwd|passphrase|pwd|secret|token|api[ _-]?key|key|credential)s?\b"
105 r"(?:\s+(?:is|=|:)|\s*[:=])\s*(?P<value>[^\s\"'`“”,;]+)",
106 re.IGNORECASE,
107)
108_CONTEXT_CHARS = 80
111def _declared_values(task_text: str) -> set[str]:
112 """Values the task statement discloses as secrets.
114 A quoted or backticked string counts only when secret wording precedes it nearby: a task
115 quotes endpoints and identifiers too, and those must not become critical findings. An
116 unquoted value after the wording counts unless it is a plain lowercase word ("the token is
117 stored ...")."""
118 values: set[str] = set()
119 for match in _QUOTED_IN_TEXT.finditer(task_text):
120 value = next(group for group in match.groups() if group is not None).strip()
121 context = task_text[max(0, match.start() - _CONTEXT_CHARS) : match.start()]
122 context = context.rsplit("\n", 1)[-1]
123 if len(value) >= _MIN_SECRET_LENGTH and _SECRET_WORDING.search(context):
124 values.add(value)
125 for match in _WORDED_IN_TEXT.finditer(task_text):
126 value = match.group("value").rstrip(".)]")
127 if len(value) >= _MIN_SECRET_LENGTH and not (value.isalpha() and value.islower()):
128 values.add(value)
129 return values
132# ----- line patterns -------------------------------------------------------------------------
133_CREDENTIAL_FORMAT = re.compile(
134 r"\bgh[oprsu]_[A-Za-z0-9]{30,}|\bgithub_pat_[A-Za-z0-9_]{20,}|\bAKIA[0-9A-Z]{16}\b"
135 r"|\bsk-[A-Za-z0-9]{20,}|\bxox[baprs]-[A-Za-z0-9-]+"
136 r"|-----BEGIN (?:RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----"
137)
138_SECRET_NAME = re.compile(
139 r"password|passwd|pwd|secret|token|api_key|apikey|credential|private_key", re.IGNORECASE
140)
141_NOT_A_SECRET_NAME = re.compile(
142 r"(?:^|_)(?:url|uri|name|field|type|env|var|path|file|header|prefix|label|message|msg"
143 r"|endpoint|length|len|policy|pattern|regex|format)s?$|tokeniz",
144 re.IGNORECASE,
145)
146_ASSIGNMENT = re.compile(
147 r"(?<![\w.\"'])(?P<name>[A-Za-z_][\w]*)\s*(?::\s*[\w\[\], |.]+?)?\s*=(?!=)\s*" + _LITERAL
148)
149_KEYED = re.compile(r"(?P<q>[\"']?)(?P<name>[A-Za-z_][\w-]*)(?P=q)\s*:\s*" + _LITERAL)
150_AUTH_CALL = re.compile(
151 r"(?<![\w])(?P<function>[\w.]*?(?:login|register|authenticate|set_password|hash_password"
152 r"|verify_password|create_user|signup|sign_up)\w*)\s*\(",
153 re.IGNORECASE,
154)
155_ENVIRONMENT = re.compile(
156 r"os\.environ\[\s*[\"'](?P<k1>[^\"']+)[\"']\s*\]\s*=(?!=)\s*"
157 + _literal("a")
158 + r"|os\.environ\.setdefault\(\s*[\"'](?P<k2>[^\"']+)[\"']\s*,\s*"
159 + _literal("b")
160 + r"|\b(?:os\.putenv|\w+\.setenv)\(\s*[\"'](?P<k3>[^\"']+)[\"']\s*,\s*"
161 + _literal("c")
162 + r"|process\.env(?:\.(?P<k4>\w+)|\[\s*[\"'](?P<k5>[^\"']+)[\"']\s*\])\s*=(?!=)\s*"
163 + _literal("d")
164)
165_ENVIRONMENT_KEY = re.compile(r"PASS|SECRET|TOKEN|KEY|PWD|CREDENTIAL")
166_CONSTANT = re.compile(
167 r"^\s*(?:export\s+)?(?:(?:const|let|var|final)\s+)?(?P<name>[A-Z][A-Z0-9_]*)"
168 r"\s*(?::\s*[\w\[\], |.]+?)?\s*=(?!=)\s*" + _LITERAL
169)
170_CONSTANT_WORD = re.compile(r"PASSWORD|PASSWD|PASSPHRASE|SECRET|TOKEN|CREDENTIAL")
171_CONSTANT_SEGMENTS = frozenset(
172 {"PW", "PWD", "PASS", "KEY", "KEYS", "APIKEY", "PRIVATEKEY", "CRED", "CREDS"}
173)
174_EMAIL = re.compile(r"[^@\s]+@[^@\s]+\.\w+")
175_IDENTIFIER_PATH = re.compile(r"[a-z_]\w*(?:\.[a-z_]\w*)+")
176_IDENTIFIER = re.compile(r"[a-z]+(?:_[a-z0-9]+)+|[a-z]+(?:[A-Z][a-z0-9]*)+")
179_ENVIRONMENT_GROUPS = (("k1", "a"), ("k2", "b"), ("k3", "c"), ("k4", "d"), ("k5", "d"))
182def _secret_name(name: str) -> bool:
183 return _SECRET_NAME.search(name) is not None and _NOT_A_SECRET_NAME.search(name) is None
186def _constant_name(name: str) -> bool:
187 if _NOT_A_SECRET_NAME.search(name):
188 return False
189 return any(
190 _CONSTANT_WORD.search(segment) or segment in _CONSTANT_SEGMENTS
191 for segment in name.split("_")
192 )
195def _not_a_value(value: str) -> bool:
196 """URLs without user info, absolute paths, e-mail addresses and dotted identifiers are
197 configuration, not credentials."""
198 return (
199 (("://" in value) and "@" not in value)
200 or value.startswith("/")
201 or _EMAIL.fullmatch(value) is not None
202 or _IDENTIFIER_PATH.fullmatch(value) is not None
203 )
206def _field_reference(value: str) -> bool:
207 """An identifier that itself names a secret (``"input_tokens"``, ``"userPassword"``) maps
208 one field name to another; it is not a credential."""
209 return _IDENTIFIER.fullmatch(value) is not None and _SECRET_NAME.search(value) is not None
212def _credential_literals(text: str) -> Iterator[tuple[str, str, str]]:
213 """``(name, prefix, value)`` for every credential-named assignment, keyword or key."""
214 for pattern in (_ASSIGNMENT, _KEYED):
215 for match in pattern.finditer(text):
216 if _secret_name(match.group("name")) and not _field_reference(match.group("value")):
217 yield match.group("name"), match.group("prefix"), match.group("value")
220def _auth_call_literals(text: str) -> Iterator[tuple[str, str, str]]:
221 """``(function, prefix, value)`` for positional string literals of an authentication call."""
222 if re.match(r"^\s*(?:async\s+)?(?:def|function)\b", text):
223 return
224 for call in _AUTH_CALL.finditer(text):
225 for literal in _LITERAL_RE.finditer(text, call.end()):
226 before = text[call.end() : literal.start()].rstrip()
227 if before.endswith(("=", ":")):
228 continue
229 value = literal.group("value")
230 if not _not_a_value(value):
231 yield call.group("function"), literal.group("prefix"), value
234def _environment_literals(text: str) -> Iterator[tuple[str, str, str]]:
235 for match in _ENVIRONMENT.finditer(text):
236 for key_group, letter in _ENVIRONMENT_GROUPS: 236 ↛ 235line 236 didn't jump to line 235 because the loop on line 236 didn't complete
237 key = match.group(key_group)
238 if key is not None:
239 yield key, match.group(f"{letter}prefix"), match.group(f"{letter}value")
240 break
243class _Collector:
244 """Issues keyed by (path, line, rule): one per rule and line, first wins."""
246 def __init__(self) -> None:
247 self.issues: dict[tuple[str, int, str], Issue] = {}
249 def add(
250 self,
251 rule: str,
252 severity: FindingSeverity,
253 message: str,
254 path: str,
255 line: int,
256 recommendation: str,
257 ) -> None:
258 self.issues.setdefault(
259 (path, line, rule),
260 Issue(
261 rule_id=rule,
262 severity=severity,
263 message=message,
264 path=path,
265 line=line,
266 category=CATEGORY,
267 recommendation=recommendation,
268 ),
269 )
272_MOVE_TO_SECRET_STORE = "Read the value from the environment or a secret store; never commit it." # nosec B105 - remediation text, not a secret
273_TEST_RECOMMENDATION = "Use an obvious dummy value or generate the credential in a fixture."
276def _graded(
277 collector: _Collector,
278 rule: str,
279 what: str,
280 value: str,
281 prefix: str,
282 path: str,
283 line: int,
284 *,
285 in_tests: bool,
286) -> None:
287 """Record a credential under the path context: critical in production, medium in tests,
288 informational for a dummy value in tests."""
289 if not value.strip() or _is_template(value, prefix):
290 return
291 message = f"{what} with a literal value {_mask(value)}"
292 if not in_tests:
293 collector.add(rule, FindingSeverity.CRITICAL, message, path, line, _MOVE_TO_SECRET_STORE)
294 elif is_dummy_value(value):
295 collector.add(
296 "secrets.test-dummy",
297 FindingSeverity.INFO,
298 message + " (dummy test value)",
299 path,
300 line,
301 _TEST_RECOMMENDATION,
302 )
303 else:
304 test_rule = "secrets.test-credential" if rule == _CREDENTIAL_RULE else rule
305 collector.add(test_rule, FindingSeverity.MEDIUM, message, path, line, _TEST_RECOMMENDATION)
308def scan_secrets(diff: Sequence[DiffFile], *, task_text: str = "") -> list[Issue]:
309 """Secret issues on the added lines of ``diff``, in diff and line order."""
310 declared = _declared_values(task_text)
311 collector = _Collector()
312 for file in diff:
313 if file.is_deleted:
314 continue
315 path = file.path
316 in_tests = is_test_path(path)
317 for diff_line in file.added:
318 _scan_line(collector, declared, diff_line.text, path, diff_line.number, in_tests)
319 return list(collector.issues.values())
322def _scan_line(
323 collector: _Collector,
324 declared: set[str],
325 text: str,
326 path: str,
327 number: int,
328 in_tests: bool,
329) -> None:
330 """Record the secret issues of one added line, in the order the rules run."""
331 _declared_value(collector, declared, text, path, number)
332 credential_format = _CREDENTIAL_FORMAT.search(text)
333 if credential_format:
334 collector.add(
335 "secrets.credential-format",
336 FindingSeverity.CRITICAL,
337 f"A known credential format appears {_mask(credential_format.group(0))}",
338 path,
339 number,
340 "Revoke the credential and remove it from the change.",
341 )
342 named = _named_credentials(collector, text, path, number, in_tests)
343 constant = None if named else _CONSTANT.match(text)
344 if constant and _secret_constant(constant):
345 collector.add(
346 "secrets.constant",
347 FindingSeverity.MEDIUM if in_tests else FindingSeverity.CRITICAL,
348 f"constant {constant.group('name')!r} holds a literal {_mask(constant.group('value'))}",
349 path,
350 number,
351 _TEST_RECOMMENDATION if in_tests else _MOVE_TO_SECRET_STORE,
352 )
355def _declared_value(
356 collector: _Collector, declared: set[str], text: str, path: str, number: int
357) -> None:
358 """Record the first value the task discloses as a secret that the line contains."""
359 for value in sorted(declared):
360 if value in text: 360 ↛ 359line 360 didn't jump to line 359 because the condition on line 360 was always true
361 collector.add(
362 "secrets.declared-value",
363 FindingSeverity.CRITICAL,
364 f"A value the task discloses as a secret is written into the code {_mask(value)}",
365 path,
366 number,
367 "Never copy a secret from the task; read it from the environment.",
368 )
369 return
372def _named_credentials(
373 collector: _Collector, text: str, path: str, number: int, in_tests: bool
374) -> bool:
375 """Grade the credentials the line assigns to a name; whether any name was found."""
376 named = False
377 for name, prefix, value in _credential_literals(text):
378 named = True
379 _graded(
380 collector,
381 _CREDENTIAL_RULE,
382 f"{name!r} is assigned",
383 value,
384 prefix,
385 path,
386 number,
387 in_tests=in_tests,
388 )
389 for function, prefix, value in _auth_call_literals(text):
390 named = True
391 _graded(
392 collector,
393 _CREDENTIAL_RULE,
394 f"{function}() receives a positional string",
395 value,
396 prefix,
397 path,
398 number,
399 in_tests=in_tests,
400 )
401 for key, prefix, value in _environment_literals(text):
402 if _ENVIRONMENT_KEY.search(key.upper()):
403 named = True
404 _graded(
405 collector,
406 "secrets.environment-assignment",
407 f"environment variable {key!r} is set",
408 value,
409 prefix,
410 path,
411 number,
412 in_tests=in_tests,
413 )
414 return named
417def _secret_constant(constant: re.Match[str]) -> bool:
418 """Whether a constant assignment holds a literal that looks like a real secret."""
419 return bool(
420 _constant_name(constant.group("name"))
421 and constant.group("value").strip()
422 and not is_dummy_value(constant.group("value"))
423 and not _is_template(constant.group("value"), constant.group("prefix"))
424 and not _not_a_value(constant.group("value"))
425 )