Coverage for src/governed_harness/checks/security_patterns.py: 95%
212 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""Security patterns an agent introduces without being asked: unsafe deserialization, stored card
2data, card numbers in source and weak password hashing.
4Each one is a defect a test suite rarely exercises (the code works, it is just unsafe), so it has
5to be recognised in the change itself. Diff-based rules read only the added lines, so a change is
6not blamed for code it did not write; AST-based rules read the current text of the changed Python
7files. Test paths are skipped: fixtures legitimately contain test card numbers and pickles."""
9from __future__ import annotations
11import ast
12import re
13from collections.abc import Iterator, Mapping, Sequence
15from governed_harness.checks.model import DiffFile, Issue, is_test_path
16from governed_harness.domain.enums import FindingSeverity
18_CATEGORY = "security"
20_PICKLE_PATTERNS = (
21 re.compile(r"\bpickle\.(?:loads?|Unpickler)\s*\("),
22 re.compile(r"\bcPickle\b"),
23)
24_OTHER_DESERIALIZATION = (
25 re.compile(r"\bmarshal\.loads?\s*\("),
26 re.compile(r"\bshelve\.open\s*\("),
27 re.compile(r"\bdill\.loads?\s*\("),
28 re.compile(r"\bjsonpickle\.decode\s*\("),
29 re.compile(r"\byaml\.unsafe_load\s*\("),
30)
31_YAML_LOAD = re.compile(r"\byaml\.load\s*\(")
32_SAFE_LOADER = re.compile(r"\bC?SafeLoader\b")
34# A run of 13-19 digits, optionally grouped by single spaces or dashes. A digit or a decimal
35# point right before it means it is part of a longer number or a float, not a card number.
36_CARD_DIGITS = re.compile(r"(?<![\d.])\d(?:[ -]?\d){12,18}(?!\d)")
38_CARD_NAME = re.compile(
39 r"(?:^|_)(?:card_?number|pan|cvc|cvv2?|card_cvc|security_code|card_code)(?:_|$)"
40)
41_CARD_SAFE = re.compile(r"last_?4|last_?four|masked|token|hash|fingerprint")
42_CREATE_TABLE = re.compile(r"create\s+table", re.IGNORECASE)
43_IDENTIFIER = re.compile(r"[A-Za-z_]\w*", re.ASCII)
45_HASH_FUNCTIONS = frozenset({"md5", "sha1", "sha224", "sha256", "sha384", "sha512", "new"})
46_PASSWORD_NAME = re.compile(r"password|passwd|passphrase|pwd|(?:^|_)pass(?:_|$|\d)")
47_MIN_PBKDF2_ITERATIONS = 100_000
50def normalize_name(name: str) -> str:
51 """A name in snake case, lowercase: ``cardNumber`` and ``card-number`` -> ``card_number``,
52 so one pattern covers the spellings of Python, JSON and SQL."""
53 name = re.sub(r"([A-Z])(?=[A-Z][a-z])", r"\1_", name)
54 name = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name)
55 return name.replace("-", "_").lower()
58def is_card_data_name(name: str) -> bool:
59 """Whether a name denotes a full card number or a card security code. Names that say the
60 value is reduced (last four digits, masked, a token, a hash, a fingerprint) are safe to
61 store."""
62 normalized = normalize_name(name)
63 return bool(_CARD_NAME.search(normalized)) and not _CARD_SAFE.search(normalized)
66def luhn_valid(digits: str) -> bool:
67 """The Luhn checksum every payment card number satisfies."""
68 total = 0
69 for index, char in enumerate(reversed(digits)):
70 value = int(char)
71 if index % 2 == 1:
72 value *= 2
73 if value > 9: 73 ↛ 74line 73 didn't jump to line 74 because the condition on line 73 was never true
74 value -= 9
75 total += value
76 return total % 10 == 0
79def _issue(rule_id: str, message: str, path: str, line: int | None, recommendation: str) -> Issue:
80 return Issue(
81 rule_id=rule_id,
82 severity=FindingSeverity.HIGH,
83 message=message,
84 path=path,
85 line=line,
86 category=_CATEGORY,
87 recommendation=recommendation,
88 )
91def _has_restricted_unpickler(tree: ast.Module | None) -> bool:
92 """Whether the file defines a ``pickle.Unpickler`` subclass that overrides ``find_class``,
93 the documented way to restrict what a pickle may construct."""
94 if tree is None:
95 return False
96 for node in ast.walk(tree):
97 if not isinstance(node, ast.ClassDef):
98 continue
99 subclasses_unpickler = any(
100 (isinstance(base, ast.Attribute) and base.attr == "Unpickler")
101 or (isinstance(base, ast.Name) and base.id == "Unpickler")
102 for base in node.bases
103 )
104 overrides = any(
105 isinstance(member, ast.FunctionDef) and member.name == "find_class"
106 for member in node.body
107 )
108 if subclasses_unpickler and overrides:
109 return True
110 return False
113def _deserialization_call(text: str, *, restricted: bool) -> str | None:
114 stripped = text.strip()
115 if stripped.startswith("#"):
116 return None
117 if not restricted:
118 for pattern in _PICKLE_PATTERNS:
119 match = pattern.search(text)
120 if match:
121 return match.group(0).rstrip("( ")
122 for pattern in _OTHER_DESERIALIZATION:
123 match = pattern.search(text)
124 if match:
125 return match.group(0).rstrip("( ")
126 if _YAML_LOAD.search(text) and not _SAFE_LOADER.search(text):
127 return "yaml.load"
128 return None
131def _card_numbers(text: str) -> Iterator[str]:
132 for match in _CARD_DIGITS.finditer(text):
133 digits = re.sub(r"[ -]", "", match.group(0))
134 if 13 <= len(digits) <= 19 and len(set(digits)) > 1 and luhn_valid(digits):
135 yield digits
138def _diff_issues(diff: Sequence[DiffFile], trees: Mapping[str, ast.Module | None]) -> list[Issue]:
139 issues: list[Issue] = []
140 for entry in diff:
141 if entry.is_deleted or is_test_path(entry.path):
142 continue
143 restricted = _has_restricted_unpickler(trees.get(entry.path))
144 for added in entry.added:
145 call = _deserialization_call(added.text, restricted=restricted)
146 if call is not None:
147 issues.append(
148 _issue(
149 "security.unsafe-deserialization",
150 f"{call} deserializes untrusted data in a way that can execute code.",
151 entry.path,
152 added.number,
153 "Use JSON (or yaml.safe_load), or a restricted pickle.Unpickler that "
154 "overrides find_class.",
155 )
156 )
157 for digits in _card_numbers(added.text):
158 issues.append(
159 _issue(
160 "security.card-number-literal",
161 f"The added line contains what looks like a card number "
162 f"(a {len(digits)}-digit Luhn-valid sequence ending in {digits[-4:]}).",
163 entry.path,
164 added.number,
165 "Never put card numbers in source; read test cards from fixtures under "
166 "tests/ and real ones only through the payment provider.",
167 )
168 )
169 break
170 return issues
173def _stored_names(tree: ast.Module) -> Iterator[tuple[str, int]]:
174 """Names under which a value is stored: attribute and subscript targets, class fields,
175 module-level names, dict literal keys and keyword arguments. Local variables of a function
176 are not storage and are left out."""
177 scopes: list[tuple[ast.AST, bool]] = [(tree, True)]
178 while scopes:
179 node, stores_names = scopes.pop()
180 for child in ast.iter_child_nodes(node):
181 if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda):
182 scopes.append((child, False))
183 continue
184 if isinstance(child, ast.ClassDef):
185 scopes.append((child, True))
186 continue
187 yield from _stores_of(child, stores_names)
188 scopes.append((child, stores_names))
191def _target_names(target: ast.expr, stores_names: bool) -> Iterator[tuple[str, int]]:
192 if isinstance(target, ast.Attribute): 192 ↛ 193line 192 didn't jump to line 193 because the condition on line 192 was never true
193 yield target.attr, target.lineno
194 elif isinstance(target, ast.Subscript):
195 key = target.slice
196 if isinstance(key, ast.Constant) and isinstance(key.value, str): 196 ↛ exitline 196 didn't return from function '_target_names' because the condition on line 196 was always true
197 yield key.value, target.lineno
198 elif isinstance(target, ast.Name) and stores_names:
199 yield target.id, target.lineno
200 elif isinstance(target, ast.Tuple | ast.List): 200 ↛ 201line 200 didn't jump to line 201 because the condition on line 200 was never true
201 for element in target.elts:
202 yield from _target_names(element, stores_names)
205def _stores_of(node: ast.AST, stores_names: bool) -> Iterator[tuple[str, int]]:
206 if isinstance(node, ast.Assign):
207 for target in node.targets:
208 yield from _target_names(target, stores_names)
209 elif isinstance(node, ast.AnnAssign | ast.AugAssign):
210 yield from _target_names(node.target, stores_names)
211 elif isinstance(node, ast.Dict):
212 yield from _dict_key_names(node)
213 elif isinstance(node, ast.Call):
214 yield from _keyword_names(node)
215 else:
216 yield from _create_table_names(node)
219def _keyword_names(node: ast.Call) -> Iterator[tuple[str, int]]:
220 for keyword in node.keywords:
221 if keyword.arg is not None: 221 ↛ 220line 221 didn't jump to line 220 because the condition on line 221 was always true
222 yield keyword.arg, keyword.value.lineno
225def _dict_key_names(node: ast.Dict) -> Iterator[tuple[str, int]]:
226 for key in node.keys:
227 if isinstance(key, ast.Constant) and isinstance(key.value, str): 227 ↛ 226line 227 didn't jump to line 226 because the condition on line 227 was always true
228 yield key.value, key.lineno
231def _create_table_names(node: ast.AST) -> Iterator[tuple[str, int]]:
232 """The identifiers of a string constant holding a ``CREATE TABLE`` statement."""
233 if (
234 isinstance(node, ast.Constant)
235 and isinstance(node.value, str)
236 and _CREATE_TABLE.search(node.value)
237 ):
238 for identifier in _IDENTIFIER.findall(node.value):
239 yield identifier, node.lineno
242def _card_data_issues(path: str, tree: ast.Module) -> list[Issue]:
243 first: dict[str, tuple[str, int]] = {}
244 for name, line in _stored_names(tree):
245 if not is_card_data_name(name):
246 continue
247 key = normalize_name(name)
248 if key not in first or line < first[key][1]: 248 ↛ 244line 248 didn't jump to line 244 because the condition on line 248 was always true
249 first[key] = (name, line)
250 return [
251 _issue(
252 "security.card-data-stored",
253 f"{name} stores a full card number or security code.",
254 path,
255 line,
256 "Store the provider's token and the last four digits; never store the CVC.",
257 )
258 for name, line in sorted(first.values(), key=lambda item: (item[1], item[0]))
259 ]
262def _hashlib_aliases(tree: ast.Module) -> tuple[set[str], dict[str, str]]:
263 """The names bound to the ``hashlib`` module and to functions imported from it."""
264 modules: set[str] = set()
265 functions: dict[str, str] = {}
266 for node in ast.walk(tree):
267 if isinstance(node, ast.Import):
268 modules.update(_hashlib_module_names(node))
269 elif isinstance(node, ast.ImportFrom) and node.module == "hashlib" and not node.level:
270 for alias in node.names:
271 functions[alias.asname or alias.name] = alias.name
272 return modules, functions
275def _hashlib_module_names(node: ast.Import) -> Iterator[str]:
276 """The names an ``import`` statement binds to the ``hashlib`` module."""
277 for alias in node.names:
278 if alias.name == "hashlib":
279 yield alias.asname or "hashlib"
282def _hash_function(call: ast.Call, modules: set[str], functions: dict[str, str]) -> str | None:
283 func = call.func
284 if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name):
285 if func.value.id in modules or func.value.id == "hashlib":
286 return func.attr
287 elif isinstance(func, ast.Name) and func.id in functions:
288 return functions[func.id]
289 return None
292def _mentions_password(call: ast.Call) -> bool:
293 for argument in [*call.args, *(keyword.value for keyword in call.keywords)]:
294 for node in ast.walk(argument):
295 name = None
296 if isinstance(node, ast.Name):
297 name = node.id
298 elif isinstance(node, ast.Attribute):
299 name = node.attr
300 if name is not None and _PASSWORD_NAME.search(normalize_name(name)):
301 return True
302 return False
305def _iterations(call: ast.Call) -> int | None:
306 value: ast.expr | None = call.args[3] if len(call.args) > 3 else None
307 for keyword in call.keywords:
308 if keyword.arg == "iterations": 308 ↛ 307line 308 didn't jump to line 307 because the condition on line 308 was always true
309 value = keyword.value
310 if ( 310 ↛ 316line 310 didn't jump to line 316 because the condition on line 310 was always true
311 isinstance(value, ast.Constant)
312 and isinstance(value.value, int)
313 and not isinstance(value.value, bool)
314 ):
315 return value.value
316 return None
319def _password_hash_issues(path: str, tree: ast.Module) -> list[Issue]:
320 modules, functions = _hashlib_aliases(tree)
321 issues: list[Issue] = []
322 calls = sorted(
323 (node for node in ast.walk(tree) if isinstance(node, ast.Call)),
324 key=lambda call: (call.lineno, call.col_offset),
325 )
326 for node in calls:
327 name = _hash_function(node, modules, functions)
328 if name in _HASH_FUNCTIONS and _mentions_password(node):
329 issues.append(
330 _issue(
331 "security.weak-password-hash",
332 f"A password is hashed with hashlib.{name}, a fast unsalted digest that is "
333 "cheap to brute-force.",
334 path,
335 node.lineno,
336 "Use hashlib.scrypt, or hashlib.pbkdf2_hmac with a per-user salt and at "
337 "least 100000 iterations.",
338 )
339 )
340 elif name == "pbkdf2_hmac":
341 iterations = _iterations(node)
342 if iterations is not None and iterations < _MIN_PBKDF2_ITERATIONS:
343 issues.append(
344 _issue(
345 "security.weak-password-hash",
346 f"hashlib.pbkdf2_hmac runs {iterations} iterations; at least "
347 f"{_MIN_PBKDF2_ITERATIONS} are needed.",
348 path,
349 node.lineno,
350 "Raise the iteration count to at least 100000 (or use hashlib.scrypt).",
351 )
352 )
353 return issues
356def check_security_patterns(diff: Sequence[DiffFile], files: Mapping[str, str]) -> list[Issue]:
357 """The security issues of a change: diff rules over the added lines, AST rules over the
358 current text of the changed, non-test Python files. Every issue is HIGH."""
359 trees: dict[str, ast.Module | None] = {}
360 issues: list[Issue] = []
361 for path in sorted(files):
362 if not path.endswith(".py") or is_test_path(path):
363 continue
364 try:
365 trees[path] = ast.parse(files[path], filename=path)
366 except SyntaxError as error:
367 trees[path] = None
368 issues.append(
369 Issue(
370 rule_id="security.unparsable",
371 severity=FindingSeverity.INFO,
372 message=f"{path} could not be parsed ({error.msg}); "
373 "its AST-based security rules were not applied.",
374 path=path,
375 line=error.lineno,
376 category=_CATEGORY,
377 )
378 )
379 issues.extend(_diff_issues(diff, trees))
380 for path, tree in trees.items():
381 if tree is not None:
382 issues.extend(_card_data_issues(path, tree))
383 issues.extend(_password_hash_issues(path, tree))
384 return issues