Coverage for src/governed_harness/checks/security_patterns.py: 95%

212 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Security patterns an agent introduces without being asked: unsafe deserialization, stored card 

2data, card numbers in source and weak password hashing. 

3 

4Each one is a defect a test suite rarely exercises (the code works, it is just unsafe), so it has 

5to be recognised in the change itself. Diff-based rules read only the added lines, so a change is 

6not blamed for code it did not write; AST-based rules read the current text of the changed Python 

7files. Test paths are skipped: fixtures legitimately contain test card numbers and pickles.""" 

8 

9from __future__ import annotations 

10 

11import ast 

12import re 

13from collections.abc import Iterator, Mapping, Sequence 

14 

15from governed_harness.checks.model import DiffFile, Issue, is_test_path 

16from governed_harness.domain.enums import FindingSeverity 

17 

18_CATEGORY = "security" 

19 

20_PICKLE_PATTERNS = ( 

21 re.compile(r"\bpickle\.(?:loads?|Unpickler)\s*\("), 

22 re.compile(r"\bcPickle\b"), 

23) 

24_OTHER_DESERIALIZATION = ( 

25 re.compile(r"\bmarshal\.loads?\s*\("), 

26 re.compile(r"\bshelve\.open\s*\("), 

27 re.compile(r"\bdill\.loads?\s*\("), 

28 re.compile(r"\bjsonpickle\.decode\s*\("), 

29 re.compile(r"\byaml\.unsafe_load\s*\("), 

30) 

31_YAML_LOAD = re.compile(r"\byaml\.load\s*\(") 

32_SAFE_LOADER = re.compile(r"\bC?SafeLoader\b") 

33 

34# A run of 13-19 digits, optionally grouped by single spaces or dashes. A digit or a decimal 

35# point right before it means it is part of a longer number or a float, not a card number. 

36_CARD_DIGITS = re.compile(r"(?<![\d.])\d(?:[ -]?\d){12,18}(?!\d)") 

37 

38_CARD_NAME = re.compile( 

39 r"(?:^|_)(?:card_?number|pan|cvc|cvv2?|card_cvc|security_code|card_code)(?:_|$)" 

40) 

41_CARD_SAFE = re.compile(r"last_?4|last_?four|masked|token|hash|fingerprint") 

42_CREATE_TABLE = re.compile(r"create\s+table", re.IGNORECASE) 

43_IDENTIFIER = re.compile(r"[A-Za-z_]\w*", re.ASCII) 

44 

45_HASH_FUNCTIONS = frozenset({"md5", "sha1", "sha224", "sha256", "sha384", "sha512", "new"}) 

46_PASSWORD_NAME = re.compile(r"password|passwd|passphrase|pwd|(?:^|_)pass(?:_|$|\d)") 

47_MIN_PBKDF2_ITERATIONS = 100_000 

48 

49 

50def normalize_name(name: str) -> str: 

51 """A name in snake case, lowercase: ``cardNumber`` and ``card-number`` -> ``card_number``, 

52 so one pattern covers the spellings of Python, JSON and SQL.""" 

53 name = re.sub(r"([A-Z])(?=[A-Z][a-z])", r"\1_", name) 

54 name = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name) 

55 return name.replace("-", "_").lower() 

56 

57 

58def is_card_data_name(name: str) -> bool: 

59 """Whether a name denotes a full card number or a card security code. Names that say the 

60 value is reduced (last four digits, masked, a token, a hash, a fingerprint) are safe to 

61 store.""" 

62 normalized = normalize_name(name) 

63 return bool(_CARD_NAME.search(normalized)) and not _CARD_SAFE.search(normalized) 

64 

65 

66def luhn_valid(digits: str) -> bool: 

67 """The Luhn checksum every payment card number satisfies.""" 

68 total = 0 

69 for index, char in enumerate(reversed(digits)): 

70 value = int(char) 

71 if index % 2 == 1: 

72 value *= 2 

73 if value > 9: 73 ↛ 74line 73 didn't jump to line 74 because the condition on line 73 was never true

74 value -= 9 

75 total += value 

76 return total % 10 == 0 

77 

78 

79def _issue(rule_id: str, message: str, path: str, line: int | None, recommendation: str) -> Issue: 

80 return Issue( 

81 rule_id=rule_id, 

82 severity=FindingSeverity.HIGH, 

83 message=message, 

84 path=path, 

85 line=line, 

86 category=_CATEGORY, 

87 recommendation=recommendation, 

88 ) 

89 

90 

91def _has_restricted_unpickler(tree: ast.Module | None) -> bool: 

92 """Whether the file defines a ``pickle.Unpickler`` subclass that overrides ``find_class``, 

93 the documented way to restrict what a pickle may construct.""" 

94 if tree is None: 

95 return False 

96 for node in ast.walk(tree): 

97 if not isinstance(node, ast.ClassDef): 

98 continue 

99 subclasses_unpickler = any( 

100 (isinstance(base, ast.Attribute) and base.attr == "Unpickler") 

101 or (isinstance(base, ast.Name) and base.id == "Unpickler") 

102 for base in node.bases 

103 ) 

104 overrides = any( 

105 isinstance(member, ast.FunctionDef) and member.name == "find_class" 

106 for member in node.body 

107 ) 

108 if subclasses_unpickler and overrides: 

109 return True 

110 return False 

111 

112 

113def _deserialization_call(text: str, *, restricted: bool) -> str | None: 

114 stripped = text.strip() 

115 if stripped.startswith("#"): 

116 return None 

117 if not restricted: 

118 for pattern in _PICKLE_PATTERNS: 

119 match = pattern.search(text) 

120 if match: 

121 return match.group(0).rstrip("( ") 

122 for pattern in _OTHER_DESERIALIZATION: 

123 match = pattern.search(text) 

124 if match: 

125 return match.group(0).rstrip("( ") 

126 if _YAML_LOAD.search(text) and not _SAFE_LOADER.search(text): 

127 return "yaml.load" 

128 return None 

129 

130 

131def _card_numbers(text: str) -> Iterator[str]: 

132 for match in _CARD_DIGITS.finditer(text): 

133 digits = re.sub(r"[ -]", "", match.group(0)) 

134 if 13 <= len(digits) <= 19 and len(set(digits)) > 1 and luhn_valid(digits): 

135 yield digits 

136 

137 

138def _diff_issues(diff: Sequence[DiffFile], trees: Mapping[str, ast.Module | None]) -> list[Issue]: 

139 issues: list[Issue] = [] 

140 for entry in diff: 

141 if entry.is_deleted or is_test_path(entry.path): 

142 continue 

143 restricted = _has_restricted_unpickler(trees.get(entry.path)) 

144 for added in entry.added: 

145 call = _deserialization_call(added.text, restricted=restricted) 

146 if call is not None: 

147 issues.append( 

148 _issue( 

149 "security.unsafe-deserialization", 

150 f"{call} deserializes untrusted data in a way that can execute code.", 

151 entry.path, 

152 added.number, 

153 "Use JSON (or yaml.safe_load), or a restricted pickle.Unpickler that " 

154 "overrides find_class.", 

155 ) 

156 ) 

157 for digits in _card_numbers(added.text): 

158 issues.append( 

159 _issue( 

160 "security.card-number-literal", 

161 f"The added line contains what looks like a card number " 

162 f"(a {len(digits)}-digit Luhn-valid sequence ending in {digits[-4:]}).", 

163 entry.path, 

164 added.number, 

165 "Never put card numbers in source; read test cards from fixtures under " 

166 "tests/ and real ones only through the payment provider.", 

167 ) 

168 ) 

169 break 

170 return issues 

171 

172 

173def _stored_names(tree: ast.Module) -> Iterator[tuple[str, int]]: 

174 """Names under which a value is stored: attribute and subscript targets, class fields, 

175 module-level names, dict literal keys and keyword arguments. Local variables of a function 

176 are not storage and are left out.""" 

177 scopes: list[tuple[ast.AST, bool]] = [(tree, True)] 

178 while scopes: 

179 node, stores_names = scopes.pop() 

180 for child in ast.iter_child_nodes(node): 

181 if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef | ast.Lambda): 

182 scopes.append((child, False)) 

183 continue 

184 if isinstance(child, ast.ClassDef): 

185 scopes.append((child, True)) 

186 continue 

187 yield from _stores_of(child, stores_names) 

188 scopes.append((child, stores_names)) 

189 

190 

191def _target_names(target: ast.expr, stores_names: bool) -> Iterator[tuple[str, int]]: 

192 if isinstance(target, ast.Attribute): 192 ↛ 193line 192 didn't jump to line 193 because the condition on line 192 was never true

193 yield target.attr, target.lineno 

194 elif isinstance(target, ast.Subscript): 

195 key = target.slice 

196 if isinstance(key, ast.Constant) and isinstance(key.value, str): 196 ↛ exitline 196 didn't return from function '_target_names' because the condition on line 196 was always true

197 yield key.value, target.lineno 

198 elif isinstance(target, ast.Name) and stores_names: 

199 yield target.id, target.lineno 

200 elif isinstance(target, ast.Tuple | ast.List): 200 ↛ 201line 200 didn't jump to line 201 because the condition on line 200 was never true

201 for element in target.elts: 

202 yield from _target_names(element, stores_names) 

203 

204 

205def _stores_of(node: ast.AST, stores_names: bool) -> Iterator[tuple[str, int]]: 

206 if isinstance(node, ast.Assign): 

207 for target in node.targets: 

208 yield from _target_names(target, stores_names) 

209 elif isinstance(node, ast.AnnAssign | ast.AugAssign): 

210 yield from _target_names(node.target, stores_names) 

211 elif isinstance(node, ast.Dict): 

212 yield from _dict_key_names(node) 

213 elif isinstance(node, ast.Call): 

214 yield from _keyword_names(node) 

215 else: 

216 yield from _create_table_names(node) 

217 

218 

219def _keyword_names(node: ast.Call) -> Iterator[tuple[str, int]]: 

220 for keyword in node.keywords: 

221 if keyword.arg is not None: 221 ↛ 220line 221 didn't jump to line 220 because the condition on line 221 was always true

222 yield keyword.arg, keyword.value.lineno 

223 

224 

225def _dict_key_names(node: ast.Dict) -> Iterator[tuple[str, int]]: 

226 for key in node.keys: 

227 if isinstance(key, ast.Constant) and isinstance(key.value, str): 227 ↛ 226line 227 didn't jump to line 226 because the condition on line 227 was always true

228 yield key.value, key.lineno 

229 

230 

231def _create_table_names(node: ast.AST) -> Iterator[tuple[str, int]]: 

232 """The identifiers of a string constant holding a ``CREATE TABLE`` statement.""" 

233 if ( 

234 isinstance(node, ast.Constant) 

235 and isinstance(node.value, str) 

236 and _CREATE_TABLE.search(node.value) 

237 ): 

238 for identifier in _IDENTIFIER.findall(node.value): 

239 yield identifier, node.lineno 

240 

241 

242def _card_data_issues(path: str, tree: ast.Module) -> list[Issue]: 

243 first: dict[str, tuple[str, int]] = {} 

244 for name, line in _stored_names(tree): 

245 if not is_card_data_name(name): 

246 continue 

247 key = normalize_name(name) 

248 if key not in first or line < first[key][1]: 248 ↛ 244line 248 didn't jump to line 244 because the condition on line 248 was always true

249 first[key] = (name, line) 

250 return [ 

251 _issue( 

252 "security.card-data-stored", 

253 f"{name} stores a full card number or security code.", 

254 path, 

255 line, 

256 "Store the provider's token and the last four digits; never store the CVC.", 

257 ) 

258 for name, line in sorted(first.values(), key=lambda item: (item[1], item[0])) 

259 ] 

260 

261 

262def _hashlib_aliases(tree: ast.Module) -> tuple[set[str], dict[str, str]]: 

263 """The names bound to the ``hashlib`` module and to functions imported from it.""" 

264 modules: set[str] = set() 

265 functions: dict[str, str] = {} 

266 for node in ast.walk(tree): 

267 if isinstance(node, ast.Import): 

268 modules.update(_hashlib_module_names(node)) 

269 elif isinstance(node, ast.ImportFrom) and node.module == "hashlib" and not node.level: 

270 for alias in node.names: 

271 functions[alias.asname or alias.name] = alias.name 

272 return modules, functions 

273 

274 

275def _hashlib_module_names(node: ast.Import) -> Iterator[str]: 

276 """The names an ``import`` statement binds to the ``hashlib`` module.""" 

277 for alias in node.names: 

278 if alias.name == "hashlib": 

279 yield alias.asname or "hashlib" 

280 

281 

282def _hash_function(call: ast.Call, modules: set[str], functions: dict[str, str]) -> str | None: 

283 func = call.func 

284 if isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name): 

285 if func.value.id in modules or func.value.id == "hashlib": 

286 return func.attr 

287 elif isinstance(func, ast.Name) and func.id in functions: 

288 return functions[func.id] 

289 return None 

290 

291 

292def _mentions_password(call: ast.Call) -> bool: 

293 for argument in [*call.args, *(keyword.value for keyword in call.keywords)]: 

294 for node in ast.walk(argument): 

295 name = None 

296 if isinstance(node, ast.Name): 

297 name = node.id 

298 elif isinstance(node, ast.Attribute): 

299 name = node.attr 

300 if name is not None and _PASSWORD_NAME.search(normalize_name(name)): 

301 return True 

302 return False 

303 

304 

305def _iterations(call: ast.Call) -> int | None: 

306 value: ast.expr | None = call.args[3] if len(call.args) > 3 else None 

307 for keyword in call.keywords: 

308 if keyword.arg == "iterations": 308 ↛ 307line 308 didn't jump to line 307 because the condition on line 308 was always true

309 value = keyword.value 

310 if ( 310 ↛ 316line 310 didn't jump to line 316 because the condition on line 310 was always true

311 isinstance(value, ast.Constant) 

312 and isinstance(value.value, int) 

313 and not isinstance(value.value, bool) 

314 ): 

315 return value.value 

316 return None 

317 

318 

319def _password_hash_issues(path: str, tree: ast.Module) -> list[Issue]: 

320 modules, functions = _hashlib_aliases(tree) 

321 issues: list[Issue] = [] 

322 calls = sorted( 

323 (node for node in ast.walk(tree) if isinstance(node, ast.Call)), 

324 key=lambda call: (call.lineno, call.col_offset), 

325 ) 

326 for node in calls: 

327 name = _hash_function(node, modules, functions) 

328 if name in _HASH_FUNCTIONS and _mentions_password(node): 

329 issues.append( 

330 _issue( 

331 "security.weak-password-hash", 

332 f"A password is hashed with hashlib.{name}, a fast unsalted digest that is " 

333 "cheap to brute-force.", 

334 path, 

335 node.lineno, 

336 "Use hashlib.scrypt, or hashlib.pbkdf2_hmac with a per-user salt and at " 

337 "least 100000 iterations.", 

338 ) 

339 ) 

340 elif name == "pbkdf2_hmac": 

341 iterations = _iterations(node) 

342 if iterations is not None and iterations < _MIN_PBKDF2_ITERATIONS: 

343 issues.append( 

344 _issue( 

345 "security.weak-password-hash", 

346 f"hashlib.pbkdf2_hmac runs {iterations} iterations; at least " 

347 f"{_MIN_PBKDF2_ITERATIONS} are needed.", 

348 path, 

349 node.lineno, 

350 "Raise the iteration count to at least 100000 (or use hashlib.scrypt).", 

351 ) 

352 ) 

353 return issues 

354 

355 

356def check_security_patterns(diff: Sequence[DiffFile], files: Mapping[str, str]) -> list[Issue]: 

357 """The security issues of a change: diff rules over the added lines, AST rules over the 

358 current text of the changed, non-test Python files. Every issue is HIGH.""" 

359 trees: dict[str, ast.Module | None] = {} 

360 issues: list[Issue] = [] 

361 for path in sorted(files): 

362 if not path.endswith(".py") or is_test_path(path): 

363 continue 

364 try: 

365 trees[path] = ast.parse(files[path], filename=path) 

366 except SyntaxError as error: 

367 trees[path] = None 

368 issues.append( 

369 Issue( 

370 rule_id="security.unparsable", 

371 severity=FindingSeverity.INFO, 

372 message=f"{path} could not be parsed ({error.msg}); " 

373 "its AST-based security rules were not applied.", 

374 path=path, 

375 line=error.lineno, 

376 category=_CATEGORY, 

377 ) 

378 ) 

379 issues.extend(_diff_issues(diff, trees)) 

380 for path, tree in trees.items(): 

381 if tree is not None: 

382 issues.extend(_card_data_issues(path, tree)) 

383 issues.extend(_password_hash_issues(path, tree)) 

384 return issues