Coverage for src/governed_harness/evidence/redaction.py: 89%

83 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1from __future__ import annotations 

2 

3import re 

4from collections.abc import Iterable 

5from dataclasses import dataclass 

6from typing import Any 

7 

8PRIVATE_KEY = re.compile( 

9 rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----.*?" 

10 rb"-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----", 

11 re.DOTALL, 

12) 

13PRIVATE_KEY_MARK = b"<REDACTED_PRIVATE_KEY>" 

14 

15 

16@dataclass(frozen=True) 

17class RedactionResult: 

18 data: bytes 

19 redacted: bool 

20 rule_ids: tuple[str, ...] 

21 

22 

23class SecretRedactor: 

24 """Best-effort pre-persistence redaction; it is not a substitute for secret isolation.""" 

25 

26 _RULES: tuple[tuple[str, re.Pattern[bytes], bytes], ...] = ( 

27 ( 

28 "authorization_header", 

29 # The value starts after the blanks; when no such value ends its line, it is the 

30 # last blank before a line end. 

31 re.compile( 

32 rb"(?im)^(authorization\s*:(?:\s*(?=\S[^\r\n]*$)|\s*(?=[^\S\r\n]$)))" 

33 rb"(\S[^\r\n]*|[^\S\r\n])$" 

34 ), 

35 rb"\1<REDACTED>", 

36 ), 

37 ( 

38 "generic_secret_assignment", 

39 re.compile( 

40 rb"(?i)\b(api[_-]?key|token|secret|password|passwd|access[_-]?token)\b" 

41 rb"(\s*[:=]\s*)" 

42 rb"([^\s,;\}\]\"']{4,}|\"[^\"]{4,}\"|'[^']{4,}')" 

43 ), 

44 rb"\1\2<REDACTED>", 

45 ), 

46 ( 

47 "github_token", 

48 re.compile(rb"\b(?:gh[pousr]_\w{20,}|github_pat_\w{20,})\b"), 

49 b"<REDACTED_GITHUB_TOKEN>", 

50 ), 

51 ( 

52 "aws_access_key", 

53 re.compile(rb"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"), 

54 b"<REDACTED_AWS_KEY>", 

55 ), 

56 ("private_key", PRIVATE_KEY, PRIVATE_KEY_MARK), 

57 ) 

58 

59 def __init__(self, *, literals: Iterable[bytes] = (), extended: bool = False) -> None: 

60 # Values configured for the whole store (the environment a provider receives through 

61 # passEnv or fromEnv); short values are left alone so that "1" or "true" does not 

62 # erase every occurrence in an output. 

63 self.literals = tuple( 

64 sorted( 

65 {item for item in literals if len(item) >= MIN_LITERAL_BYTES}, 

66 key=lambda item: (-len(item), item), 

67 ) 

68 ) 

69 self.extended = extended 

70 

71 @property 

72 def configured(self) -> bool: 

73 """Whether redaction beyond the 1.0.0 rules is configured.""" 

74 return bool(self.literals) or self.extended 

75 

76 def redact( 

77 self, data: bytes, extra_literals: Iterable[bytes] = (), *, code: bool = False 

78 ) -> RedactionResult: 

79 """``code``: ``data`` is source code (a diff), where an unquoted value after ``token``, 

80 ``password`` or ``secret`` is an annotation or an expression (``password: str``, 

81 ``token = self._token``), not a secret; only a quoted literal is redacted there (F4). 

82 Everywhere else (tool output, logs, ``.env`` text) unquoted values still are.""" 

83 result = data 

84 applied: list[str] = [] 

85 for rule_id, pattern, replacement in self._RULES: 

86 if code and rule_id == "generic_secret_assignment": 

87 pattern = QUOTED_SECRET_ASSIGNMENT 

88 result, count = pattern.subn(replacement, result) 

89 if count: 

90 applied.append(rule_id) 

91 configured, configured_rules = self._configured(result) 

92 result = configured 

93 applied.extend(configured_rules) 

94 for index, literal in enumerate(extra_literals): 94 ↛ 95line 94 didn't jump to line 95 because the loop on line 94 never started

95 if literal and literal in result: 

96 result = result.replace(literal, b"<REDACTED_LITERAL>") 

97 applied.append(f"literal_{index}") 

98 return RedactionResult(result, bool(applied), tuple(applied)) 

99 

100 def redact_json(self, value: Any) -> tuple[Any, tuple[str, ...]]: 

101 """``value`` with every string in it (object keys included) redacted on its own, and 

102 the rules applied. The serialized form of a JSON document escapes quotes and 

103 backslashes, so a rule run over it can cut an escape or span two strings and leave a 

104 document that no longer parses; redacting the decoded strings cannot. A value of a type 

105 JSON does not know is redacted in its ``str`` form, the form ``default=str`` stores.""" 

106 applied: list[str] = [] 

107 return self._redact_json_value(value, applied), tuple(applied) 

108 

109 def _redact_json_value(self, item: Any, applied: list[str]) -> Any: 

110 if isinstance(item, str): 

111 return self._redact_json_text(item, applied) 

112 if isinstance(item, dict): 

113 return { 

114 (self._redact_json_text(key, applied) if isinstance(key, str) else key): ( 

115 self._redact_json_value(entry, applied) 

116 ) 

117 for key, entry in item.items() 

118 } 

119 if isinstance(item, list | tuple): 

120 return [self._redact_json_value(entry, applied) for entry in item] 

121 if item is None or isinstance(item, bool | int | float): 121 ↛ 123line 121 didn't jump to line 123 because the condition on line 121 was always true

122 return item 

123 return self._redact_json_text(str(item), applied) 

124 

125 def _redact_json_text(self, item: str, applied: list[str]) -> str: 

126 result = self.redact(item.encode("utf-8", "surrogatepass")) 

127 if not result.redacted: 

128 return item 

129 applied.extend(rule for rule in result.rule_ids if rule not in applied) 

130 try: 

131 return result.data.decode("utf-8", "surrogatepass") 

132 except UnicodeDecodeError: 

133 return result.data.decode("utf-8", "replace") 

134 

135 def redact_configured_text(self, text: str) -> str: 

136 """Apply only the redaction configured for the store (provider environment values 

137 and, with ``runtime.extendedRedaction``, the extended rules) to a text that is not an 

138 artifact, such as an agent's summary recorded in an event.""" 

139 if not self.configured: 

140 return text 

141 data, _ = self._configured(text.encode("utf-8")) 

142 return data.decode("utf-8", "replace") 

143 

144 def _configured(self, data: bytes) -> tuple[bytes, list[str]]: 

145 applied: list[str] = [] 

146 # The configured values first: they are exact, the extended rules are patterns. 

147 for literal in self.literals: 

148 if literal in data: 

149 data = data.replace(literal, b"<REDACTED_ENV>") 

150 if "provider_environment" not in applied: 150 ↛ 147line 150 didn't jump to line 147 because the condition on line 150 was always true

151 applied.append("provider_environment") 

152 if self.extended: 

153 for rule_id, pattern, replacement in EXTENDED_RULES: 

154 data, count = pattern.subn(replacement, data) 

155 if count: 155 ↛ 156line 155 didn't jump to line 156 because the condition on line 155 was never true

156 applied.append(rule_id) 

157 return data, applied 

158 

159 

160QUOTED_SECRET_ASSIGNMENT = re.compile( 

161 rb"(?i)\b(api[_-]?key|token|secret|password|passwd|access[_-]?token)\b" 

162 rb"(\s*[:=]\s*)" 

163 rb"(\"[^\"]{4,}\"|'[^']{4,}')" 

164) 

165"""``generic_secret_assignment`` in code: the quoted values only.""" 

166 

167MIN_LITERAL_BYTES = 8 

168"""Environment values shorter than this are not redacted as literals.""" 

169 

170_API_KEY_MARK = b"<REDACTED_API_KEY>" 

171 

172EXTENDED_RULES: tuple[tuple[str, re.Pattern[bytes], bytes], ...] = ( 

173 # runtime.extendedRedaction (since 2.0) 

174 ( 

175 "anthropic_api_key", 

176 re.compile(rb"\bsk-ant-[A-Za-z0-9_-]{16,}"), 

177 _API_KEY_MARK, 

178 ), 

179 ( 

180 "openai_api_key", 

181 re.compile(rb"\bsk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}"), 

182 _API_KEY_MARK, 

183 ), 

184 ("google_api_key", re.compile(rb"\bAIza[0-9A-Za-z_-]{35}\b"), _API_KEY_MARK), 

185 ("slack_token", re.compile(rb"\bxox[abposr]-[A-Za-z0-9-]{10,}"), b"<REDACTED_SLACK_TOKEN>"), 

186 ( 

187 "json_web_token", 

188 re.compile(rb"\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"), 

189 b"<REDACTED_JWT>", 

190 ), 

191 ( 

192 "url_credentials", 

193 re.compile(rb"(\b[a-zA-Z][a-zA-Z0-9+.-]{1,20}://[^:/\s@]{1,256}:)[^@\s/]{1,256}@"), 

194 rb"\1<REDACTED>@", 

195 ), 

196)