Coverage for src/governed_harness/evidence/redaction.py: 89%
83 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1from __future__ import annotations
3import re
4from collections.abc import Iterable
5from dataclasses import dataclass
6from typing import Any
8PRIVATE_KEY = re.compile(
9 rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----.*?"
10 rb"-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----",
11 re.DOTALL,
12)
13PRIVATE_KEY_MARK = b"<REDACTED_PRIVATE_KEY>"
16@dataclass(frozen=True)
17class RedactionResult:
18 data: bytes
19 redacted: bool
20 rule_ids: tuple[str, ...]
23class SecretRedactor:
24 """Best-effort pre-persistence redaction; it is not a substitute for secret isolation."""
26 _RULES: tuple[tuple[str, re.Pattern[bytes], bytes], ...] = (
27 (
28 "authorization_header",
29 # The value starts after the blanks; when no such value ends its line, it is the
30 # last blank before a line end.
31 re.compile(
32 rb"(?im)^(authorization\s*:(?:\s*(?=\S[^\r\n]*$)|\s*(?=[^\S\r\n]$)))"
33 rb"(\S[^\r\n]*|[^\S\r\n])$"
34 ),
35 rb"\1<REDACTED>",
36 ),
37 (
38 "generic_secret_assignment",
39 re.compile(
40 rb"(?i)\b(api[_-]?key|token|secret|password|passwd|access[_-]?token)\b"
41 rb"(\s*[:=]\s*)"
42 rb"([^\s,;\}\]\"']{4,}|\"[^\"]{4,}\"|'[^']{4,}')"
43 ),
44 rb"\1\2<REDACTED>",
45 ),
46 (
47 "github_token",
48 re.compile(rb"\b(?:gh[pousr]_\w{20,}|github_pat_\w{20,})\b"),
49 b"<REDACTED_GITHUB_TOKEN>",
50 ),
51 (
52 "aws_access_key",
53 re.compile(rb"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"),
54 b"<REDACTED_AWS_KEY>",
55 ),
56 ("private_key", PRIVATE_KEY, PRIVATE_KEY_MARK),
57 )
59 def __init__(self, *, literals: Iterable[bytes] = (), extended: bool = False) -> None:
60 # Values configured for the whole store (the environment a provider receives through
61 # passEnv or fromEnv); short values are left alone so that "1" or "true" does not
62 # erase every occurrence in an output.
63 self.literals = tuple(
64 sorted(
65 {item for item in literals if len(item) >= MIN_LITERAL_BYTES},
66 key=lambda item: (-len(item), item),
67 )
68 )
69 self.extended = extended
71 @property
72 def configured(self) -> bool:
73 """Whether redaction beyond the 1.0.0 rules is configured."""
74 return bool(self.literals) or self.extended
76 def redact(
77 self, data: bytes, extra_literals: Iterable[bytes] = (), *, code: bool = False
78 ) -> RedactionResult:
79 """``code``: ``data`` is source code (a diff), where an unquoted value after ``token``,
80 ``password`` or ``secret`` is an annotation or an expression (``password: str``,
81 ``token = self._token``), not a secret; only a quoted literal is redacted there (F4).
82 Everywhere else (tool output, logs, ``.env`` text) unquoted values still are."""
83 result = data
84 applied: list[str] = []
85 for rule_id, pattern, replacement in self._RULES:
86 if code and rule_id == "generic_secret_assignment":
87 pattern = QUOTED_SECRET_ASSIGNMENT
88 result, count = pattern.subn(replacement, result)
89 if count:
90 applied.append(rule_id)
91 configured, configured_rules = self._configured(result)
92 result = configured
93 applied.extend(configured_rules)
94 for index, literal in enumerate(extra_literals): 94 ↛ 95line 94 didn't jump to line 95 because the loop on line 94 never started
95 if literal and literal in result:
96 result = result.replace(literal, b"<REDACTED_LITERAL>")
97 applied.append(f"literal_{index}")
98 return RedactionResult(result, bool(applied), tuple(applied))
100 def redact_json(self, value: Any) -> tuple[Any, tuple[str, ...]]:
101 """``value`` with every string in it (object keys included) redacted on its own, and
102 the rules applied. The serialized form of a JSON document escapes quotes and
103 backslashes, so a rule run over it can cut an escape or span two strings and leave a
104 document that no longer parses; redacting the decoded strings cannot. A value of a type
105 JSON does not know is redacted in its ``str`` form, the form ``default=str`` stores."""
106 applied: list[str] = []
107 return self._redact_json_value(value, applied), tuple(applied)
109 def _redact_json_value(self, item: Any, applied: list[str]) -> Any:
110 if isinstance(item, str):
111 return self._redact_json_text(item, applied)
112 if isinstance(item, dict):
113 return {
114 (self._redact_json_text(key, applied) if isinstance(key, str) else key): (
115 self._redact_json_value(entry, applied)
116 )
117 for key, entry in item.items()
118 }
119 if isinstance(item, list | tuple):
120 return [self._redact_json_value(entry, applied) for entry in item]
121 if item is None or isinstance(item, bool | int | float): 121 ↛ 123line 121 didn't jump to line 123 because the condition on line 121 was always true
122 return item
123 return self._redact_json_text(str(item), applied)
125 def _redact_json_text(self, item: str, applied: list[str]) -> str:
126 result = self.redact(item.encode("utf-8", "surrogatepass"))
127 if not result.redacted:
128 return item
129 applied.extend(rule for rule in result.rule_ids if rule not in applied)
130 try:
131 return result.data.decode("utf-8", "surrogatepass")
132 except UnicodeDecodeError:
133 return result.data.decode("utf-8", "replace")
135 def redact_configured_text(self, text: str) -> str:
136 """Apply only the redaction configured for the store (provider environment values
137 and, with ``runtime.extendedRedaction``, the extended rules) to a text that is not an
138 artifact, such as an agent's summary recorded in an event."""
139 if not self.configured:
140 return text
141 data, _ = self._configured(text.encode("utf-8"))
142 return data.decode("utf-8", "replace")
144 def _configured(self, data: bytes) -> tuple[bytes, list[str]]:
145 applied: list[str] = []
146 # The configured values first: they are exact, the extended rules are patterns.
147 for literal in self.literals:
148 if literal in data:
149 data = data.replace(literal, b"<REDACTED_ENV>")
150 if "provider_environment" not in applied: 150 ↛ 147line 150 didn't jump to line 147 because the condition on line 150 was always true
151 applied.append("provider_environment")
152 if self.extended:
153 for rule_id, pattern, replacement in EXTENDED_RULES:
154 data, count = pattern.subn(replacement, data)
155 if count: 155 ↛ 156line 155 didn't jump to line 156 because the condition on line 155 was never true
156 applied.append(rule_id)
157 return data, applied
160QUOTED_SECRET_ASSIGNMENT = re.compile(
161 rb"(?i)\b(api[_-]?key|token|secret|password|passwd|access[_-]?token)\b"
162 rb"(\s*[:=]\s*)"
163 rb"(\"[^\"]{4,}\"|'[^']{4,}')"
164)
165"""``generic_secret_assignment`` in code: the quoted values only."""
167MIN_LITERAL_BYTES = 8
168"""Environment values shorter than this are not redacted as literals."""
170_API_KEY_MARK = b"<REDACTED_API_KEY>"
172EXTENDED_RULES: tuple[tuple[str, re.Pattern[bytes], bytes], ...] = (
173 # runtime.extendedRedaction (since 2.0)
174 (
175 "anthropic_api_key",
176 re.compile(rb"\bsk-ant-[A-Za-z0-9_-]{16,}"),
177 _API_KEY_MARK,
178 ),
179 (
180 "openai_api_key",
181 re.compile(rb"\bsk-(?:proj-|svcacct-|admin-)?[A-Za-z0-9_-]{20,}"),
182 _API_KEY_MARK,
183 ),
184 ("google_api_key", re.compile(rb"\bAIza[0-9A-Za-z_-]{35}\b"), _API_KEY_MARK),
185 ("slack_token", re.compile(rb"\bxox[abposr]-[A-Za-z0-9-]{10,}"), b"<REDACTED_SLACK_TOKEN>"),
186 (
187 "json_web_token",
188 re.compile(rb"\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}"),
189 b"<REDACTED_JWT>",
190 ),
191 (
192 "url_credentials",
193 re.compile(rb"(\b[a-zA-Z][a-zA-Z0-9+.-]{1,20}://[^:/\s@]{1,256}:)[^@\s/]{1,256}@"),
194 rb"\1<REDACTED>@",
195 ),
196)