Coverage for src/governed_harness/evidence/artifact_store.py: 90%
192 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1from __future__ import annotations
3import json
4import os
5import sys
6import tempfile
7import time
8from dataclasses import dataclass
9from pathlib import Path
10from typing import Any
12from .hashing import sha256_bytes
13from .redaction import PRIVATE_KEY, PRIVATE_KEY_MARK, SecretRedactor
15# Windows only: how many times the move into place is tried, and the first pause (doubled each time).
16_WINDOWS_REPLACE_ATTEMPTS = 8
17_WINDOWS_REPLACE_BACKOFF_SECONDS = 0.01
20CODE_MEDIA_TYPES = frozenset({"text/x-diff"})
21"""Artifacts that hold source code: redacted with the code rules (quoted secrets only)."""
24@dataclass(frozen=True)
25class ArtifactRef:
26 uri: str
27 digest: str
28 size_bytes: int
29 media_type: str = "application/octet-stream"
30 redacted: bool = False
31 metadata: dict[str, Any] | None = None
34class LocalArtifactStore:
35 """Content-addressed, atomic local store with digest verification."""
37 def __init__(self, root: Path, redactor: SecretRedactor | None = None) -> None:
38 self.root = root.resolve(strict=False)
39 self.blob_root = self.root / "blobs" / "sha256"
40 self.meta_root = self.root / "metadata"
41 self.blob_root.mkdir(parents=True, exist_ok=True, mode=0o700)
42 self.meta_root.mkdir(parents=True, exist_ok=True, mode=0o700)
43 self.redactor = redactor or SecretRedactor()
45 def put(
46 self,
47 data: bytes,
48 *,
49 media_type: str = "application/octet-stream",
50 metadata: dict[str, Any] | None = None,
51 redact: bool = True,
52 secret_literals: tuple[bytes, ...] = (),
53 ) -> ArtifactRef:
54 code = media_type in CODE_MEDIA_TYPES
55 redaction = self.redactor.redact(data, secret_literals, code=code) if redact else None
56 stored = redaction.data if redaction else data
57 rule_ids = redaction.rule_ids if redaction and redaction.redacted else ()
58 return self._store(stored, media_type, metadata, rule_ids)
60 def _store(
61 self,
62 stored: bytes,
63 media_type: str,
64 metadata: dict[str, Any] | None,
65 rule_ids: tuple[str, ...],
66 ) -> ArtifactRef:
67 digest = sha256_bytes(stored)
68 hex_digest = digest.split(":", 1)[1]
69 path = self._blob_path(hex_digest)
70 self._atomic_write(path, stored)
71 meta = dict(metadata or {})
72 if rule_ids:
73 meta["redactionRuleIds"] = list(rule_ids)
74 redacted = bool(rule_ids)
75 descriptor = {
76 "digest": digest,
77 "sizeBytes": len(stored),
78 "mediaType": media_type,
79 "redacted": redacted,
80 "metadata": meta,
81 }
82 self._atomic_write(
83 self.meta_root / f"{hex_digest}.json",
84 json.dumps(descriptor, sort_keys=True, indent=2).encode("utf-8"),
85 )
86 return ArtifactRef(
87 uri=f"artifact://sha256/{hex_digest}",
88 digest=digest,
89 size_bytes=len(stored),
90 media_type=media_type,
91 redacted=redacted,
92 metadata=meta,
93 )
95 def put_json(self, value: Any, *, metadata: dict[str, Any] | None = None) -> ArtifactRef:
96 """Store ``value`` as JSON, redacted as text like any artifact and on its decoded
97 strings. The serialized text escapes quotes, so a rule run over it misses a secret whose
98 value is quoted inside a JSON string (``password = \\"...\\"``, #178), and it can cut an
99 escape or span two strings and leave a document that no longer parses (#152). When any
100 decoded string redacts, or the text redaction breaks the document, the strings are
101 redacted one by one instead; a document whose strings have nothing to redact is stored
102 exactly as before, with the same bytes and digest.
104 A redacted JSON artifact is evidence, not data: a caller that reads one back to act on
105 it (a pinned task, a workspace snapshot) checks the ``redacted`` flag of its descriptor
106 and never applies a redacted value."""
107 data = _json_bytes(value)
108 redaction = self.redactor.redact(data)
109 redacted, rule_ids = self.redactor.redact_json(value)
110 if not rule_ids and (not redaction.redacted or _parses(redaction.data)):
111 return self._store(redaction.data, "application/json", metadata, redaction.rule_ids)
112 data = _json_bytes(redacted)
113 # A private key whose lines are separate strings is caught only on the whole text: the
114 # text redaction is kept when it parses, otherwise nothing of the document is kept.
115 spanning, count = PRIVATE_KEY.subn(PRIVATE_KEY_MARK, data)
116 if count:
117 data = (
118 spanning
119 if _parses(spanning)
120 else _json_bytes({"redactedDocument": PRIVATE_KEY_MARK.decode()})
121 )
122 if "private_key" not in rule_ids: 122 ↛ 124line 122 didn't jump to line 124 because the condition on line 122 was always true
123 rule_ids = (*rule_ids, "private_key")
124 return self._store(data, "application/json", metadata, rule_ids)
126 def get(self, reference: ArtifactRef | str) -> bytes:
127 digest = (
128 reference.digest
129 if isinstance(reference, ArtifactRef)
130 else self._digest_from_uri(reference)
131 )
132 hex_digest = digest.split(":", 1)[1]
133 path = self._blob_path(hex_digest)
134 data = path.read_bytes()
135 if sha256_bytes(data) != digest:
136 raise ValueError("artifact digest mismatch")
137 return data
139 def verify(self, reference: ArtifactRef | str) -> bool:
140 try:
141 self.get(reference)
142 return True
143 except (OSError, ValueError):
144 return False
146 def delete(self, reference: ArtifactRef | str) -> bool:
147 """Remove a blob and its descriptor (retention); ``False`` when it was not stored."""
148 digest = (
149 reference.digest
150 if isinstance(reference, ArtifactRef)
151 else self._digest_from_uri(reference)
152 )
153 hex_digest = digest.split(":", 1)[1]
154 path = self._blob_path(hex_digest)
155 existed = path.exists()
156 path.unlink(missing_ok=True)
157 (self.meta_root / f"{hex_digest}.json").unlink(missing_ok=True)
158 return existed
160 def describe(self, uri: str) -> ArtifactRef:
161 digest = self._digest_from_uri(uri)
162 hex_digest = digest.split(":", 1)[1]
163 descriptor = json.loads((self.meta_root / f"{hex_digest}.json").read_text(encoding="utf-8"))
164 return ArtifactRef(
165 uri=uri,
166 digest=digest,
167 size_bytes=descriptor["sizeBytes"],
168 media_type=descriptor["mediaType"],
169 redacted=descriptor.get("redacted", False),
170 metadata=descriptor.get("metadata", {}),
171 )
173 def list(self) -> list[ArtifactRef]:
174 refs: list[ArtifactRef] = []
175 for path in sorted(self.meta_root.glob("*.json")):
176 hex_digest = path.stem
177 refs.append(self.describe(f"artifact://sha256/{hex_digest}"))
178 return refs
180 def path_for(self, reference: ArtifactRef | str) -> Path:
181 """Where the content of an artifact is stored (read-only use: an agent may read a
182 person's attachment from there)."""
183 uri = reference.uri if isinstance(reference, ArtifactRef) else reference
184 return self._blob_path(self._digest_from_uri(uri).split(":", 1)[1])
186 def _blob_path(self, hex_digest: str) -> Path:
187 if len(hex_digest) != 64 or any(char not in "0123456789abcdef" for char in hex_digest): 187 ↛ 188line 187 didn't jump to line 188 because the condition on line 187 was never true
188 raise ValueError("invalid sha256 digest")
189 return self.blob_root / hex_digest[:2] / hex_digest[2:]
191 @staticmethod
192 def _digest_from_uri(uri: str) -> str:
193 prefix = "artifact://sha256/"
194 if not uri.startswith(prefix): 194 ↛ 195line 194 didn't jump to line 195 because the condition on line 194 was never true
195 raise ValueError(f"unsupported artifact URI: {uri}")
196 return f"sha256:{uri[len(prefix) :]}"
198 @staticmethod
199 def _atomic_write(path: Path, data: bytes) -> None:
200 try:
201 path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
202 if path.exists():
203 return
204 descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
205 except OSError as error:
206 _raise_if_path_too_long(path, error)
207 raise
208 temporary = Path(temporary_name)
209 try:
210 # Wrap the descriptor first so that it is always closed. mkstemp already creates the
211 # file readable only by its owner; fchmod restates 0600 where it exists (POSIX).
212 with os.fdopen(descriptor, "wb") as handle:
213 if hasattr(os, "fchmod"): 213 ↛ 215line 213 didn't jump to line 215 because the condition on line 213 was always true
214 os.fchmod(handle.fileno(), 0o600)
215 handle.write(data)
216 handle.flush()
217 os.fsync(handle.fileno())
218 if sys.platform == "win32":
219 _move_into_place_on_windows(temporary, path, data)
220 else:
221 try:
222 os.replace(temporary, path)
223 except FileExistsError:
224 temporary.unlink(missing_ok=True)
225 finally:
226 temporary.unlink(missing_ok=True)
229WINDOWS_MAX_PATH = 260
230"""The Windows path limit (``MAX_PATH``, the final NUL included) without long path support."""
232ARTIFACT_PATH_HEADROOM = 90
233"""Characters an artifact file adds to the store directory: ``\\blobs\\sha256\\ab\\.`` plus
23462 hex digits and the temporary suffix, or ``\\metadata\\.`` plus 64 hex digits,
235``.json.`` and the suffix (89 at most)."""
238def windows_long_paths_enabled() -> bool:
239 """Whether Windows accepts paths over ``MAX_PATH`` (``LongPathsEnabled``); ``True``
240 elsewhere."""
241 if sys.platform != "win32":
242 return True
243 import winreg
245 try:
246 with winreg.OpenKey(
247 winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\CurrentControlSet\Control\FileSystem"
248 ) as key:
249 value, _kind = winreg.QueryValueEx(key, "LongPathsEnabled")
250 except OSError:
251 return False
252 return bool(value)
255def artifact_path_too_long(root: Path) -> bool:
256 """Whether artifact files under ``root`` would exceed the Windows path limit."""
257 return (
258 sys.platform == "win32"
259 and len(str(root)) + ARTIFACT_PATH_HEADROOM >= WINDOWS_MAX_PATH
260 and not windows_long_paths_enabled()
261 )
264def path_too_long_message(root: Path) -> str:
265 return (
266 f"the state directory {root} is {len(str(root))} characters long: its artifact files "
267 f"need about {ARTIFACT_PATH_HEADROOM} more and Windows refuses paths of "
268 f"{WINDOWS_MAX_PATH} characters or more without long path support. Set "
269 "runtime.stateDir (or move the workspace) to a shorter path, or enable long paths "
270 "(LongPathsEnabled=1 under HKLM\\SYSTEM\\CurrentControlSet\\Control\\FileSystem)"
271 )
274def _raise_if_path_too_long(path: Path, error: OSError) -> None:
275 """A clear error instead of a raw ``FileNotFoundError`` when an artifact path exceeds
276 the Windows limit."""
277 if ( 277 ↛ exitline 277 didn't return from function '_raise_if_path_too_long' because the condition on line 277 was always true
278 sys.platform == "win32"
279 and len(str(path)) + 10 >= WINDOWS_MAX_PATH
280 and not windows_long_paths_enabled()
281 ):
282 from governed_harness.domain.errors import ConfigurationError
284 store = next((item for item in path.parents if item.name in {"blobs", "metadata"}), path)
285 raise ConfigurationError(path_too_long_message(store.parent)) from error
288def _json_bytes(value: Any) -> bytes:
289 return json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False, default=str).encode(
290 "utf-8"
291 )
294def _parses(data: bytes) -> bool:
295 try:
296 json.loads(data)
297 except ValueError:
298 return False
299 return True
302def _move_into_place_on_windows(temporary: Path, path: Path, data: bytes) -> None:
303 """Move a written temporary file into place on Windows without disturbing readers.
305 ``os.replace`` over an existing file is not invisible on Windows as ``rename`` is on POSIX:
306 while ``MoveFileEx`` swaps the destination, another writer's move fails (``WinError 5``) and a
307 reader's open fails (``Errno 13``). Concurrent stores of the same blob hit both, because the
308 ``exists`` check before writing is a race. Blobs are named by the digest of their bytes, so:
310 - the temporary file is moved with ``os.rename``, which never replaces an existing file;
311 - a destination that already holds the bytes of ``data`` (same digest) is the store already
312 done: the temporary file is dropped and the destination is left untouched;
313 - a destination with other bytes (a descriptor with other metadata, a damaged blob) is never
314 accepted: it is overwritten with ``os.replace``, as POSIX does.
316 A failed move is retried with a short backoff; the last error is raised."""
317 expected = sha256_bytes(data)
318 delay = _WINDOWS_REPLACE_BACKOFF_SECONDS
319 for attempt in range(_WINDOWS_REPLACE_ATTEMPTS): 319 ↛ exitline 319 didn't return from function '_move_into_place_on_windows' because the loop on line 319 didn't complete
320 try:
321 _move_once_on_windows(temporary, path, expected)
322 return
323 except OSError:
324 if attempt == _WINDOWS_REPLACE_ATTEMPTS - 1:
325 raise
326 time.sleep(delay)
327 delay *= 2
330def _move_once_on_windows(temporary: Path, path: Path, expected: str) -> None:
331 """One attempt of :func:`_move_into_place_on_windows`; raises ``OSError`` when it failed."""
332 try:
333 os.rename(temporary, path)
334 except OSError:
335 existing = _digest_of(path)
336 if existing == expected:
337 temporary.unlink(missing_ok=True)
338 return
339 if existing is None:
340 raise
341 # Other bytes, read in full: overwrite them as POSIX does, never accept them.
342 os.replace(temporary, path)
345def _digest_of(path: Path) -> str | None:
346 """The digest of the bytes at ``path``; ``None`` when absent or not readable right now."""
347 try:
348 return sha256_bytes(path.read_bytes())
349 except OSError:
350 return None