Coverage for src/governed_harness/evidence/artifact_store.py: 90%

192 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1from __future__ import annotations 

2 

3import json 

4import os 

5import sys 

6import tempfile 

7import time 

8from dataclasses import dataclass 

9from pathlib import Path 

10from typing import Any 

11 

12from .hashing import sha256_bytes 

13from .redaction import PRIVATE_KEY, PRIVATE_KEY_MARK, SecretRedactor 

14 

15# Windows only: how many times the move into place is tried, and the first pause (doubled each time). 

16_WINDOWS_REPLACE_ATTEMPTS = 8 

17_WINDOWS_REPLACE_BACKOFF_SECONDS = 0.01 

18 

19 

20CODE_MEDIA_TYPES = frozenset({"text/x-diff"}) 

21"""Artifacts that hold source code: redacted with the code rules (quoted secrets only).""" 

22 

23 

24@dataclass(frozen=True) 

25class ArtifactRef: 

26 uri: str 

27 digest: str 

28 size_bytes: int 

29 media_type: str = "application/octet-stream" 

30 redacted: bool = False 

31 metadata: dict[str, Any] | None = None 

32 

33 

34class LocalArtifactStore: 

35 """Content-addressed, atomic local store with digest verification.""" 

36 

37 def __init__(self, root: Path, redactor: SecretRedactor | None = None) -> None: 

38 self.root = root.resolve(strict=False) 

39 self.blob_root = self.root / "blobs" / "sha256" 

40 self.meta_root = self.root / "metadata" 

41 self.blob_root.mkdir(parents=True, exist_ok=True, mode=0o700) 

42 self.meta_root.mkdir(parents=True, exist_ok=True, mode=0o700) 

43 self.redactor = redactor or SecretRedactor() 

44 

45 def put( 

46 self, 

47 data: bytes, 

48 *, 

49 media_type: str = "application/octet-stream", 

50 metadata: dict[str, Any] | None = None, 

51 redact: bool = True, 

52 secret_literals: tuple[bytes, ...] = (), 

53 ) -> ArtifactRef: 

54 code = media_type in CODE_MEDIA_TYPES 

55 redaction = self.redactor.redact(data, secret_literals, code=code) if redact else None 

56 stored = redaction.data if redaction else data 

57 rule_ids = redaction.rule_ids if redaction and redaction.redacted else () 

58 return self._store(stored, media_type, metadata, rule_ids) 

59 

60 def _store( 

61 self, 

62 stored: bytes, 

63 media_type: str, 

64 metadata: dict[str, Any] | None, 

65 rule_ids: tuple[str, ...], 

66 ) -> ArtifactRef: 

67 digest = sha256_bytes(stored) 

68 hex_digest = digest.split(":", 1)[1] 

69 path = self._blob_path(hex_digest) 

70 self._atomic_write(path, stored) 

71 meta = dict(metadata or {}) 

72 if rule_ids: 

73 meta["redactionRuleIds"] = list(rule_ids) 

74 redacted = bool(rule_ids) 

75 descriptor = { 

76 "digest": digest, 

77 "sizeBytes": len(stored), 

78 "mediaType": media_type, 

79 "redacted": redacted, 

80 "metadata": meta, 

81 } 

82 self._atomic_write( 

83 self.meta_root / f"{hex_digest}.json", 

84 json.dumps(descriptor, sort_keys=True, indent=2).encode("utf-8"), 

85 ) 

86 return ArtifactRef( 

87 uri=f"artifact://sha256/{hex_digest}", 

88 digest=digest, 

89 size_bytes=len(stored), 

90 media_type=media_type, 

91 redacted=redacted, 

92 metadata=meta, 

93 ) 

94 

95 def put_json(self, value: Any, *, metadata: dict[str, Any] | None = None) -> ArtifactRef: 

96 """Store ``value`` as JSON, redacted as text like any artifact and on its decoded 

97 strings. The serialized text escapes quotes, so a rule run over it misses a secret whose 

98 value is quoted inside a JSON string (``password = \\"...\\"``, #178), and it can cut an 

99 escape or span two strings and leave a document that no longer parses (#152). When any 

100 decoded string redacts, or the text redaction breaks the document, the strings are 

101 redacted one by one instead; a document whose strings have nothing to redact is stored 

102 exactly as before, with the same bytes and digest. 

103 

104 A redacted JSON artifact is evidence, not data: a caller that reads one back to act on 

105 it (a pinned task, a workspace snapshot) checks the ``redacted`` flag of its descriptor 

106 and never applies a redacted value.""" 

107 data = _json_bytes(value) 

108 redaction = self.redactor.redact(data) 

109 redacted, rule_ids = self.redactor.redact_json(value) 

110 if not rule_ids and (not redaction.redacted or _parses(redaction.data)): 

111 return self._store(redaction.data, "application/json", metadata, redaction.rule_ids) 

112 data = _json_bytes(redacted) 

113 # A private key whose lines are separate strings is caught only on the whole text: the 

114 # text redaction is kept when it parses, otherwise nothing of the document is kept. 

115 spanning, count = PRIVATE_KEY.subn(PRIVATE_KEY_MARK, data) 

116 if count: 

117 data = ( 

118 spanning 

119 if _parses(spanning) 

120 else _json_bytes({"redactedDocument": PRIVATE_KEY_MARK.decode()}) 

121 ) 

122 if "private_key" not in rule_ids: 122 ↛ 124line 122 didn't jump to line 124 because the condition on line 122 was always true

123 rule_ids = (*rule_ids, "private_key") 

124 return self._store(data, "application/json", metadata, rule_ids) 

125 

126 def get(self, reference: ArtifactRef | str) -> bytes: 

127 digest = ( 

128 reference.digest 

129 if isinstance(reference, ArtifactRef) 

130 else self._digest_from_uri(reference) 

131 ) 

132 hex_digest = digest.split(":", 1)[1] 

133 path = self._blob_path(hex_digest) 

134 data = path.read_bytes() 

135 if sha256_bytes(data) != digest: 

136 raise ValueError("artifact digest mismatch") 

137 return data 

138 

139 def verify(self, reference: ArtifactRef | str) -> bool: 

140 try: 

141 self.get(reference) 

142 return True 

143 except (OSError, ValueError): 

144 return False 

145 

146 def delete(self, reference: ArtifactRef | str) -> bool: 

147 """Remove a blob and its descriptor (retention); ``False`` when it was not stored.""" 

148 digest = ( 

149 reference.digest 

150 if isinstance(reference, ArtifactRef) 

151 else self._digest_from_uri(reference) 

152 ) 

153 hex_digest = digest.split(":", 1)[1] 

154 path = self._blob_path(hex_digest) 

155 existed = path.exists() 

156 path.unlink(missing_ok=True) 

157 (self.meta_root / f"{hex_digest}.json").unlink(missing_ok=True) 

158 return existed 

159 

160 def describe(self, uri: str) -> ArtifactRef: 

161 digest = self._digest_from_uri(uri) 

162 hex_digest = digest.split(":", 1)[1] 

163 descriptor = json.loads((self.meta_root / f"{hex_digest}.json").read_text(encoding="utf-8")) 

164 return ArtifactRef( 

165 uri=uri, 

166 digest=digest, 

167 size_bytes=descriptor["sizeBytes"], 

168 media_type=descriptor["mediaType"], 

169 redacted=descriptor.get("redacted", False), 

170 metadata=descriptor.get("metadata", {}), 

171 ) 

172 

173 def list(self) -> list[ArtifactRef]: 

174 refs: list[ArtifactRef] = [] 

175 for path in sorted(self.meta_root.glob("*.json")): 

176 hex_digest = path.stem 

177 refs.append(self.describe(f"artifact://sha256/{hex_digest}")) 

178 return refs 

179 

180 def path_for(self, reference: ArtifactRef | str) -> Path: 

181 """Where the content of an artifact is stored (read-only use: an agent may read a 

182 person's attachment from there).""" 

183 uri = reference.uri if isinstance(reference, ArtifactRef) else reference 

184 return self._blob_path(self._digest_from_uri(uri).split(":", 1)[1]) 

185 

186 def _blob_path(self, hex_digest: str) -> Path: 

187 if len(hex_digest) != 64 or any(char not in "0123456789abcdef" for char in hex_digest): 187 ↛ 188line 187 didn't jump to line 188 because the condition on line 187 was never true

188 raise ValueError("invalid sha256 digest") 

189 return self.blob_root / hex_digest[:2] / hex_digest[2:] 

190 

191 @staticmethod 

192 def _digest_from_uri(uri: str) -> str: 

193 prefix = "artifact://sha256/" 

194 if not uri.startswith(prefix): 194 ↛ 195line 194 didn't jump to line 195 because the condition on line 194 was never true

195 raise ValueError(f"unsupported artifact URI: {uri}") 

196 return f"sha256:{uri[len(prefix) :]}" 

197 

198 @staticmethod 

199 def _atomic_write(path: Path, data: bytes) -> None: 

200 try: 

201 path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) 

202 if path.exists(): 

203 return 

204 descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) 

205 except OSError as error: 

206 _raise_if_path_too_long(path, error) 

207 raise 

208 temporary = Path(temporary_name) 

209 try: 

210 # Wrap the descriptor first so that it is always closed. mkstemp already creates the 

211 # file readable only by its owner; fchmod restates 0600 where it exists (POSIX). 

212 with os.fdopen(descriptor, "wb") as handle: 

213 if hasattr(os, "fchmod"): 213 ↛ 215line 213 didn't jump to line 215 because the condition on line 213 was always true

214 os.fchmod(handle.fileno(), 0o600) 

215 handle.write(data) 

216 handle.flush() 

217 os.fsync(handle.fileno()) 

218 if sys.platform == "win32": 

219 _move_into_place_on_windows(temporary, path, data) 

220 else: 

221 try: 

222 os.replace(temporary, path) 

223 except FileExistsError: 

224 temporary.unlink(missing_ok=True) 

225 finally: 

226 temporary.unlink(missing_ok=True) 

227 

228 

229WINDOWS_MAX_PATH = 260 

230"""The Windows path limit (``MAX_PATH``, the final NUL included) without long path support.""" 

231 

232ARTIFACT_PATH_HEADROOM = 90 

233"""Characters an artifact file adds to the store directory: ``\\blobs\\sha256\\ab\\.`` plus 

23462 hex digits and the temporary suffix, or ``\\metadata\\.`` plus 64 hex digits, 

235``.json.`` and the suffix (89 at most).""" 

236 

237 

238def windows_long_paths_enabled() -> bool: 

239 """Whether Windows accepts paths over ``MAX_PATH`` (``LongPathsEnabled``); ``True`` 

240 elsewhere.""" 

241 if sys.platform != "win32": 

242 return True 

243 import winreg 

244 

245 try: 

246 with winreg.OpenKey( 

247 winreg.HKEY_LOCAL_MACHINE, r"SYSTEM\CurrentControlSet\Control\FileSystem" 

248 ) as key: 

249 value, _kind = winreg.QueryValueEx(key, "LongPathsEnabled") 

250 except OSError: 

251 return False 

252 return bool(value) 

253 

254 

255def artifact_path_too_long(root: Path) -> bool: 

256 """Whether artifact files under ``root`` would exceed the Windows path limit.""" 

257 return ( 

258 sys.platform == "win32" 

259 and len(str(root)) + ARTIFACT_PATH_HEADROOM >= WINDOWS_MAX_PATH 

260 and not windows_long_paths_enabled() 

261 ) 

262 

263 

264def path_too_long_message(root: Path) -> str: 

265 return ( 

266 f"the state directory {root} is {len(str(root))} characters long: its artifact files " 

267 f"need about {ARTIFACT_PATH_HEADROOM} more and Windows refuses paths of " 

268 f"{WINDOWS_MAX_PATH} characters or more without long path support. Set " 

269 "runtime.stateDir (or move the workspace) to a shorter path, or enable long paths " 

270 "(LongPathsEnabled=1 under HKLM\\SYSTEM\\CurrentControlSet\\Control\\FileSystem)" 

271 ) 

272 

273 

274def _raise_if_path_too_long(path: Path, error: OSError) -> None: 

275 """A clear error instead of a raw ``FileNotFoundError`` when an artifact path exceeds 

276 the Windows limit.""" 

277 if ( 277 ↛ exitline 277 didn't return from function '_raise_if_path_too_long' because the condition on line 277 was always true

278 sys.platform == "win32" 

279 and len(str(path)) + 10 >= WINDOWS_MAX_PATH 

280 and not windows_long_paths_enabled() 

281 ): 

282 from governed_harness.domain.errors import ConfigurationError 

283 

284 store = next((item for item in path.parents if item.name in {"blobs", "metadata"}), path) 

285 raise ConfigurationError(path_too_long_message(store.parent)) from error 

286 

287 

288def _json_bytes(value: Any) -> bytes: 

289 return json.dumps(value, sort_keys=True, indent=2, ensure_ascii=False, default=str).encode( 

290 "utf-8" 

291 ) 

292 

293 

294def _parses(data: bytes) -> bool: 

295 try: 

296 json.loads(data) 

297 except ValueError: 

298 return False 

299 return True 

300 

301 

302def _move_into_place_on_windows(temporary: Path, path: Path, data: bytes) -> None: 

303 """Move a written temporary file into place on Windows without disturbing readers. 

304 

305 ``os.replace`` over an existing file is not invisible on Windows as ``rename`` is on POSIX: 

306 while ``MoveFileEx`` swaps the destination, another writer's move fails (``WinError 5``) and a 

307 reader's open fails (``Errno 13``). Concurrent stores of the same blob hit both, because the 

308 ``exists`` check before writing is a race. Blobs are named by the digest of their bytes, so: 

309 

310 - the temporary file is moved with ``os.rename``, which never replaces an existing file; 

311 - a destination that already holds the bytes of ``data`` (same digest) is the store already 

312 done: the temporary file is dropped and the destination is left untouched; 

313 - a destination with other bytes (a descriptor with other metadata, a damaged blob) is never 

314 accepted: it is overwritten with ``os.replace``, as POSIX does. 

315 

316 A failed move is retried with a short backoff; the last error is raised.""" 

317 expected = sha256_bytes(data) 

318 delay = _WINDOWS_REPLACE_BACKOFF_SECONDS 

319 for attempt in range(_WINDOWS_REPLACE_ATTEMPTS): 319 ↛ exitline 319 didn't return from function '_move_into_place_on_windows' because the loop on line 319 didn't complete

320 try: 

321 _move_once_on_windows(temporary, path, expected) 

322 return 

323 except OSError: 

324 if attempt == _WINDOWS_REPLACE_ATTEMPTS - 1: 

325 raise 

326 time.sleep(delay) 

327 delay *= 2 

328 

329 

330def _move_once_on_windows(temporary: Path, path: Path, expected: str) -> None: 

331 """One attempt of :func:`_move_into_place_on_windows`; raises ``OSError`` when it failed.""" 

332 try: 

333 os.rename(temporary, path) 

334 except OSError: 

335 existing = _digest_of(path) 

336 if existing == expected: 

337 temporary.unlink(missing_ok=True) 

338 return 

339 if existing is None: 

340 raise 

341 # Other bytes, read in full: overwrite them as POSIX does, never accept them. 

342 os.replace(temporary, path) 

343 

344 

345def _digest_of(path: Path) -> str | None: 

346 """The digest of the bytes at ``path``; ``None`` when absent or not readable right now.""" 

347 try: 

348 return sha256_bytes(path.read_bytes()) 

349 except OSError: 

350 return None