Coverage for src/governed_harness/runtime/snapshots.py: 84%

267 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Workspace snapshots for large repositories (``workspace.snapshot``, ``workspace.baseline``, 

2``workspace.snapshotCache``; since 2.0). 

3 

4In 1.0.0 every snapshot walks the whole workspace, hashes every file, reads the text of every 

5file and stores the baseline as one JSON artifact with the text of every file: on a repository 

6of 10,000 files (117 MB) one run took 21.5 s, peaked at 1.29 GB and stored a 107 MB blob that 

7also held ``.env``. The settings, each optional: 

8 

9* ``snapshot: git`` lists the files through Git: tracked files plus untracked files that 

10 ``.gitignore`` does not exclude, minus the directories the ChangeSet always excludes and 

11 symbolic links. Ignored files are never read, hashed or stored. A workspace that is not a Git 

12 repository is walked as before. 

13* ``baseline: manifest`` stores a snapshot as a manifest of digests. The text is kept only for 

14 files whose content Git cannot give back later (untracked or modified at the time of the 

15 snapshot); every other file is read from the recorded ``HEAD`` when, and only when, it enters 

16 a diff. A file whose blob does not match its recorded digest (a Git filter changed it) diffs 

17 as binary. 

18* ``snapshotCache: true`` keeps, in ``.harness/cache/snapshot-cache.json``, the digest of each 

19 file with its size, modification time, change time and inode; a file whose four values did 

20 not change is not hashed again. The change time cannot be set by a process (``utime`` updates 

21 it), so restoring a file's modification time does not hide an edit. Entries younger than two 

22 seconds are not cached. Windows reports the creation time instead of the change time, so the 

23 cache is not used there. 

24 

25Without any of them a snapshot is taken, stored and diffed exactly as in 1.0.0.""" 

26 

27from __future__ import annotations 

28 

29import json 

30import os 

31import subprocess # nosec B404 - git ls-files with a fixed argv, no shell 

32import tempfile 

33import time 

34from collections.abc import Callable, Iterator 

35from dataclasses import dataclass 

36from pathlib import Path, PurePosixPath 

37from typing import Any 

38 

39from governed_harness.configuration.models import WorkspaceConfig 

40from governed_harness.evidence.artifact_store import ArtifactRef, LocalArtifactStore 

41from governed_harness.evidence.hashing import sha256_bytes, sha256_file, sha256_json 

42from governed_harness.runtime.workspace import ( 

43 DEFAULT_EXCLUDES, 

44 FileState, 

45 TextResolver, 

46 WorkspaceDiff, 

47 WorkspaceSnapshot, 

48 WorkspaceSnapshotter, 

49 decode_text, 

50 recorded_bytes, 

51) 

52 

53MANIFEST_FORMAT = "manifest/1" 

54CACHE_FILE = Path("cache") / "snapshot-cache.json" 

55SNAPSHOT_CACHE_SEAL = "snapshot-cache-seal" 

56"""State flag holding the digest of the cache file.""" 

57RACY_SECONDS = 2.0 

58_GIT_TIMEOUT_SECONDS = 300 

59 

60 

61@dataclass(frozen=True) 

62class SnapshotSettings: 

63 git_listing: bool = False 

64 manifest: bool = False 

65 cache: bool = False 

66 

67 @classmethod 

68 def from_config(cls, workspace: WorkspaceConfig) -> SnapshotSettings: 

69 return cls( 

70 git_listing=workspace.snapshot == "git", 

71 manifest=workspace.baseline == "manifest", 

72 cache=bool(workspace.snapshot_cache), 

73 ) 

74 

75 @property 

76 def scaled(self) -> bool: 

77 """Whether any 2.0 setting is on (otherwise everything works as in 1.0.0).""" 

78 return self.git_listing or self.manifest or self.cache 

79 

80 

81def _git(root: Path, *args: str) -> subprocess.CompletedProcess[bytes] | None: 

82 try: 

83 return subprocess.run( # nosec B603 B607 - git from PATH, argv built here, no shell 

84 ["git", *args], 

85 cwd=root, 

86 stdin=subprocess.DEVNULL, 

87 capture_output=True, 

88 check=False, 

89 timeout=_GIT_TIMEOUT_SECONDS, 

90 ) 

91 except (OSError, subprocess.SubprocessError): 

92 return None 

93 

94 

95def git_head(root: Path) -> str | None: 

96 result = _git(root, "rev-parse", "--verify", "--quiet", "HEAD^{commit}") 

97 if result is None or result.returncode != 0: 97 ↛ 98line 97 didn't jump to line 98 because the condition on line 97 was never true

98 return None 

99 return result.stdout.decode().strip() or None 

100 

101 

102def git_listing(root: Path) -> list[str] | None: 

103 """Tracked files and untracked files Git does not ignore; ``None`` outside a repository.""" 

104 result = _git(root, "ls-files", "-z", "--cached", "--others", "--exclude-standard") 

105 if result is None or result.returncode != 0: 105 ↛ 106line 105 didn't jump to line 106 because the condition on line 105 was never true

106 return None 

107 names = result.stdout.decode("utf-8", "surrogateescape").split("\x00") 

108 return sorted({name for name in names if name}) 

109 

110 

111def git_ignored(root: Path) -> list[str]: 

112 """Untracked files ``.gitignore`` excludes (for the out-of-ChangeSet guard).""" 

113 result = _git( 

114 root, "ls-files", "-z", "--others", "--ignored", "--exclude-standard", "--directory" 

115 ) 

116 if result is None or result.returncode != 0: 116 ↛ 117line 116 didn't jump to line 117 because the condition on line 116 was never true

117 return [] 

118 return sorted( 

119 name for name in result.stdout.decode("utf-8", "surrogateescape").split("\x00") if name 

120 ) 

121 

122 

123def git_dirty(root: Path) -> set[str] | None: 

124 """Paths whose content Git cannot give back from ``HEAD`` (modified, added, untracked).""" 

125 result = _git(root, "status", "--porcelain=v1", "-z", "--untracked-files=all", "--no-renames") 

126 if result is None or result.returncode != 0: 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true

127 return None 

128 dirty: set[str] = set() 

129 for entry in result.stdout.decode("utf-8", "surrogateescape").split("\x00"): 

130 if len(entry) > 3: 

131 dirty.add(entry[3:]) 

132 return dirty 

133 

134 

135def git_blob(root: Path, revision: str, path: str) -> bytes | None: 

136 result = _git(root, "cat-file", "blob", f"{revision}:{path}") 

137 if result is None or result.returncode != 0: 137 ↛ 138line 137 didn't jump to line 138 because the condition on line 137 was never true

138 return None 

139 return result.stdout 

140 

141 

142def _excluded(relative: str, excludes: set[str]) -> bool: 

143 return any(part in excludes for part in PurePosixPath(relative).parts[:-1]) 

144 

145 

146Seal = tuple[Callable[[], str | None], Callable[[str], None]] 

147"""Read and write the digest of the cache file kept outside it (in the state database).""" 

148 

149 

150class SnapshotCache: 

151 """The digest cache. Its file is sealed: the digest of its content is kept in the state 

152 database after every write, and a file that does not match the seal (edited by anything 

153 but the harness) is ignored, so every file is hashed again.""" 

154 

155 def __init__(self, path: Path, seal: Seal | None = None) -> None: 

156 self.path = path 

157 self.seal = seal 

158 self.entries: dict[str, list[Any]] = {} 

159 self.hits = 0 

160 self.misses = 0 

161 try: 

162 data = path.read_bytes() 

163 if seal is not None and seal[0]() != sha256_bytes(data): 

164 return 

165 value = json.loads(data) 

166 if value.get("version") == 1 and isinstance(value.get("entries"), dict): 166 ↛ exitline 166 didn't return from function '__init__' because the condition on line 166 was always true

167 self.entries = value["entries"] 

168 except (OSError, ValueError, AttributeError): 

169 self.entries = {} 

170 

171 @staticmethod 

172 def key(stat: os.stat_result) -> list[int]: 

173 return [stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns, stat.st_ino] 

174 

175 def digest(self, relative: str, stat: os.stat_result) -> str | None: 

176 entry = self.entries.get(relative) 

177 if entry is not None and entry[:4] == self.key(stat): 

178 self.hits += 1 

179 return str(entry[4]) 

180 self.misses += 1 

181 return None 

182 

183 def save(self, current: dict[str, list[Any]]) -> None: 

184 self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) 

185 data = json.dumps({"version": 1, "entries": current}, separators=(",", ":")).encode() 

186 handle, name = tempfile.mkstemp(prefix=".snapshot-cache.", dir=self.path.parent) 

187 try: 

188 with os.fdopen(handle, "wb") as stream: 

189 stream.write(data) 

190 os.replace(name, self.path) 

191 finally: 

192 Path(name).unlink(missing_ok=True) 

193 if self.seal is not None: 

194 self.seal[1](sha256_bytes(data)) 

195 

196 

197class ScaledSnapshotter(WorkspaceSnapshotter): 

198 """A snapshotter that lists through Git, caches digests and reads text lazily.""" 

199 

200 def __init__( 

201 self, 

202 root: Path, 

203 settings: SnapshotSettings, 

204 cache_path: Path | None = None, 

205 seal: Seal | None = None, 

206 ) -> None: 

207 super().__init__(root) 

208 self.settings = settings 

209 self.cache_path = cache_path 

210 self.seal = seal 

211 

212 def _paths(self) -> Iterator[str]: 

213 listed = git_listing(self.root) if self.settings.git_listing else None 

214 if listed is None: 214 ↛ 215line 214 didn't jump to line 215 because the condition on line 214 was never true

215 for directory, dirnames, filenames in os.walk(self.root, followlinks=False): 

216 dirnames[:] = [ 

217 name 

218 for name in sorted(dirnames) 

219 if name not in self.excludes and not (Path(directory) / name).is_symlink() 

220 ] 

221 for filename in sorted(filenames): 

222 yield (Path(directory) / filename).relative_to(self.root).as_posix() 

223 return 

224 for relative in listed: 

225 if not _excluded(relative, self.excludes): 

226 yield relative 

227 

228 def _cache(self) -> SnapshotCache | None: 

229 # Windows reports the creation time as st_ctime, not the change time: a restored 

230 # modification time would hide an edit, so every file is hashed there. 

231 return ( 

232 SnapshotCache(self.cache_path, self.seal) 

233 if self.settings.cache and self.cache_path and os.name != "nt" 

234 else None 

235 ) 

236 

237 def _hashed( 

238 self, relative: str, cache: SnapshotCache | None, read_text: bool 

239 ) -> tuple[os.stat_result, str, str | None] | None: 

240 """The stat, digest and text of a regular file; ``None`` when it cannot be read.""" 

241 path = self.root / relative 

242 try: 

243 stat = path.lstat() 

244 except OSError: 

245 return None 

246 if not os.path.isfile(path) or path.is_symlink(): 246 ↛ 247line 246 didn't jump to line 247 because the condition on line 246 was never true

247 return None 

248 try: 

249 digest = cache.digest(relative, stat) if cache else None 

250 if digest is None: 

251 digest = sha256_file(path) 

252 text = self._read_text(path, stat.st_size) if read_text else None 

253 except OSError: 

254 return None 

255 return stat, digest, text 

256 

257 def snapshot(self, *, read_text: bool = False) -> WorkspaceSnapshot: 

258 cache = self._cache() 

259 fresh: dict[str, list[Any]] = {} 

260 states: dict[str, FileState] = {} 

261 now_ns = time.time_ns() 

262 for relative in self._paths(): 

263 hashed = self._hashed(relative, cache, read_text) 

264 if hashed is None: 264 ↛ 265line 264 didn't jump to line 265 because the condition on line 264 was never true

265 continue 

266 stat, digest, text = hashed 

267 if cache is not None and now_ns - stat.st_mtime_ns > RACY_SECONDS * 1e9: 

268 fresh[relative] = [*SnapshotCache.key(stat), digest] 

269 states[relative] = FileState(relative, digest, stat.st_size, text, read_text) 

270 if cache is not None: 

271 cache.save(fresh) 

272 digest = sha256_json({path: state.digest for path, state in sorted(states.items())}) 

273 return WorkspaceSnapshot(files=states, digest=digest) 

274 

275 

276@dataclass 

277class StoredSnapshot: 

278 snapshot: WorkspaceSnapshot 

279 git_head: str | None = None 

280 inline: dict[str, str | None] | None = None 

281 """Manifest format: path -> text kept in the manifest (files Git cannot give back).""" 

282 

283 

284class SnapshotStore: 

285 """Take, store and load workspace snapshots under the workspace settings.""" 

286 

287 def __init__( 

288 self, 

289 workspace: Path, 

290 artifacts: LocalArtifactStore, 

291 settings: SnapshotSettings, 

292 harness_dir: Path | None = None, 

293 seal: Seal | None = None, 

294 ) -> None: 

295 self.workspace = workspace 

296 self.artifacts = artifacts 

297 self.settings = settings 

298 self.cache_path = (harness_dir or workspace / ".harness") / CACHE_FILE 

299 self.seal = seal 

300 

301 def snapshotter(self) -> WorkspaceSnapshotter: 

302 if not self.settings.scaled: 

303 return WorkspaceSnapshotter(self.workspace) 

304 return ScaledSnapshotter(self.workspace, self.settings, self.cache_path, self.seal) 

305 

306 def take(self, *, for_storage: bool = False) -> WorkspaceSnapshot: 

307 """A snapshot of the workspace. Under the 1.0.0 settings it reads every text; a scaled 

308 snapshot reads text only when it is stored in the 1.0.0 format.""" 

309 snapshotter = self.snapshotter() 

310 if isinstance(snapshotter, ScaledSnapshotter): 

311 return snapshotter.snapshot(read_text=for_storage and not self.settings.manifest) 

312 return snapshotter.snapshot() 

313 

314 def store(self, snapshot: WorkspaceSnapshot, metadata: dict[str, Any]) -> ArtifactRef: 

315 if not self.settings.manifest: 

316 return self.artifacts.put_json(text_document(snapshot), metadata=metadata) 

317 head = git_head(self.workspace) 

318 dirty = git_dirty(self.workspace) if head else None 

319 files: dict[str, Any] = {} 

320 for path, state in snapshot.files.items(): 

321 entry: dict[str, Any] = { 

322 "path": state.path, 

323 "digest": state.digest, 

324 "sizeBytes": state.size_bytes, 

325 } 

326 if head is None or dirty is None or path in dirty: 

327 entry["text"] = self._manifest_text(path, state) 

328 files[path] = entry 

329 document = { 

330 "format": MANIFEST_FORMAT, 

331 "digest": snapshot.digest, 

332 "gitHead": head, 

333 "files": files, 

334 } 

335 return self.artifacts.put_json(document, metadata=metadata) 

336 

337 def _manifest_text(self, path: str, state: FileState) -> str | None: 

338 """The text a manifest keeps for a file: the snapshot's own, or the workspace bytes 

339 while they still match the recorded digest.""" 

340 if state.text_loaded: 340 ↛ 341line 340 didn't jump to line 341 because the condition on line 340 was never true

341 return state.text 

342 target = self.workspace / path 

343 try: 

344 data = target.read_bytes() 

345 except OSError: 

346 data = None 

347 return ( 

348 decode_text(data) if data is not None and sha256_bytes(data) == state.digest else None 

349 ) 

350 

351 def load(self, uri: str) -> StoredSnapshot: 

352 value = json.loads(self.artifacts.get(uri)) 

353 if value.get("format") != MANIFEST_FORMAT: 

354 return StoredSnapshot(from_text_document(value)) 

355 inline: dict[str, str | None] = {} 

356 states: dict[str, FileState] = {} 

357 for path, item in value["files"].items(): 

358 if "text" in item: 

359 inline[path] = item["text"] 

360 states[path] = FileState(path, item["digest"], item["sizeBytes"], item["text"]) 

361 else: 

362 states[path] = FileState(path, item["digest"], item["sizeBytes"], None, False) 

363 return StoredSnapshot( 

364 WorkspaceSnapshot(files=states, digest=value["digest"]), value.get("gitHead"), inline 

365 ) 

366 

367 def content(self, stored: StoredSnapshot, path: str) -> bytes | None: 

368 """The bytes of a file as the snapshot recorded it, when they can be given back.""" 

369 state = stored.snapshot.files.get(path) 

370 if state is None: 370 ↛ 371line 370 didn't jump to line 371 because the condition on line 370 was never true

371 return None 

372 if state.text_loaded: 

373 if state.text is None: 373 ↛ 374line 373 didn't jump to line 374 because the condition on line 373 was never true

374 return None 

375 recorded = recorded_bytes(state) 

376 if recorded is not None: 

377 return recorded 

378 # A text stored redacted is not the file's content (#178): Git may still give the 

379 # recorded bytes back, the redacted text never is. 

380 if stored.git_head is None: 

381 return None 

382 data = git_blob(self.workspace, stored.git_head, path) 

383 return data if data is not None and sha256_bytes(data) == state.digest else None 

384 

385 def resolver(self, stored: StoredSnapshot) -> TextResolver: 

386 def resolve(side: str, state: FileState) -> str | None: 

387 if side == "before": 

388 data = self.content(stored, state.path) 

389 return decode_text(data) if data is not None else None 

390 target = self.workspace / state.path 

391 try: 

392 data = target.read_bytes() 

393 except OSError: 

394 return None 

395 return decode_text(data) if sha256_bytes(data) == state.digest else None 

396 

397 return resolve 

398 

399 def diff(self, stored: StoredSnapshot, current: WorkspaceSnapshot) -> WorkspaceDiff: 

400 return self.snapshotter().diff(stored.snapshot, current, resolve_text=self.resolver(stored)) 

401 

402 

403def text_document(snapshot: WorkspaceSnapshot) -> dict[str, Any]: 

404 """The 1.0.0 stored form: every file with its text.""" 

405 return { 

406 "digest": snapshot.digest, 

407 "files": { 

408 path: { 

409 "path": state.path, 

410 "digest": state.digest, 

411 "sizeBytes": state.size_bytes, 

412 "text": state.text, 

413 } 

414 for path, state in snapshot.files.items() 

415 }, 

416 } 

417 

418 

419def from_text_document(value: dict[str, Any]) -> WorkspaceSnapshot: 

420 return WorkspaceSnapshot( 

421 files={ 

422 path: FileState( 

423 path=item["path"], 

424 digest=item["digest"], 

425 size_bytes=item["sizeBytes"], 

426 text=item.get("text"), 

427 ) 

428 for path, item in value["files"].items() 

429 }, 

430 digest=value["digest"], 

431 ) 

432 

433 

434def ignored_outside_excludes(root: Path) -> list[str]: 

435 """Ignored files outside the directories the ChangeSet always excludes: under ``snapshot: 

436 git`` they leave the ChangeSet, so the out-of-ChangeSet guard watches them.""" 

437 found: list[str] = [] 

438 for name in git_ignored(root): 

439 if name.endswith("/"): 

440 found.extend(_ignored_directory_files(root, name.rstrip("/"))) 

441 elif not _excluded(name, set(DEFAULT_EXCLUDES)): 

442 found.append(name) 

443 return sorted(found) 

444 

445 

446def _ignored_directory_files(root: Path, directory: str) -> list[str]: 

447 """The files under an ignored directory, unless the ChangeSet excludes it anyway.""" 

448 found: list[str] = [] 

449 if _excluded(f"{directory}/x", set(DEFAULT_EXCLUDES)): 449 ↛ 451line 449 didn't jump to line 451 because the condition on line 449 was always true

450 return found 

451 base = root / directory 

452 if base.is_symlink() or not base.is_dir(): 

453 return found 

454 for current, dirnames, filenames in os.walk(base, followlinks=False): 

455 dirnames[:] = [item for item in dirnames if item not in DEFAULT_EXCLUDES] 

456 for filename in filenames: 

457 found.append((Path(current) / filename).relative_to(root).as_posix()) 

458 return found