Coverage for src/governed_harness/orchestration/stop_line.py: 89%

233 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""Stop the line (#52, N2): what happens to the changes of a run that stops without approval. 

2 

3Before 2.0 a rejected, failed, timed-out or cancelled run left its changes in the workspace: the 

4next step of a plan started on top of code the harness never approved, and the evaluated 

5deliverable contained seven such steps. Under ``governance.stopTheLine``: 

6 

7* ``restore``: the changes since DISCOVERY are kept as a quarantined patch (an artifact on the 

8 run's chain, with the list of paths and the content of every restored path) and the workspace 

9 is restored to the baseline; a file whose baseline content was not recorded as text cannot be 

10 restored and the line is blocked instead. A run that stopped after IMPLEMENTATION and is 

11 continued gets its quarantined changes back before it is evaluated again, unless a new 

12 attempt was written over a restored path (the quarantine is then superseded): the recorded 

13 content must match its digest, otherwise nothing is written and the run is BLOCKED; 

14* ``block``: the changes stay, and no new run starts in the workspace until a person 

15 quarantines them with ``harness run quarantine`` (which restores the baseline) or the run is 

16 approved. 

17 

18Under either mode a task that declares ``ownedPaths`` gets a HIGH finding in VERIFICATION for 

19every changed path outside them, so an out-of-scope write is no longer silent.""" 

20 

21from __future__ import annotations 

22 

23import fnmatch 

24from pathlib import Path 

25from typing import TYPE_CHECKING, Any 

26 

27from governed_harness.capabilities.authorizer import contained_path 

28from governed_harness.domain.enums import ( 

29 FindingSeverity, 

30 PhaseId, 

31 ResultStatus, 

32 ValidationKind, 

33) 

34from governed_harness.domain.errors import PolicyViolationError 

35from governed_harness.domain.models import ChangeSet, Execution, utc_now 

36from governed_harness.evidence.hashing import sha256_bytes, sha256_file 

37from governed_harness.orchestration.workspace_ops import restore_changes 

38from governed_harness.runtime.guard import IGNORED_PATTERNS 

39from governed_harness.runtime.workspace import WorkspaceDiff, WorkspaceSnapshot 

40from governed_harness.validators import ValidatorOutput 

41 

42if TYPE_CHECKING: 

43 from governed_harness.orchestration.hosts import ResultsHost 

44 

45OWNED_PATHS_ID = "harness.owned-paths" 

46STOPPED_STATUSES = frozenset({ResultStatus.FAILED, ResultStatus.TIMED_OUT, ResultStatus.ERROR}) 

47_STOPPED_PHASES = frozenset({PhaseId.IMPLEMENTATION, PhaseId.VERIFICATION}) 

48REAPPLY_PHASES = frozenset( 

49 {PhaseId.VERIFICATION, PhaseId.INDEPENDENT_REVIEW, PhaseId.DECISION, PhaseId.CLOSURE} 

50) 

51"""The phases that evaluate the run's ChangeSet: a run quarantined in one of them and continued 

52gets its quarantined changes back. A run quarantined in IMPLEMENTATION implements again from the 

53baseline.""" 

54 

55 

56def _write_checked(path: str, target: Path, data: bytes | None) -> None: 

57 """Write ``data`` to ``target`` (``None`` deletes it) and check it holds it afterwards.""" 

58 if data is None: 

59 target.unlink(missing_ok=True) 

60 else: 

61 target.parent.mkdir(parents=True, exist_ok=True) 

62 target.write_bytes(data) 

63 expected = sha256_bytes(data) if data is not None else None 

64 if (sha256_file(target) if target.is_file() else None) != expected: 64 ↛ 65line 64 didn't jump to line 65 because the condition on line 64 was never true

65 raise OSError(f"{path} does not hold the quarantined content after writing") 

66 

67 

68def _undo_writes(written: list[tuple[Path, bytes | None]]) -> None: 

69 """Put back the previous content of each written target, the last written first.""" 

70 for target, previous in reversed(written): 

71 if previous is None: 

72 target.unlink(missing_ok=True) 

73 else: 

74 target.write_bytes(previous) 

75 

76 

77class StopLine: 

78 def __init__(self, results: ResultsHost) -> None: 

79 self.results = results 

80 

81 @property 

82 def mode(self) -> str: 

83 return self.results.project.governance_settings.stop_the_line or "off" 

84 

85 def _flag(self, project_id: str) -> str: 

86 return f"stopline:{project_id}" 

87 

88 # ----- new runs -------------------------------------------------------------------------- 

89 def check_new_run(self, project_id: str) -> None: 

90 """Refuse a new run while a stopped run's changes are in the workspace (``block``).""" 

91 if self.mode == "off": 

92 return 

93 stopped = self.results.flag_json(self._flag(project_id)) 

94 if isinstance(stopped, dict) and stopped.get("executionId"): 

95 raise PolicyViolationError( 

96 f"run {stopped['executionId']} stopped without approval ({stopped.get('reason')}) " 

97 f"and its changes are still in the workspace: quarantine them with harness run " 

98 f"quarantine --run {stopped['executionId']} (governance.stopTheLine)" 

99 ) 

100 

101 # ----- stops ---------------------------------------------------------------------------- 

102 def after_continue(self, execution: Execution) -> Execution: 

103 """A run that ends a ``run continue`` failed in IMPLEMENTATION or VERIFICATION, with no 

104 correction left, has stopped: its changes are quarantined or the line is blocked.""" 

105 if self.mode == "off": 

106 return execution 

107 if execution.status in STOPPED_STATUSES and execution.current_phase in _STOPPED_PHASES: 

108 self.stop(execution, f"{execution.current_phase} {execution.status}") 

109 if execution.status is ResultStatus.PASSED: 

110 self.release(execution) 

111 return self.results.engine.get_execution(execution.execution_id) 

112 

113 def stop(self, execution: Execution, reason: str) -> dict[str, Any] | None: 

114 if self.mode == "off": 

115 return None 

116 if self.mode == "restore": 

117 return self.quarantine(execution, reason) 

118 record = self._record(execution, reason, restore=False) 

119 if record is not None: 119 ↛ 124line 119 didn't jump to line 124 because the condition on line 119 was always true

120 self.results.set_flag_json( 

121 self._flag(execution.project_id), 

122 {"executionId": execution.execution_id, "reason": reason}, 

123 ) 

124 return record 

125 

126 def quarantine(self, execution: Execution, reason: str) -> dict[str, Any] | None: 

127 """Keep the run's changes as a patch and restore the baseline (also the command 

128 ``harness run quarantine``).""" 

129 record = self._record(execution, reason, restore=True) 

130 stopped = self.results.flag_json(self._flag(execution.project_id)) 

131 unrestorable = bool(record and record["unrestorablePaths"]) 

132 if unrestorable: 132 ↛ 133line 132 didn't jump to line 133 because the condition on line 132 was never true

133 self.results.set_flag_json( 

134 self._flag(execution.project_id), 

135 {"executionId": execution.execution_id, "reason": reason + " (unrestorable)"}, 

136 ) 

137 elif isinstance(stopped, dict) and stopped.get("executionId") == execution.execution_id: 

138 self.results.s.state.set_flag(self._flag(execution.project_id), "") 

139 return record 

140 

141 def release(self, execution: Execution) -> None: 

142 stopped = self.results.flag_json(self._flag(execution.project_id)) 

143 if isinstance(stopped, dict) and stopped.get("executionId") == execution.execution_id: 143 ↛ 144line 143 didn't jump to line 144 because the condition on line 143 was never true

144 self.results.s.state.set_flag(self._flag(execution.project_id), "") 

145 

146 def _kept(self, execution: Execution) -> set[str]: 

147 """The frozen acceptance tests of a run that is still open stay in the workspace when 

148 its changes are restored (#81): a person approved them and the harness wrote them, and 

149 the next VERIFICATION of the run checks that they are unchanged. A closed, cancelled or 

150 rejected run keeps nothing.""" 

151 from governed_harness.orchestration.engine import run_is_open 

152 

153 latest = self.results.engine.get_execution(execution.execution_id) 

154 if not run_is_open(latest): 

155 return set() 

156 return self.results.acceptance.untouched(latest) 

157 

158 def _record(self, execution: Execution, reason: str, *, restore: bool) -> dict[str, Any] | None: 

159 results = self.results 

160 diff = results.baseline_changes(execution) 

161 contents = results.baseline_contents(execution) 

162 if diff is None or contents is None: 162 ↛ 163line 162 didn't jump to line 163 because the condition on line 162 was never true

163 return None 

164 workspace = results.s.paths.workspace 

165 if not diff.changes: 165 ↛ 166line 165 didn't jump to line 166 because the condition on line 165 was never true

166 return None 

167 patch = results.s.artifacts.put( 

168 diff.unified_diff, 

169 media_type="text/x-diff", 

170 metadata={"kind": "quarantined-patch", "executionId": execution.execution_id}, 

171 ) 

172 restored: list[str] = [] 

173 unrestorable: list[str] = [] 

174 kept = self._kept(execution) 

175 captured: dict[str, dict[str, Any]] = {} 

176 if restore: 

177 captured = self._capture(execution, diff, kept) 

178 restored, unrestorable = restore_changes( 

179 workspace, 

180 contents, 

181 WorkspaceDiff( 

182 changes=tuple(item for item in diff.changes if item.path not in kept), 

183 unified_diff=diff.unified_diff, 

184 digest=diff.digest, 

185 ), 

186 ) 

187 record: dict[str, Any] = { 

188 "executionId": execution.execution_id, 

189 "reason": reason, 

190 "mode": "restore" if restore else "block", 

191 "paths": [{"path": item.path, "status": item.status} for item in diff.changes], 

192 "patchRef": patch.uri, 

193 "restoredPaths": restored, 

194 "unrestorablePaths": unrestorable, 

195 "phase": execution.current_phase.value, 

196 } 

197 if restore: 

198 record["quarantinedContents"] = [ 

199 captured[path] for path in restored if path in captured 

200 ] 

201 if kept: 

202 record["keptPaths"] = sorted(kept) 

203 ref = results.record_json( 

204 execution, 

205 execution.current_phase, 

206 record, 

207 kind="workspace-quarantine", 

208 summary=( 

209 f"Stop the line ({reason}): {len(diff.changes)} changed path(s) " 

210 + (f"quarantined, {len(restored)} restored" if restore else "kept; line blocked") 

211 ), 

212 ) 

213 results.s.events.append( 

214 execution.execution_id, 

215 "workspace.quarantined" if restore else "workspace.line-stopped", 

216 {**record, "evidenceRef": ref}, 

217 ) 

218 return record 

219 

220 def _capture( 

221 self, execution: Execution, diff: WorkspaceDiff, kept: set[str] 

222 ) -> dict[str, dict[str, Any]]: 

223 """The content of every changed path about to be restored, stored as an artifact with 

224 the digest it had, so that ``run continue`` can put it back intact.""" 

225 workspace = self.results.s.paths.workspace 

226 captured: dict[str, dict[str, Any]] = {} 

227 for item in diff.changes: 

228 if item.path in kept: 

229 continue 

230 entry: dict[str, Any] = {"path": item.path, "digest": item.after_digest} 

231 if item.after_digest is not None: 231 ↛ 249line 231 didn't jump to line 249 because the condition on line 231 was always true

232 try: 

233 data: bytes | None = contained_path(workspace, Path(item.path)).read_bytes() 

234 except OSError: 

235 data = None 

236 if data is not None and sha256_bytes(data) == item.after_digest: 236 ↛ 249line 236 didn't jump to line 249 because the condition on line 236 was always true

237 # Stored as written (F4): run continue puts back exactly these bytes, which 

238 # a redaction would change; the workspace held them already. 

239 entry["contentRef"] = self.results.s.artifacts.put( 

240 data, 

241 metadata={ 

242 "kind": "quarantined-content", 

243 "executionId": execution.execution_id, 

244 "path": item.path, 

245 "unredacted": True, 

246 }, 

247 redact=False, 

248 ).uri 

249 captured[item.path] = entry 

250 return captured 

251 

252 # ----- run continue after a quarantine --------------------------------------------------- 

253 def pending(self, execution: Execution) -> dict[str, Any] | None: 

254 """The latest quarantine of the run that was not put back yet.""" 

255 pending: dict[str, Any] | None = None 

256 for event in self.results.s.events.list(execution.execution_id): 

257 if event.event_type == "workspace.quarantined": 

258 pending = dict(event.payload) 

259 elif event.event_type in { 259 ↛ 263line 259 didn't jump to line 263 because the condition on line 259 was never true

260 "workspace.quarantine.reapplied", 

261 "workspace.quarantine.superseded", 

262 }: 

263 pending = None 

264 return pending 

265 

266 def _rewritten(self, execution: Execution, record: dict[str, Any]) -> list[str]: 

267 """The restored paths that now hold neither their baseline nor their quarantined 

268 content: a new attempt was written over the restored baseline (a person or an 

269 embedded session correcting the run).""" 

270 workspace = self.results.s.paths.workspace 

271 baseline = self.results.baseline_snapshot(execution) 

272 if baseline is None: 272 ↛ 273line 272 didn't jump to line 273 because the condition on line 272 was never true

273 return [] 

274 digests = { 

275 str(item["path"]): item.get("digest") 

276 for item in record.get("quarantinedContents") or [] 

277 } 

278 rewritten: list[str] = [] 

279 for path in record["restoredPaths"]: 

280 target = contained_path(workspace, Path(path)) 

281 current = sha256_file(target) if target.is_file() else None 

282 state = baseline.files.get(path) 

283 if current != (state.digest if state else None) and ( 

284 path not in digests or current != digests[path] 

285 ): 

286 rewritten.append(path) 

287 return rewritten 

288 

289 def reapply(self, execution: Execution) -> Execution | None: 

290 """Put the quarantined changes of a run back into the workspace before ``run continue`` 

291 evaluates it again; the run BLOCKED when they cannot be put back intact, ``None`` 

292 otherwise. Nothing is left written unless every path is put back, and nothing is ever 

293 written over a new attempt: when a restored path holds new content the quarantine is 

294 superseded and the run is evaluated on the workspace as it is.""" 

295 execution = self.results.engine.get_execution(execution.execution_id) 

296 if execution.current_phase not in REAPPLY_PHASES: 

297 return None 

298 record = self.pending(execution) 

299 if record is None or not record.get("restoredPaths"): 

300 return None 

301 phase = record.get("phase") 

302 if phase is None or PhaseId(phase) not in REAPPLY_PHASES: 302 ↛ 303line 302 didn't jump to line 303 because the condition on line 302 was never true

303 return None 

304 events = self.results.s.events 

305 rewritten = self._rewritten(execution, record) 

306 if rewritten: 

307 events.append( 

308 execution.execution_id, 

309 "workspace.quarantine.superseded", 

310 {"evidenceRef": record.get("evidenceRef"), "rewrittenPaths": rewritten}, 

311 ) 

312 return None 

313 planned, problems = self._plan_reapply(execution, record) 

314 if not problems: 

315 problems = self._write(planned) 

316 if problems: 

317 events.append( 

318 execution.execution_id, 

319 "workspace.quarantine.reapply-refused", 

320 {"evidenceRef": record.get("evidenceRef"), "problems": problems}, 

321 ) 

322 blocked = execution.model_copy( 

323 update={ 

324 "status": ResultStatus.BLOCKED, 

325 "terminal_reason": ( 

326 "The quarantined changes of the run cannot be put back intact " 

327 "(governance.stopTheLine): " 

328 + "; ".join(problems[:10]) 

329 + ". Write the change again (the quarantined patch is on the run's " 

330 "chain) and run continue, or cancel the run." 

331 ), 

332 "updated_at": utc_now(), 

333 } 

334 ) 

335 self.results.engine._save_execution(blocked) 

336 return blocked 

337 events.append( 

338 execution.execution_id, 

339 "workspace.quarantine.reapplied", 

340 { 

341 "evidenceRef": record.get("evidenceRef"), 

342 "patchRef": record.get("patchRef"), 

343 "paths": [path for path, _target, _data in planned], 

344 }, 

345 ) 

346 return None 

347 

348 def _plan_reapply( 

349 self, execution: Execution, record: dict[str, Any] 

350 ) -> tuple[list[tuple[str, Path, bytes | None]], list[str]]: 

351 """What to write for each restored path (``None`` deletes it) and why it cannot be.""" 

352 workspace = self.results.s.paths.workspace 

353 baseline = self.results.baseline_snapshot(execution) 

354 entries = {str(item["path"]): item for item in record.get("quarantinedContents") or []} 

355 planned: list[tuple[str, Path, bytes | None]] = [] 

356 problems: list[str] = [] 

357 for path in record["restoredPaths"]: 

358 item, problem = self._plan_path(workspace, baseline, entries.get(path), path) 

359 if problem is not None: 

360 problems.append(problem) 

361 elif item is not None: 

362 planned.append(item) 

363 return planned, problems 

364 

365 def _plan_path( 

366 self, 

367 workspace: Path, 

368 baseline: WorkspaceSnapshot | None, 

369 entry: dict[str, Any] | None, 

370 path: str, 

371 ) -> tuple[tuple[str, Path, bytes | None] | None, str | None]: 

372 """What to write for one restored path (nothing when it already holds the quarantined 

373 content), or why it cannot be written.""" 

374 if entry is None or baseline is None: 

375 return None, f"{path}: no recorded content" 

376 target = contained_path(workspace, Path(path)) 

377 digest = entry.get("digest") 

378 current = sha256_file(target) if target.is_file() else None 

379 state = baseline.files.get(path) 

380 if current == digest: 

381 return None, None # already holds the quarantined content 

382 if current != (state.digest if state else None): 

383 return None, f"{path}: changed since the quarantine" 

384 if digest is None: 

385 return (path, target, None), None 

386 data = self._quarantined_content(entry) 

387 if data is None or sha256_bytes(data) != digest: 

388 return None, f"{path}: quarantined content missing or not matching its digest" 

389 return (path, target, data), None 

390 

391 def _quarantined_content(self, entry: dict[str, Any]) -> bytes | None: 

392 """The stored quarantined content of a path, ``None`` when it cannot be read.""" 

393 if not entry.get("contentRef"): 393 ↛ 394line 393 didn't jump to line 394 because the condition on line 393 was never true

394 return None 

395 try: 

396 return self.results.s.artifacts.get(str(entry["contentRef"])) 

397 except (OSError, ValueError): 

398 return None 

399 

400 def _write(self, planned: list[tuple[str, Path, bytes | None]]) -> list[str]: 

401 """Write the planned contents; on a failure undo what was written and report it.""" 

402 written: list[tuple[Path, bytes | None]] = [] 

403 try: 

404 for path, target, data in planned: 

405 written.append((target, target.read_bytes() if target.is_file() else None)) 

406 _write_checked(path, target, data) 

407 except OSError as error: 

408 _undo_writes(written) 

409 return [f"writing the quarantined content failed: {error}"] 

410 return [] 

411 

412 # ----- out-of-scope writes --------------------------------------------------------------- 

413 def owned_paths_output( 

414 self, execution: Execution, change_set: ChangeSet 

415 ) -> ValidatorOutput | None: 

416 if self.mode == "off": 

417 return None 

418 task = self.results.engine.run_task(execution) 

419 owned = task.metadata.get("ownedPaths") 

420 if not owned or task.implementation.mode == "patch": 

421 return None 

422 owned_set = {str(item) for item in owned} 

423 # The frozen acceptance tests (or BDD feature files) were written by the harness after 

424 # a person approved them, not by the agent: they are not outside the task's scope. 

425 acceptance = self.results.acceptance.state(execution) or {} 

426 owned_set |= set(acceptance.get("frozen") or {}) 

427 since = self.results.baseline_changes(execution) 

428 if since is None: 428 ↛ 429line 428 didn't jump to line 429 because the condition on line 428 was never true

429 return None 

430 outside = [ 

431 item.path 

432 for item in since.changes 

433 if item.path not in owned_set 

434 and not any(fnmatch.fnmatchcase(item.path, pattern) for pattern in IGNORED_PATTERNS) 

435 ] 

436 findings = tuple( 

437 self.results.record_finding( 

438 execution, 

439 validator_id=OWNED_PATHS_ID, 

440 rule_id="workspace.outside-owned-paths", 

441 category="workspace-integrity", 

442 severity=FindingSeverity.HIGH, 

443 message=f"{path} changed but is not one of the task's ownedPaths", 

444 path=path, 

445 recommendation=( 

446 "Keep the change inside the declared paths, or declare the path in the task." 

447 ), 

448 introduced=True, 

449 ) 

450 for path in outside 

451 ) 

452 result = self.results.record_validation( 

453 execution, 

454 validator_id=OWNED_PATHS_ID, 

455 digest=change_set.digest, 

456 status=ResultStatus.FAILED if findings else ResultStatus.PASSED, 

457 kind=ValidationKind.POLICY_VIOLATION if findings else ValidationKind.SUCCESS, 

458 mandatory=True, 

459 summary=( 

460 f"{len(findings)} changed path(s) outside the task's ownedPaths" 

461 if findings 

462 else "Every changed path is one of the task's ownedPaths" 

463 ), 

464 findings=findings, 

465 evidence_refs=(change_set.diff_ref,), 

466 ) 

467 return ValidatorOutput(result, ())