Coverage for src/governed_harness/runtime/lease.py: 85%

160 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 16:06 +0000

1"""One harness process at a time per workspace (``governance.workspaceLease``). 

2 

3Two ``run start`` on one workspace interfered (each one's agent overwrote the other's files and 

4both runs failed), and a harness killed during IMPLEMENTATION left a phase ``RUNNING`` that 

5``run continue`` implemented again on top. The lease is a file, ``.harness/lease.json``, created 

6exclusively by the process that executes phases and refreshed by a heartbeat while it runs. A 

7second process finds it and stops with a clear error, unless the holder is gone: its process no 

8longer exists on this host, or (on another host, or where liveness cannot be checked) its 

9heartbeat is older than ``STALE_SECONDS``. 

10 

11The module also keeps the process groups the governed runner starts, so that a harness that is 

12interrupted terminates them and a later recovery can terminate the ones a killed harness left. 

13""" 

14 

15from __future__ import annotations 

16 

17import contextlib 

18import json 

19import os 

20import signal 

21import socket 

22import sys 

23import threading 

24import time 

25import uuid 

26from collections.abc import Iterator 

27from dataclasses import dataclass 

28from datetime import UTC, datetime 

29from pathlib import Path 

30from types import FrameType 

31from typing import Any 

32 

33from governed_harness.domain.errors import PolicyViolationError 

34 

35LEASE_FILE = "lease.json" 

36HEARTBEAT_SECONDS = 10.0 

37STALE_SECONDS = 60.0 

38 

39 

40class WorkspaceBusyError(PolicyViolationError): 

41 """Another harness process holds the workspace lease; exit code 5.""" 

42 

43 

44@dataclass(frozen=True) 

45class LeaseHolder: 

46 token: str 

47 pid: int 

48 host: str 

49 command: str 

50 run_id: str | None 

51 acquired_at: str 

52 heartbeat_at: float 

53 

54 @classmethod 

55 def from_dict(cls, value: dict[str, Any]) -> LeaseHolder: 

56 return cls( 

57 token=str(value.get("token", "")), 

58 pid=int(value.get("pid", 0)), 

59 host=str(value.get("host", "")), 

60 command=str(value.get("command", "")), 

61 run_id=value.get("runId"), 

62 acquired_at=str(value.get("acquiredAt", "")), 

63 heartbeat_at=float(value.get("heartbeatAt", 0.0)), 

64 ) 

65 

66 def as_dict(self) -> dict[str, Any]: 

67 return { 

68 "token": self.token, 

69 "pid": self.pid, 

70 "host": self.host, 

71 "command": self.command, 

72 "runId": self.run_id, 

73 "acquiredAt": self.acquired_at, 

74 "heartbeatAt": self.heartbeat_at, 

75 } 

76 

77 def describe(self) -> str: 

78 run = f"run {self.run_id}" if self.run_id else "a harness command" 

79 return f"{run} ({self.command}, pid {self.pid} on {self.host}, since {self.acquired_at})" 

80 

81 

82def process_alive(pid: int) -> bool | None: 

83 """Whether a process exists on this host; ``None`` where it cannot be checked safely.""" 

84 if sys.platform == "win32" or pid <= 0: 84 ↛ 85line 84 didn't jump to line 85 because the condition on line 84 was never true

85 return None 

86 try: 

87 os.kill(pid, 0) 

88 except ProcessLookupError: 

89 return False 

90 except PermissionError: 

91 return True 

92 return True 

93 

94 

95class WorkspaceLease: 

96 def __init__(self, harness_dir: Path, command: str) -> None: 

97 self.path = harness_dir / LEASE_FILE 

98 self.command = command 

99 self.token = uuid.uuid4().hex 

100 self.host = socket.gethostname() 

101 self.run_id: str | None = None 

102 self.recovered: LeaseHolder | None = None 

103 """The stale holder this lease replaced, if any.""" 

104 self._stop = threading.Event() 

105 self._thread: threading.Thread | None = None 

106 self._lock = threading.Lock() 

107 

108 # ----- acquisition -------------------------------------------------------------- 

109 def acquire(self) -> WorkspaceLease: 

110 for _ in range(3): 110 ↛ 126line 110 didn't jump to line 126 because the loop on line 110 didn't complete

111 if self._create(): 

112 self._start_heartbeat() 

113 return self 

114 holder = self.holder() 

115 if holder is None: 115 ↛ 116line 115 didn't jump to line 116 because the condition on line 115 was never true

116 continue # removed meanwhile, or unreadable: try again 

117 if not self.is_stale(holder): 

118 raise WorkspaceBusyError( 

119 f"the workspace is in use by {holder.describe()}; wait for it to finish or " 

120 f"cancel it (harness run cancel). A holder that no longer runs is taken over " 

121 f"after its heartbeat is {int(STALE_SECONDS)} s old" 

122 ) 

123 self.recovered = holder 

124 with contextlib.suppress(FileNotFoundError): 

125 self.path.unlink() 

126 raise WorkspaceBusyError(f"could not acquire the workspace lease {self.path}") 

127 

128 def holder(self) -> LeaseHolder | None: 

129 try: 

130 return LeaseHolder.from_dict(json.loads(self.path.read_text(encoding="utf-8"))) 

131 except FileNotFoundError: 

132 return None 

133 except (OSError, ValueError, TypeError): 

134 # A lease that cannot be read is treated as held by nobody alive: its heartbeat 

135 # cannot be refreshed either. 

136 return LeaseHolder("", 0, "", "unreadable lease", None, "", 0.0) 

137 

138 def is_stale(self, holder: LeaseHolder) -> bool: 

139 age = time.time() - holder.heartbeat_at 

140 if holder.host == self.host and holder.pid != os.getpid(): 

141 alive = process_alive(holder.pid) 

142 if alive is not None: 142 ↛ 144line 142 didn't jump to line 144 because the condition on line 142 was always true

143 return not alive 

144 return age > STALE_SECONDS 

145 

146 def bind(self, run_id: str) -> None: 

147 """Name the run the lease protects (known after ``run start`` created it).""" 

148 self.run_id = run_id 

149 self._write() 

150 

151 def release(self) -> None: 

152 self._stop.set() 

153 if self._thread is not None: 153 ↛ 155line 153 didn't jump to line 155 because the condition on line 153 was always true

154 self._thread.join(timeout=HEARTBEAT_SECONDS) 

155 with self._lock: 

156 holder = self.holder() 

157 if holder is not None and holder.token == self.token: 157 ↛ exitline 157 didn't jump to the function exit

158 with contextlib.suppress(FileNotFoundError): 

159 self.path.unlink() 

160 

161 # ----- internals ---------------------------------------------------------------- 

162 def _record(self) -> LeaseHolder: 

163 return LeaseHolder( 

164 token=self.token, 

165 pid=os.getpid(), 

166 host=self.host, 

167 command=self.command, 

168 run_id=self.run_id, 

169 acquired_at=datetime.now(UTC).isoformat(), 

170 heartbeat_at=time.time(), 

171 ) 

172 

173 def _create(self) -> bool: 

174 self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) 

175 try: 

176 descriptor = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) 

177 except FileExistsError: 

178 return False 

179 with os.fdopen(descriptor, "w", encoding="utf-8") as handle: 

180 json.dump(self._record().as_dict(), handle) 

181 return True 

182 

183 def _write(self) -> None: 

184 with self._lock: 

185 current = self.holder() 

186 if current is None or current.token != self.token: 186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true

187 return # taken over: never overwrite another holder 

188 record = self._record() 

189 record = LeaseHolder( 

190 record.token, 

191 record.pid, 

192 record.host, 

193 record.command, 

194 record.run_id, 

195 current.acquired_at, 

196 record.heartbeat_at, 

197 ) 

198 temporary = self.path.with_name(f"{LEASE_FILE}.{self.token}") 

199 temporary.write_text(json.dumps(record.as_dict()), encoding="utf-8") 

200 os.replace(temporary, self.path) 

201 

202 def _start_heartbeat(self) -> None: 

203 def beat() -> None: 

204 while not self._stop.wait(HEARTBEAT_SECONDS): 204 ↛ 205line 204 didn't jump to line 205 because the condition on line 204 was never true

205 with contextlib.suppress(OSError): 

206 self._write() 

207 

208 self._thread = threading.Thread(target=beat, name="harness-lease", daemon=True) 

209 self._thread.start() 

210 

211 

212@contextlib.contextmanager 

213def interruptible() -> Iterator[None]: 

214 """Turn ``SIGTERM`` into ``SystemExit`` while a run executes (main thread only), so the 

215 governed runner terminates the process groups it started and the interrupted phase is 

216 recorded, instead of the harness dying with its agent still running.""" 

217 if threading.current_thread() is not threading.main_thread(): 

218 yield 

219 return 

220 

221 def handler(signum: int, frame: FrameType | None) -> None: 

222 raise SystemExit(128 + signum) 

223 

224 previous = signal.signal(signal.SIGTERM, handler) 

225 try: 

226 yield 

227 finally: 

228 signal.signal(signal.SIGTERM, previous) 

229 

230 

231def terminate_process_group(pgid: int, *, grace_seconds: float = 1.0) -> bool: 

232 """Terminate a process group a killed harness left behind (``SIGTERM``, then ``SIGKILL``). 

233 Returns whether a group existed. POSIX only.""" 

234 if sys.platform == "win32" or pgid <= 1 or pgid == os.getpgid(0): 234 ↛ 235line 234 didn't jump to line 235 because the condition on line 234 was never true

235 return False 

236 try: 

237 os.killpg(pgid, signal.SIGTERM) 

238 except ProcessLookupError: 

239 return False 

240 except PermissionError: 

241 return False 

242 deadline = time.monotonic() + grace_seconds 

243 while time.monotonic() < deadline: 243 ↛ 253line 243 didn't jump to line 253 because the condition on line 243 was always true

244 try: 

245 os.killpg(pgid, 0) 

246 except ProcessLookupError: 

247 return True 

248 except PermissionError: 

249 # macOS answers EPERM, not ESRCH, while the group holds only zombies not reaped yet: 

250 # the group existed and got the signal, so the SIGKILL below settles it. 

251 break 

252 time.sleep(0.05) 

253 with contextlib.suppress(ProcessLookupError, PermissionError): 

254 os.killpg(pgid, signal.SIGKILL) 

255 return True