Coverage for src/governed_harness/runtime/lease.py: 85%
160 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 16:06 +0000
1"""One harness process at a time per workspace (``governance.workspaceLease``).
3Two ``run start`` on one workspace interfered (each one's agent overwrote the other's files and
4both runs failed), and a harness killed during IMPLEMENTATION left a phase ``RUNNING`` that
5``run continue`` implemented again on top. The lease is a file, ``.harness/lease.json``, created
6exclusively by the process that executes phases and refreshed by a heartbeat while it runs. A
7second process finds it and stops with a clear error, unless the holder is gone: its process no
8longer exists on this host, or (on another host, or where liveness cannot be checked) its
9heartbeat is older than ``STALE_SECONDS``.
11The module also keeps the process groups the governed runner starts, so that a harness that is
12interrupted terminates them and a later recovery can terminate the ones a killed harness left.
13"""
15from __future__ import annotations
17import contextlib
18import json
19import os
20import signal
21import socket
22import sys
23import threading
24import time
25import uuid
26from collections.abc import Iterator
27from dataclasses import dataclass
28from datetime import UTC, datetime
29from pathlib import Path
30from types import FrameType
31from typing import Any
33from governed_harness.domain.errors import PolicyViolationError
35LEASE_FILE = "lease.json"
36HEARTBEAT_SECONDS = 10.0
37STALE_SECONDS = 60.0
40class WorkspaceBusyError(PolicyViolationError):
41 """Another harness process holds the workspace lease; exit code 5."""
44@dataclass(frozen=True)
45class LeaseHolder:
46 token: str
47 pid: int
48 host: str
49 command: str
50 run_id: str | None
51 acquired_at: str
52 heartbeat_at: float
54 @classmethod
55 def from_dict(cls, value: dict[str, Any]) -> LeaseHolder:
56 return cls(
57 token=str(value.get("token", "")),
58 pid=int(value.get("pid", 0)),
59 host=str(value.get("host", "")),
60 command=str(value.get("command", "")),
61 run_id=value.get("runId"),
62 acquired_at=str(value.get("acquiredAt", "")),
63 heartbeat_at=float(value.get("heartbeatAt", 0.0)),
64 )
66 def as_dict(self) -> dict[str, Any]:
67 return {
68 "token": self.token,
69 "pid": self.pid,
70 "host": self.host,
71 "command": self.command,
72 "runId": self.run_id,
73 "acquiredAt": self.acquired_at,
74 "heartbeatAt": self.heartbeat_at,
75 }
77 def describe(self) -> str:
78 run = f"run {self.run_id}" if self.run_id else "a harness command"
79 return f"{run} ({self.command}, pid {self.pid} on {self.host}, since {self.acquired_at})"
82def process_alive(pid: int) -> bool | None:
83 """Whether a process exists on this host; ``None`` where it cannot be checked safely."""
84 if sys.platform == "win32" or pid <= 0: 84 ↛ 85line 84 didn't jump to line 85 because the condition on line 84 was never true
85 return None
86 try:
87 os.kill(pid, 0)
88 except ProcessLookupError:
89 return False
90 except PermissionError:
91 return True
92 return True
95class WorkspaceLease:
96 def __init__(self, harness_dir: Path, command: str) -> None:
97 self.path = harness_dir / LEASE_FILE
98 self.command = command
99 self.token = uuid.uuid4().hex
100 self.host = socket.gethostname()
101 self.run_id: str | None = None
102 self.recovered: LeaseHolder | None = None
103 """The stale holder this lease replaced, if any."""
104 self._stop = threading.Event()
105 self._thread: threading.Thread | None = None
106 self._lock = threading.Lock()
108 # ----- acquisition --------------------------------------------------------------
109 def acquire(self) -> WorkspaceLease:
110 for _ in range(3): 110 ↛ 126line 110 didn't jump to line 126 because the loop on line 110 didn't complete
111 if self._create():
112 self._start_heartbeat()
113 return self
114 holder = self.holder()
115 if holder is None: 115 ↛ 116line 115 didn't jump to line 116 because the condition on line 115 was never true
116 continue # removed meanwhile, or unreadable: try again
117 if not self.is_stale(holder):
118 raise WorkspaceBusyError(
119 f"the workspace is in use by {holder.describe()}; wait for it to finish or "
120 f"cancel it (harness run cancel). A holder that no longer runs is taken over "
121 f"after its heartbeat is {int(STALE_SECONDS)} s old"
122 )
123 self.recovered = holder
124 with contextlib.suppress(FileNotFoundError):
125 self.path.unlink()
126 raise WorkspaceBusyError(f"could not acquire the workspace lease {self.path}")
128 def holder(self) -> LeaseHolder | None:
129 try:
130 return LeaseHolder.from_dict(json.loads(self.path.read_text(encoding="utf-8")))
131 except FileNotFoundError:
132 return None
133 except (OSError, ValueError, TypeError):
134 # A lease that cannot be read is treated as held by nobody alive: its heartbeat
135 # cannot be refreshed either.
136 return LeaseHolder("", 0, "", "unreadable lease", None, "", 0.0)
138 def is_stale(self, holder: LeaseHolder) -> bool:
139 age = time.time() - holder.heartbeat_at
140 if holder.host == self.host and holder.pid != os.getpid():
141 alive = process_alive(holder.pid)
142 if alive is not None: 142 ↛ 144line 142 didn't jump to line 144 because the condition on line 142 was always true
143 return not alive
144 return age > STALE_SECONDS
146 def bind(self, run_id: str) -> None:
147 """Name the run the lease protects (known after ``run start`` created it)."""
148 self.run_id = run_id
149 self._write()
151 def release(self) -> None:
152 self._stop.set()
153 if self._thread is not None: 153 ↛ 155line 153 didn't jump to line 155 because the condition on line 153 was always true
154 self._thread.join(timeout=HEARTBEAT_SECONDS)
155 with self._lock:
156 holder = self.holder()
157 if holder is not None and holder.token == self.token: 157 ↛ exitline 157 didn't jump to the function exit
158 with contextlib.suppress(FileNotFoundError):
159 self.path.unlink()
161 # ----- internals ----------------------------------------------------------------
162 def _record(self) -> LeaseHolder:
163 return LeaseHolder(
164 token=self.token,
165 pid=os.getpid(),
166 host=self.host,
167 command=self.command,
168 run_id=self.run_id,
169 acquired_at=datetime.now(UTC).isoformat(),
170 heartbeat_at=time.time(),
171 )
173 def _create(self) -> bool:
174 self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
175 try:
176 descriptor = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
177 except FileExistsError:
178 return False
179 with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
180 json.dump(self._record().as_dict(), handle)
181 return True
183 def _write(self) -> None:
184 with self._lock:
185 current = self.holder()
186 if current is None or current.token != self.token: 186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true
187 return # taken over: never overwrite another holder
188 record = self._record()
189 record = LeaseHolder(
190 record.token,
191 record.pid,
192 record.host,
193 record.command,
194 record.run_id,
195 current.acquired_at,
196 record.heartbeat_at,
197 )
198 temporary = self.path.with_name(f"{LEASE_FILE}.{self.token}")
199 temporary.write_text(json.dumps(record.as_dict()), encoding="utf-8")
200 os.replace(temporary, self.path)
202 def _start_heartbeat(self) -> None:
203 def beat() -> None:
204 while not self._stop.wait(HEARTBEAT_SECONDS): 204 ↛ 205line 204 didn't jump to line 205 because the condition on line 204 was never true
205 with contextlib.suppress(OSError):
206 self._write()
208 self._thread = threading.Thread(target=beat, name="harness-lease", daemon=True)
209 self._thread.start()
212@contextlib.contextmanager
213def interruptible() -> Iterator[None]:
214 """Turn ``SIGTERM`` into ``SystemExit`` while a run executes (main thread only), so the
215 governed runner terminates the process groups it started and the interrupted phase is
216 recorded, instead of the harness dying with its agent still running."""
217 if threading.current_thread() is not threading.main_thread():
218 yield
219 return
221 def handler(signum: int, frame: FrameType | None) -> None:
222 raise SystemExit(128 + signum)
224 previous = signal.signal(signal.SIGTERM, handler)
225 try:
226 yield
227 finally:
228 signal.signal(signal.SIGTERM, previous)
231def terminate_process_group(pgid: int, *, grace_seconds: float = 1.0) -> bool:
232 """Terminate a process group a killed harness left behind (``SIGTERM``, then ``SIGKILL``).
233 Returns whether a group existed. POSIX only."""
234 if sys.platform == "win32" or pgid <= 1 or pgid == os.getpgid(0): 234 ↛ 235line 234 didn't jump to line 235 because the condition on line 234 was never true
235 return False
236 try:
237 os.killpg(pgid, signal.SIGTERM)
238 except ProcessLookupError:
239 return False
240 except PermissionError:
241 return False
242 deadline = time.monotonic() + grace_seconds
243 while time.monotonic() < deadline: 243 ↛ 253line 243 didn't jump to line 253 because the condition on line 243 was always true
244 try:
245 os.killpg(pgid, 0)
246 except ProcessLookupError:
247 return True
248 except PermissionError:
249 # macOS answers EPERM, not ESRCH, while the group holds only zombies not reaped yet:
250 # the group existed and got the signal, so the SIGKILL below settles it.
251 break
252 time.sleep(0.05)
253 with contextlib.suppress(ProcessLookupError, PermissionError):
254 os.killpg(pgid, signal.SIGKILL)
255 return True