"""Trusted subprocess boundary; no claim, lease extension or completion transport.""" import json import os import pathlib import signal import subprocess import tempfile import threading import time import uuid HEARTBEAT_INTERVAL = 5.0 HEARTBEAT_TIMEOUT = 5.0 MAX_WALL_SECONDS = 90.0 class LeaseGuard: def __init__(self, lease, heartbeat): self.lease, self.heartbeat = lease, heartbeat self.stop = threading.Event() self.ready = threading.Event() self.error = None self.started = time.monotonic() self.last = self.started self.thread = threading.Thread(target=self._run, daemon=True) def _run(self): while not self.stop.is_set(): try: if self.heartbeat(dict(self.lease)) is False: raise PermissionError("Authenticated heartbeat denied") self.last = time.monotonic() self.ready.set() except BaseException as exc: self.error = exc self.ready.set() return if self.stop.wait(HEARTBEAT_INTERVAL): return def check(self): if self.error is not None: raise self.error now = time.monotonic() if now - self.started >= MAX_WALL_SECONDS - 5: raise TimeoutError('Pilot wall deadline (cancellation reserve included)') if now - self.last >= HEARTBEAT_INTERVAL + HEARTBEAT_TIMEOUT: raise TimeoutError('Authenticated heartbeat stalled') def __enter__(self): self.thread.start() if not self.ready.wait(HEARTBEAT_TIMEOUT): self.stop.set() raise TimeoutError('Initial authenticated heartbeat stalled') try: self.check() except BaseException: self.stop.set() raise return self def __exit__(self, *args): self.stop.set() self.thread.join(timeout=.05) class ProcessInference: """Trusted argv only. cancel_remote MUST kill the in-container process group.""" def __init__(self, argv, input_text=None, cancel_remote=None): self.argv, self.input_text, self.cancel_remote = argv, input_text, cancel_remote self.used = False self.lock = threading.Lock() def run(self, guard): with self.lock: if self.used: raise RuntimeError('Exactly one inference allowed; no retry') self.used = True guard.check() # Files avoid pipe deadlocks and unbounded captured output in memory. with tempfile.TemporaryFile(mode='w+') as incoming, tempfile.TemporaryFile(mode='w+') as out, tempfile.TemporaryFile(mode='w+') as err: incoming.write(self.input_text or '') incoming.seek(0) child = subprocess.Popen(self.argv, stdin=incoming, stdout=out, stderr=err, start_new_session=True) try: while child.poll() is None: guard.check() time.sleep(.02) guard.check() if child.returncode: raise RuntimeError('Inference failed (exit %d)' % child.returncode) out.seek(0) result = out.read(4097) if len(result) > 4096: raise ValueError('Oversized inference response') return result finally: # Always cancel remote, even when docker exec itself failed. try: if self.cancel_remote: self.cancel_remote() finally: try: os.killpg(child.pid, signal.SIGKILL) except ProcessLookupError: pass child.wait(timeout=1) def pinned_inference(image): actual = subprocess.check_output(['docker','inspect','--format','{{.Image}}','canina-hermes'], text=True, timeout=2).strip() if actual != image: raise RuntimeError('Pinned installed image mismatch') token = uuid.uuid4().hex pidfile = '/tmp/canina-pilot-' + token + '.pid' script = pathlib.Path(__file__).with_name('infer.py').read_text() # Session leader is created INSIDE the container. Inner kill is independent of client. wrapper = ('import os,signal,subprocess,sys; ' 'p=subprocess.Popen(["timeout","--signal=KILL","85","/opt/hermes/.venv/bin/python","-"],stdin=sys.stdin,start_new_session=True); ' 'open(%r,"w").write(str(p.pid)); ' 'os.path.exists(%r) and os.killpg(p.pid,signal.SIGKILL); sys.exit(p.wait())') % (pidfile, pidfile+'.cancel') cancel = ('import os,signal; p=%r; ' '\nopen(p+".cancel","w").close()\ntry:\n pid=int(open(p).read()); os.killpg(pid,signal.SIGKILL)\n' 'except (FileNotFoundError,ProcessLookupError): pass\n' 'finally:\n' ' try: os.unlink(p)\n except FileNotFoundError: pass\n') % pidfile def cancel_remote(): subprocess.run(['docker','exec','canina-hermes','/opt/hermes/.venv/bin/python','-c',cancel], check=True, timeout=2, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) return ProcessInference(['docker','exec','-i','canina-hermes','/opt/hermes/.venv/bin/python','-c',wrapper], script, cancel_remote)