Files
canina/infra/hermes-pilot/runtime.py
T
2026-10-23 19:36:56 +00:00

129 lines
5.3 KiB
Python

"""Trusted subprocess boundary; no claim, lease extension or completion transport."""
import json
import os
import pathlib
import signal
import subprocess
import tempfile
import threading
import time
import uuid
HEARTBEAT_INTERVAL = 5.0
HEARTBEAT_TIMEOUT = 5.0
MAX_WALL_SECONDS = 90.0
class LeaseGuard:
def __init__(self, lease, heartbeat):
self.lease, self.heartbeat = lease, heartbeat
self.stop = threading.Event()
self.ready = threading.Event()
self.error = None
self.started = time.monotonic()
self.last = self.started
self.thread = threading.Thread(target=self._run, daemon=True)
def _run(self):
while not self.stop.is_set():
try:
if self.heartbeat(dict(self.lease)) is False:
raise PermissionError("Authenticated heartbeat denied")
self.last = time.monotonic()
self.ready.set()
except BaseException as exc:
self.error = exc
self.ready.set()
return
if self.stop.wait(HEARTBEAT_INTERVAL):
return
def check(self):
if self.error is not None:
raise self.error
now = time.monotonic()
if now - self.started >= MAX_WALL_SECONDS - 5:
raise TimeoutError('Pilot wall deadline (cancellation reserve included)')
if now - self.last >= HEARTBEAT_INTERVAL + HEARTBEAT_TIMEOUT:
raise TimeoutError('Authenticated heartbeat stalled')
def __enter__(self):
self.thread.start()
if not self.ready.wait(HEARTBEAT_TIMEOUT):
self.stop.set()
raise TimeoutError('Initial authenticated heartbeat stalled')
try:
self.check()
except BaseException:
self.stop.set()
raise
return self
def __exit__(self, *args):
self.stop.set()
self.thread.join(timeout=.05)
class ProcessInference:
"""Trusted argv only. cancel_remote MUST kill the in-container process group."""
def __init__(self, argv, input_text=None, cancel_remote=None):
self.argv, self.input_text, self.cancel_remote = argv, input_text, cancel_remote
self.used = False
self.lock = threading.Lock()
def run(self, guard):
with self.lock:
if self.used:
raise RuntimeError('Exactly one inference allowed; no retry')
self.used = True
guard.check()
# Files avoid pipe deadlocks and unbounded captured output in memory.
with tempfile.TemporaryFile(mode='w+') as incoming, tempfile.TemporaryFile(mode='w+') as out, tempfile.TemporaryFile(mode='w+') as err:
incoming.write(self.input_text or '')
incoming.seek(0)
child = subprocess.Popen(self.argv, stdin=incoming, stdout=out, stderr=err, start_new_session=True)
try:
while child.poll() is None:
guard.check()
time.sleep(.02)
guard.check()
if child.returncode:
raise RuntimeError('Inference failed (exit %d)' % child.returncode)
out.seek(0)
result = out.read(4097)
if len(result) > 4096:
raise ValueError('Oversized inference response')
return result
finally:
# Always cancel remote, even when docker exec itself failed.
try:
if self.cancel_remote:
self.cancel_remote()
finally:
try:
os.killpg(child.pid, signal.SIGKILL)
except ProcessLookupError:
pass
child.wait(timeout=1)
def pinned_inference(image):
actual = subprocess.check_output(['docker','inspect','--format','{{.Image}}','canina-hermes'], text=True, timeout=2).strip()
if actual != image:
raise RuntimeError('Pinned installed image mismatch')
token = uuid.uuid4().hex
pidfile = '/tmp/canina-pilot-' + token + '.pid'
script = (pathlib.Path(__file__).with_name('inference_budget.py').read_text() + '\n' +
pathlib.Path(__file__).with_name('infer.py').read_text())
# Session leader is created INSIDE the container. Inner kill is independent of client.
wrapper = ('import os,signal,subprocess,sys; '
'p=subprocess.Popen(["timeout","--signal=KILL","85","/opt/hermes/.venv/bin/python","-"],stdin=sys.stdin,start_new_session=True); '
'open(%r,"w").write(str(p.pid)); '
'os.path.exists(%r) and os.killpg(p.pid,signal.SIGKILL); sys.exit(p.wait())') % (pidfile, pidfile+'.cancel')
cancel = ('import os,signal; p=%r; '
'\nopen(p+".cancel","w").close()\ntry:\n pid=int(open(p).read()); os.killpg(pid,signal.SIGKILL)\n'
'except (FileNotFoundError,ProcessLookupError): pass\n'
'finally:\n'
' try: os.unlink(p)\n except FileNotFoundError: pass\n') % pidfile
def cancel_remote():
subprocess.run(['docker','exec','canina-hermes','/opt/hermes/.venv/bin/python','-c',cancel], check=True, timeout=2, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
return ProcessInference(['docker','exec','-i','canina-hermes','/opt/hermes/.venv/bin/python','-c',wrapper], script, cancel_remote)