129 lines
5.3 KiB
Python
129 lines
5.3 KiB
Python
"""Trusted subprocess boundary; no claim, lease extension or completion transport."""
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import signal
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
HEARTBEAT_INTERVAL = 5.0
|
|
HEARTBEAT_TIMEOUT = 5.0
|
|
MAX_WALL_SECONDS = 90.0
|
|
|
|
class LeaseGuard:
|
|
def __init__(self, lease, heartbeat):
|
|
self.lease, self.heartbeat = lease, heartbeat
|
|
self.stop = threading.Event()
|
|
self.ready = threading.Event()
|
|
self.error = None
|
|
self.started = time.monotonic()
|
|
self.last = self.started
|
|
self.thread = threading.Thread(target=self._run, daemon=True)
|
|
|
|
def _run(self):
|
|
while not self.stop.is_set():
|
|
try:
|
|
if self.heartbeat(dict(self.lease)) is False:
|
|
raise PermissionError("Authenticated heartbeat denied")
|
|
self.last = time.monotonic()
|
|
self.ready.set()
|
|
except BaseException as exc:
|
|
self.error = exc
|
|
self.ready.set()
|
|
return
|
|
if self.stop.wait(HEARTBEAT_INTERVAL):
|
|
return
|
|
|
|
def check(self):
|
|
if self.error is not None:
|
|
raise self.error
|
|
now = time.monotonic()
|
|
if now - self.started >= MAX_WALL_SECONDS - 5:
|
|
raise TimeoutError('Pilot wall deadline (cancellation reserve included)')
|
|
if now - self.last >= HEARTBEAT_INTERVAL + HEARTBEAT_TIMEOUT:
|
|
raise TimeoutError('Authenticated heartbeat stalled')
|
|
|
|
def __enter__(self):
|
|
self.thread.start()
|
|
if not self.ready.wait(HEARTBEAT_TIMEOUT):
|
|
self.stop.set()
|
|
raise TimeoutError('Initial authenticated heartbeat stalled')
|
|
try:
|
|
self.check()
|
|
except BaseException:
|
|
self.stop.set()
|
|
raise
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
self.stop.set()
|
|
self.thread.join(timeout=.05)
|
|
|
|
class ProcessInference:
|
|
"""Trusted argv only. cancel_remote MUST kill the in-container process group."""
|
|
def __init__(self, argv, input_text=None, cancel_remote=None):
|
|
self.argv, self.input_text, self.cancel_remote = argv, input_text, cancel_remote
|
|
self.used = False
|
|
self.lock = threading.Lock()
|
|
|
|
def run(self, guard):
|
|
with self.lock:
|
|
if self.used:
|
|
raise RuntimeError('Exactly one inference allowed; no retry')
|
|
self.used = True
|
|
guard.check()
|
|
# Files avoid pipe deadlocks and unbounded captured output in memory.
|
|
with tempfile.TemporaryFile(mode='w+') as incoming, tempfile.TemporaryFile(mode='w+') as out, tempfile.TemporaryFile(mode='w+') as err:
|
|
incoming.write(self.input_text or '')
|
|
incoming.seek(0)
|
|
child = subprocess.Popen(self.argv, stdin=incoming, stdout=out, stderr=err, start_new_session=True)
|
|
try:
|
|
while child.poll() is None:
|
|
guard.check()
|
|
time.sleep(.02)
|
|
guard.check()
|
|
if child.returncode:
|
|
raise RuntimeError('Inference failed (exit %d)' % child.returncode)
|
|
out.seek(0)
|
|
result = out.read(4097)
|
|
if len(result) > 4096:
|
|
raise ValueError('Oversized inference response')
|
|
return result
|
|
finally:
|
|
# Always cancel remote, even when docker exec itself failed.
|
|
try:
|
|
if self.cancel_remote:
|
|
self.cancel_remote()
|
|
finally:
|
|
try:
|
|
os.killpg(child.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
child.wait(timeout=1)
|
|
|
|
|
|
def pinned_inference(image):
|
|
actual = subprocess.check_output(['docker','inspect','--format','{{.Image}}','canina-hermes'], text=True, timeout=2).strip()
|
|
if actual != image:
|
|
raise RuntimeError('Pinned installed image mismatch')
|
|
token = uuid.uuid4().hex
|
|
pidfile = '/tmp/canina-pilot-' + token + '.pid'
|
|
script = (pathlib.Path(__file__).with_name('inference_budget.py').read_text() + '\n' +
|
|
pathlib.Path(__file__).with_name('infer.py').read_text())
|
|
# Session leader is created INSIDE the container. Inner kill is independent of client.
|
|
wrapper = ('import os,signal,subprocess,sys; '
|
|
'p=subprocess.Popen(["timeout","--signal=KILL","85","/opt/hermes/.venv/bin/python","-"],stdin=sys.stdin,start_new_session=True); '
|
|
'open(%r,"w").write(str(p.pid)); '
|
|
'os.path.exists(%r) and os.killpg(p.pid,signal.SIGKILL); sys.exit(p.wait())') % (pidfile, pidfile+'.cancel')
|
|
cancel = ('import os,signal; p=%r; '
|
|
'\nopen(p+".cancel","w").close()\ntry:\n pid=int(open(p).read()); os.killpg(pid,signal.SIGKILL)\n'
|
|
'except (FileNotFoundError,ProcessLookupError): pass\n'
|
|
'finally:\n'
|
|
' try: os.unlink(p)\n except FileNotFoundError: pass\n') % pidfile
|
|
def cancel_remote():
|
|
subprocess.run(['docker','exec','canina-hermes','/opt/hermes/.venv/bin/python','-c',cancel], check=True, timeout=2, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
return ProcessInference(['docker','exec','-i','canina-hermes','/opt/hermes/.venv/bin/python','-c',wrapper], script, cancel_remote)
|