From 8cfd44cd091fa8d71aa36f5d5e589ef12f3d17c6 Mon Sep 17 00:00:00 2001 From: devops Date: Fri, 23 Oct 2026 16:57:33 +0000 Subject: [PATCH] Bound pilot inference with ongoing heartbeat and process-group cancellation --- infra/hermes-pilot/README.md | 53 +++++++++- infra/hermes-pilot/executor.py | 23 ++++- infra/hermes-pilot/infer.py | 2 +- infra/hermes-pilot/rehearse.py | 11 +- infra/hermes-pilot/runtime.py | 127 ++++++++++++++++++++++++ infra/hermes-pilot/test_cancellation.py | 68 +++++++++++++ infra/hermes-pilot/test_heartbeat.py | 24 +++++ 7 files changed, 294 insertions(+), 14 deletions(-) create mode 100644 infra/hermes-pilot/runtime.py create mode 100644 infra/hermes-pilot/test_cancellation.py create mode 100644 infra/hermes-pilot/test_heartbeat.py diff --git a/infra/hermes-pilot/README.md b/infra/hermes-pilot/README.md index 425cdd64..0e38cba9 100644 --- a/infra/hermes-pilot/README.md +++ b/infra/hermes-pilot/README.md @@ -14,9 +14,44 @@ Installed image identity: `sha256:373ddd5e6649f525b17993e1c885cdf1352127f8e98fd3 `executor.execute(lease, artifact_root, infer, heartbeat)` only accepts a fresh staging, first-attempt, 20-cent, 40hex-source lease and exact lease fields; heartbeat failure aborts before inference and before QA/commit. Trusted adapter must obtain this lease via existing authenticated claim AFTER independent Gate1/READY, verify the explicitly allowlisted task identity/scope/criteria server-side, and supply heartbeat connected to the authenticated backend. A caller-created dict is NOT an authorization. Never expose this Python interface to browser requests. -Budget: one model inference, max_iterations1/max_tokens160, model run budget15s, process kill18s, QA3s; outer rehearsal kill28s. The18s kill is **inside** `docker exec` so killing its client cannot leave inference running. No retries. 20-cent core reservation MUST remain reserved for unknown billing; there is no measured-dollar guarantee from this provider, so live admission must remain disabled until provider-side pricing/cost cap is independently verified. Do not reconcile to0 or claim full end-to-end success. No Stage evidence/Gate2/release calls exist. +## Heartbeat runner and parent API (this revision) -`rehearse.py` is explicitly disposable: synthetic lease IDs/heartbeat local deadline, never calls core, never claims real Gate1. Run from repository root: `timeout 28s python3 infra/hermes-pilot/rehearse.py`. This is NOT a supported live adapter. Parent must preserve lease fencing and heartbeat across any longer runner and implement trusted artifact ingestion/new report verification rather than bypass tracer validation. +After independent READY/Gate1, allowlist and provider-cap admission, parent constructs +`runner = executor.pinned_inference(executor.IMAGE)` and calls +`executor.execute(authenticated_lease, artifact_root, runner, authenticated_heartbeat)`. +Only trusted Python may construct runners. Callback receives an unchanged COPY of the +original lease binding (taskId/runId/leaseToken), must use authenticated server transport, +and must raise on any non-2xx, ownership/fencing/expiry denial or malformed response. +Explicit `False` also denies; `None` means success for existing raise-on-error adapters. +Callback MUST impose <=5s network timeout, no retries; a blocked callback thread cannot +be forcibly killed by Python. It is daemonized; its result cannot restart inference or +extend a local lease. Parent must serialize executions per server lease; the runner is +single-use and concurrent reuse of that runner is locked out. This is not a browser API. + +Immediate initial heartbeat; dedicated ongoing thread waits 5s between acknowledged +calls. With required <=5s transport ceiling, heartbeat starts are <=10s apart. A stalled +heartbeat (10s without acknowledgment), exception or denial aborts actual subprocess +inference, not just its caller. No leaseUntil mutation/local extension occurs. Fresh +server-authorized heartbeats retain fencing; this code cannot create authorization. +Guard checks also fence QA/artifact commit/report return. A denial during QA prevents +report return; any partial local fixture is not accepted completion evidence. + +Monotonic wall ceiling 90s with cancellation reserve: work aborts at85s, remote kill +command <=2s and local reap <=1s; bounded QA3s/Git1s. Pinned runner uses one model +call, max_iterations1/max_tokens160, run_budget80s, tools0, no retries. An independent +in-container timeout85s survives loss of the Docker client. Cancellation addresses +the actual in-container process session via a random PID file plus cancellation +marker (handles cancellation before PID publication); it kills that process group. +Cancellation transport failure raises and prevents success, but cannot prove immediate +remote death; independent inner timeout is the fallback. Small random marker files +in container /tmp are intentional fencing tombstones, not configuration changes. + +`rehearse.py` remains synthetic, never a live lease. It uses this cancellable runner +but is NOT admission evidence. Do not run it until provider-cap approval. Existing +20-cent reservation remains UNKNOWN, never zero, and no completion/claim/release or +backend transport is implemented here. Existing tracer endpoint still rejects model +reports; parent must separately implement authenticated allowlisted artifact ingestion. +No changes to installed Hermes or runtime/profile configuration. ## Actual evidence and blockers @@ -27,3 +62,17 @@ Evidence directory `/opt/services/canina/validation/v4-real-pilot`: - `actual.log`: bounded full rehearsal failed with inner timeout exit137 at18s; no completion/QA success was assumed. Previous client-only timeout was corrected to inner process kill. **Installed model/startup cannot currently complete reliably inside30s PREPARE lease without authenticated ongoing heartbeat.** Do not extend lease locally or call the75s diagnostic a bounded lease success. Contract tests: `python3 infra/hermes-pilot/test_contract.py`. Parent still owns backend integration, authenticated heartbeat orchestration, allowlist admission, billing caps, signed artifact transport and Stage CI. This commit is a real generation/independent-QA proof and fail-closed executor seam, not a live autonomous Tech department or released product. + +## Verification of heartbeat revision + +`python3 -m unittest discover -s infra/hermes-pilot -v`: 11 passing tests, including +real OS child/descendant cancellation, ongoing rejection, stalled heartbeat, wall +ceiling, concurrent runner reuse, immutable lease and unknown-cost preservation. +The artifact success test is explicitly a deterministic process fixture, NOT model +inference. TDD red observed for missing ongoing runner and explicit false denial; +additional cancellation/concurrency regression cases were added and executed. +Previously generated real model artifact was rechecked by current AST allowlist QA: +9 cases pass, SHA256 `0125246b551896c5f9ee98fabeb02982a6690a9974c0eb24ef3ae676d46380d5`. +No new real inference was run: README provider pricing/cap gate remains unresolved. +Docker cancellation path is implemented but not runtime-exercised here; OS process +group cancellation is exercised. No live backend/claim/release, push or deploy. diff --git a/infra/hermes-pilot/executor.py b/infra/hermes-pilot/executor.py index 3e59dca5..aabb30c8 100644 --- a/infra/hermes-pilot/executor.py +++ b/infra/hermes-pilot/executor.py @@ -14,7 +14,7 @@ def validate_lease(lease): if not 5 < expiry-time.time() <= 31: raise ValueError('Fresh 30-second lease required') return lease -def execute(lease, artifact_root, infer, heartbeat): +def _execute(lease, artifact_root, infer, heartbeat): """No claim/approval generation. heartbeat must call trusted server; errors abort. infer is the pinned runner, not a browser-supplied command or prompt. Completion is deliberately NOT sent: existing transport accepts tracer JSON only. @@ -23,12 +23,12 @@ def execute(lease, artifact_root, infer, heartbeat): heartbeat(lease) root=pathlib.Path(tempfile.mkdtemp(prefix='real-pilot-',dir=artifact_root)) def git(*args): - return subprocess.check_output(['git','-C',str(root),*args],stderr=subprocess.DEVNULL,text=True).strip() + return subprocess.check_output(['git','-C',str(root),*args],stderr=subprocess.DEVNULL,text=True,timeout=1).strip() git('init','-q'); git('config','user.email','pilot@invalid'); git('config','user.name','Bounded Hermes Pilot') (root/'README').write_text('Disposable fixture; PREPARE source '+lease['commitSha']+'\n') git('add','README'); git('commit','-qm','Seed isolated pilot fixture') base=git('rev-parse','HEAD') - # One inference, no tools, 18s hard outer cap in supplied pinned runner. + # One cancellable inference, no tools; ongoing authenticated guard. source=infer() heartbeat(lease) if not isinstance(source,str) or len(source.encode())>2048: raise ValueError('Invalid generated artifact') @@ -45,3 +45,20 @@ def execute(lease, artifact_root, infer, heartbeat): 'gate1':'EXTERNAL_LEASE_REQUIRED','published':False,'deploymentExecuted':False} (root/'evidence.json').write_text(json.dumps(report,indent=2)+'\n') return root,report + +from runtime import LeaseGuard, ProcessInference, pinned_inference + +def execute(lease, artifact_root, infer, heartbeat): + validate_lease(lease) + with LeaseGuard(lease, heartbeat) as guard: + guard.check() + if not isinstance(infer, ProcessInference): + raise TypeError("Cancellable ProcessInference required") + def generate(): + response = json.loads(infer.run(guard)) + if not isinstance(response,dict) or response.get("tools") != 0 or response.get("model") != "Free": + raise ValueError("Pinned tool-less model required") + return response["source"] + result = _execute(lease, artifact_root, generate, lambda _: guard.check()) + guard.check() + return result diff --git a/infra/hermes-pilot/infer.py b/infra/hermes-pilot/infer.py index ae1aa833..0d823c6e 100644 --- a/infra/hermes-pilot/infer.py +++ b/infra/hermes-pilot/infer.py @@ -11,7 +11,7 @@ os.environ.update({'PATH':'/usr/bin:/bin','HOME':home.name,'HERMES_HOME':home.na with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): from run_agent import AIAgent agent=AIAgent(model='Free',provider='custom',base_url='https://nr.parsaaghayi.ir/v1',api_key=key, - enabled_toolsets=[],max_iterations=1,max_tokens=160,run_budget_seconds=15, + enabled_toolsets=[],max_iterations=1,max_tokens=160,run_budget_seconds=80, skip_context_files=True,skip_memory=True,skip_background_review=True,load_soul_identity=False, quiet_mode=True,fallback_model={},cwd=home.name) if agent.tools: raise RuntimeError('Tool-less invariant failed') diff --git a/infra/hermes-pilot/rehearse.py b/infra/hermes-pilot/rehearse.py index 90c6ac29..3cfacc6e 100644 --- a/infra/hermes-pilot/rehearse.py +++ b/infra/hermes-pilot/rehearse.py @@ -1,17 +1,12 @@ """Disposable model/artifact exercise, NOT a live Gate1 approval or service completion.""" import datetime, json, pathlib, subprocess, time, uuid -from executor import execute, IMAGE +from executor import execute, IMAGE, pinned_inference def main(): image=subprocess.check_output(['docker','inspect','--format','{{.Image}}','canina-hermes'],text=True).strip() if image!=IMAGE: raise RuntimeError('Pinned installed image mismatch') - script=pathlib.Path(__file__).with_name('infer.py').read_text() - # stdin is fixed tracked code, not arbitrary request text; stdout contains only generated source. - def infer(): - r=subprocess.run(['docker','exec','-i','canina-hermes','timeout','--signal=KILL','18','/opt/hermes/.venv/bin/python','-'],input=script,text=True,capture_output=True,timeout=21) - if r.returncode: raise RuntimeError('Pinned Hermes inference failed; diagnostic kept private in runner (exit '+str(r.returncode)+')') - return json.loads(r.stdout)['source'] - deadline=time.monotonic()+25 + infer=pinned_inference(IMAGE) + deadline=time.monotonic()+90 lease={k:str(uuid.uuid4()) for k in ('taskId','runId','leaseToken')} lease.update(leaseUntil=(datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(seconds=30)).isoformat(),commitSha=subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),environment='staging',maxCostCents=20,attempt=1) def fixture_heartbeat(binding): diff --git a/infra/hermes-pilot/runtime.py b/infra/hermes-pilot/runtime.py new file mode 100644 index 00000000..ab47376e --- /dev/null +++ b/infra/hermes-pilot/runtime.py @@ -0,0 +1,127 @@ +"""Trusted subprocess boundary; no claim, lease extension or completion transport.""" +import json +import os +import pathlib +import signal +import subprocess +import tempfile +import threading +import time +import uuid + +HEARTBEAT_INTERVAL = 5.0 +HEARTBEAT_TIMEOUT = 5.0 +MAX_WALL_SECONDS = 90.0 + +class LeaseGuard: + def __init__(self, lease, heartbeat): + self.lease, self.heartbeat = lease, heartbeat + self.stop = threading.Event() + self.ready = threading.Event() + self.error = None + self.started = time.monotonic() + self.last = self.started + self.thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + while not self.stop.is_set(): + try: + if self.heartbeat(dict(self.lease)) is False: + raise PermissionError("Authenticated heartbeat denied") + self.last = time.monotonic() + self.ready.set() + except BaseException as exc: + self.error = exc + self.ready.set() + return + if self.stop.wait(HEARTBEAT_INTERVAL): + return + + def check(self): + if self.error is not None: + raise self.error + now = time.monotonic() + if now - self.started >= MAX_WALL_SECONDS - 5: + raise TimeoutError('Pilot wall deadline (cancellation reserve included)') + if now - self.last >= HEARTBEAT_INTERVAL + HEARTBEAT_TIMEOUT: + raise TimeoutError('Authenticated heartbeat stalled') + + def __enter__(self): + self.thread.start() + if not self.ready.wait(HEARTBEAT_TIMEOUT): + self.stop.set() + raise TimeoutError('Initial authenticated heartbeat stalled') + try: + self.check() + except BaseException: + self.stop.set() + raise + return self + + def __exit__(self, *args): + self.stop.set() + self.thread.join(timeout=.05) + +class ProcessInference: + """Trusted argv only. cancel_remote MUST kill the in-container process group.""" + def __init__(self, argv, input_text=None, cancel_remote=None): + self.argv, self.input_text, self.cancel_remote = argv, input_text, cancel_remote + self.used = False + self.lock = threading.Lock() + + def run(self, guard): + with self.lock: + if self.used: + raise RuntimeError('Exactly one inference allowed; no retry') + self.used = True + guard.check() + # Files avoid pipe deadlocks and unbounded captured output in memory. + with tempfile.TemporaryFile(mode='w+') as incoming, tempfile.TemporaryFile(mode='w+') as out, tempfile.TemporaryFile(mode='w+') as err: + incoming.write(self.input_text or '') + incoming.seek(0) + child = subprocess.Popen(self.argv, stdin=incoming, stdout=out, stderr=err, start_new_session=True) + try: + while child.poll() is None: + guard.check() + time.sleep(.02) + guard.check() + if child.returncode: + raise RuntimeError('Inference failed (exit %d)' % child.returncode) + out.seek(0) + result = out.read(4097) + if len(result) > 4096: + raise ValueError('Oversized inference response') + return result + finally: + # Always cancel remote, even when docker exec itself failed. + try: + if self.cancel_remote: + self.cancel_remote() + finally: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.wait(timeout=1) + + +def pinned_inference(image): + actual = subprocess.check_output(['docker','inspect','--format','{{.Image}}','canina-hermes'], text=True, timeout=2).strip() + if actual != image: + raise RuntimeError('Pinned installed image mismatch') + token = uuid.uuid4().hex + pidfile = '/tmp/canina-pilot-' + token + '.pid' + script = pathlib.Path(__file__).with_name('infer.py').read_text() + # Session leader is created INSIDE the container. Inner kill is independent of client. + wrapper = ('import os,signal,subprocess,sys; ' + 'p=subprocess.Popen(["timeout","--signal=KILL","85","/opt/hermes/.venv/bin/python","-"],stdin=sys.stdin,start_new_session=True); ' + 'open(%r,"w").write(str(p.pid)); ' + 'os.path.exists(%r) and os.killpg(p.pid,signal.SIGKILL); sys.exit(p.wait())') % (pidfile, pidfile+'.cancel') + cancel = ('import os,signal; p=%r; ' + '\nopen(p+".cancel","w").close()\ntry:\n pid=int(open(p).read()); os.killpg(pid,signal.SIGKILL)\n' + 'except (FileNotFoundError,ProcessLookupError): pass\n' + 'finally:\n' + ' try: os.unlink(p)\n except FileNotFoundError: pass\n') % pidfile + def cancel_remote(): + subprocess.run(['docker','exec','canina-hermes','/opt/hermes/.venv/bin/python','-c',cancel], check=True, timeout=2, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return ProcessInference(['docker','exec','-i','canina-hermes','/opt/hermes/.venv/bin/python','-c',wrapper], script, cancel_remote) diff --git a/infra/hermes-pilot/test_cancellation.py b/infra/hermes-pilot/test_cancellation.py new file mode 100644 index 00000000..7c267297 --- /dev/null +++ b/infra/hermes-pilot/test_cancellation.py @@ -0,0 +1,68 @@ +import json, pathlib, sys, tempfile, threading, time, unittest +from unittest.mock import patch +sys.path.insert(0,str(pathlib.Path(__file__).parent)) +import executor as m +import runtime +from test_contract import lease + +class Cancellation(unittest.TestCase): + def test_wall_deadline_kills_descendant(self): + with tempfile.TemporaryDirectory() as root: + marker=pathlib.Path(root,'late') + child=f'import pathlib,time; time.sleep(.7); pathlib.Path({str(marker)!r}).touch()' + code=f'import subprocess,sys,time; subprocess.Popen([sys.executable,"-c",{child!r}]); time.sleep(5)' + with patch.object(runtime,'MAX_WALL_SECONDS',5.2): + with self.assertRaises(TimeoutError): m.execute(lease(),root,m.ProcessInference([sys.executable,'-c',code]),lambda _: None) + time.sleep(.8) + self.assertFalse(marker.exists()) + + def test_stalled_heartbeat_cancels_child(self): + with tempfile.TemporaryDirectory() as root: + marker=pathlib.Path(root,'late'); release=threading.Event(); calls=[] + def heartbeat(_): + calls.append(time.monotonic()) + if len(calls)>1: release.wait(3) + code=f'import time,pathlib; time.sleep(.7); pathlib.Path({str(marker)!r}).touch()' + try: + with patch.object(runtime,'HEARTBEAT_INTERVAL',.05),patch.object(runtime,'HEARTBEAT_TIMEOUT',.1): + with self.assertRaises(TimeoutError): m.execute(lease(),root,m.ProcessInference([sys.executable,'-c',code]),heartbeat) + finally: release.set() + time.sleep(.8); self.assertFalse(marker.exists()) + + def test_fixture_artifact_qa_and_cost_reservation(self): + # Explicit deterministic PROCESS fixture, never reported as real model inference. + source='def clamp_quantity(quantity):\n return min(99, max(0, quantity))\n' + response=json.dumps(dict(source=source,model='Free',tools=0)) + beats=[]; original=lease(); snapshot=dict(original) + def heartbeat(binding): + beats.append(time.monotonic()); binding['leaseUntil']='untrusted mutation' + with tempfile.TemporaryDirectory() as root: + infer=m.ProcessInference([sys.executable,'-c',f'import time; time.sleep(.25); print({response!r})']) + with patch.object(runtime,'HEARTBEAT_INTERVAL',.05): + _,report=m.execute(original,root,infer,heartbeat) + self.assertEqual(original,snapshot) + self.assertGreater(len(beats),2) + self.assertIsNone(report['costCents']) + self.assertEqual(report['costStatus'],'UNKNOWN_RETAIN_FULL_20_CENT_RESERVATION') + with self.assertRaises(RuntimeError): + with runtime.LeaseGuard(lease(),lambda _:None) as guard: infer.run(guard) + + def test_concurrent_reuse_rejected(self): + infer=m.ProcessInference([sys.executable,"-c","import time; time.sleep(.3); print(1)"]) + errors=[] + with runtime.LeaseGuard(lease(),lambda _:None) as guard: + t=threading.Thread(target=lambda:infer.run(guard)); t.start() + time.sleep(.05) + with self.assertRaises(RuntimeError): infer.run(guard) + t.join() + + def test_explicit_false_heartbeat_denied(self): + with tempfile.TemporaryDirectory() as root: + with self.assertRaises(PermissionError): + m.execute(lease(),root,m.ProcessInference([sys.executable,"-c","print(1)"]),lambda _:False) + + def test_callable_not_cancellable_rejected(self): + with tempfile.TemporaryDirectory() as root: + with self.assertRaises(TypeError): m.execute(lease(),root,lambda:'',lambda _:None) + +if __name__=='__main__': unittest.main() diff --git a/infra/hermes-pilot/test_heartbeat.py b/infra/hermes-pilot/test_heartbeat.py new file mode 100644 index 00000000..6a9aac12 --- /dev/null +++ b/infra/hermes-pilot/test_heartbeat.py @@ -0,0 +1,24 @@ +import datetime, pathlib, sys, tempfile, threading, time, unittest +from unittest.mock import patch +sys.path.insert(0, str(pathlib.Path(__file__).parent)) +import executor as m +import runtime +from test_contract import lease + +class Heartbeats(unittest.TestCase): + def test_ongoing_denial_kills_real_child(self): + with tempfile.TemporaryDirectory() as root: + marker=pathlib.Path(root,'late') + code=f'import time,pathlib; time.sleep(1); pathlib.Path({str(marker)!r}).touch()' + beats=[] + def beat(binding): + beats.append(time.monotonic()) + if len(beats)>1: raise PermissionError('revoked') + with patch.object(runtime,'HEARTBEAT_INTERVAL',.1): + with self.assertRaises(PermissionError): + m.execute(lease(),root,m.ProcessInference([sys.executable,'-c',code]),beat) + time.sleep(1.1) + self.assertFalse(marker.exists()) + self.assertGreaterEqual(len(beats),2) + +if __name__=='__main__': unittest.main()