Files
canina/infra/hermes-editorial/tests/test_runner.py
T

180 lines
11 KiB
Python

"""Disposable HTTP/process protocol fixtures, NOT research or model QA evidence."""
import importlib.util
import json
import os
import sys
from pathlib import Path
import tempfile
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
TASK = '565ace1f-8cfc-4447-b978-66c36e7b767b'
class RunnerTests(unittest.TestCase):
def test_ready_claim_research_synthesis_result(self):
self.assertTrue((ROOT/'runner.py').exists(), 'actual runner missing')
spec = importlib.util.spec_from_file_location('runner', ROOT/'runner.py')
r = importlib.util.module_from_spec(spec); spec.loader.exec_module(r)
with tempfile.TemporaryDirectory() as d:
home = Path(d)
(home/'editorial-v4/credentials').mkdir(parents=True)
token = home/'editorial-v4/credentials/editorial-pmo-token'
token.write_text('a'*64); token.chmod(0o400)
for profile in r.PROFILES:
(home/'profiles'/profile).mkdir(parents=True)
fixture = home/'fixture.py'
fixture.write_text('''import json,os,sys
q=sys.stdin.read()
with open(os.environ['HERMES_HOME']+'/called','w') as f:f.write(q)
print(json.dumps({'type':'tool_use','name':'web_extract','input':{'urls':['https://www.avma.org/resources']}}))
print(json.dumps({'type':'tool_result','name':'web_extract','is_error':False,'output':json.dumps({'results':[{'url':'https://www.avma.org/resources','content':'test-only retrieval boundary'}]})}))
brief={'researchDate':'2026-01-01','inputAvailability':[{'input':'test','status':'PARTIAL','details':'fixture only'}],'findings':[{'sourceUrl':'https://www.avma.org/resources','sourceDate':'2026-01-01','finding':'fixture only'}],'candidates':[{'title':x,'score':60,'rationale':'fixture only','sourceUrls':['https://www.avma.org/resources']} for x in ['A','B','C']]}
print(json.dumps({'type':'result','exit_code':0,'text':json.dumps(brief)}))
''')
requests = []
mode = {'state':'READY','heartbeat':200}
class Handler(BaseHTTPRequestHandler):
def log_message(self,*a): pass
def do_GET(self): self.respond({'taskId':TASK,'state':mode['state'],'operationMode':'PREPARE','budgetLimitCents':20,'metadata':{'scope':'owner topic','acceptanceCriteria':['three candidates'],'environment':'staging','commitSha':'a'*40}})
def do_POST(self):
body=json.loads(self.rfile.read(int(self.headers['Content-Length'])))
requests.append((self.path,body,self.headers.get('Authorization')))
if self.path.endswith('editorial-claim'): self.respond({'lease':{'taskId':TASK,'runId':TASK,'leaseToken':TASK,'commitSha':'a'*40,'environment':'staging','maxCostCents':20,'attempt':1,'leaseUntil':'2026-10-24T00:00:30.000Z','pilotKind':'editorial-research-to-brief-v1'}})
elif self.path.endswith('heartbeat'):
self.assert_lease(body)
self.respond({'leaseUntil':'2026-10-24T00:00:30.000Z'},mode['heartbeat'])
else: self.respond({'state':'VERIFYING','qaStatus':'PENDING_INDEPENDENT_QA','retainedReservationCents':20,'costStatus':'upper-bound-reserved'})
def assert_lease(self,body):
if set(body) != {'taskId','runId','leaseToken'}:
raise AssertionError('Deployed LeaseDto forbids extra keys')
def respond(self,body,code=200):
self.send_response(code);self.end_headers();self.wfile.write(json.dumps(body).encode())
server=ThreadingHTTPServer(('127.0.0.1',0),Handler)
thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
try:
api=r.API(home, endpoint='http://127.0.0.1:'+str(server.server_port))
import sys
result=r.run(home,TASK,api=api,command=[sys.executable,str(fixture)],heartbeat_seconds=.1,process_seconds=2,total_seconds=10)
self.assertEqual(result['retainedReservationCents'],20)
self.assertEqual([p for p,_,_ in requests].count('/editorial-result'),1)
self.assertTrue(all(auth=='Bearer '+'a'*64 for _,_,auth in requests))
for profile in r.PROFILES: self.assertTrue((home/'profiles'/profile/'called').exists())
self.assertNotIn('costCents',requests[-1][1])
mode['heartbeat']=403; requests.clear()
with self.assertRaises(r.RunnerError):
r.run(home,TASK,api=api,command=[sys.executable,str(fixture)])
self.assertNotIn('/editorial-result',[p for p,_,_ in requests])
mode['heartbeat']=200
# Reject an actual heartbeat HTTP request while a child is running.
requests.clear()
def failing_pulse():
mode['heartbeat']=403
api.request('heartbeat',{'taskId':TASK,'runId':TASK,'leaseToken':TASK})
import time
with self.assertRaises(r.RunnerError):
r.execute([sys.executable,'-c','import time;time.sleep(60)'],'test',dict(os.environ),home,failing_pulse,time.monotonic()+5,heartbeat_seconds=.1)
self.assertEqual(requests[-1][0],'/heartbeat')
mode['state']='GATE1_PENDING'; requests.clear()
with self.assertRaises(r.RunnerError): r.run(home,TASK,api=api,command=[sys.executable,str(fixture)])
self.assertEqual(requests,[])
finally: server.shutdown();server.server_close();thread.join()
@unittest.skipUnless(os.getuid()==0, 'requires disposable UID-drop seam')
def test_actual_uid_drop_cannot_enter_root_private_cache(self):
# Minimal real process repro of installed shim/cache ownership boundary;
# never calls Hermes, a provider, a live endpoint or the installed runner.
import subprocess
with tempfile.TemporaryDirectory() as d:
cache=Path(d)/'cache';cache.mkdir(mode=0o700)
os.chmod(d,0o755)
child=subprocess.run([sys.executable,'-c',
'import os,sys; os.setgid(10000); os.setuid(10000); os.chdir(sys.argv[1])',str(cache)],
capture_output=True)
self.assertNotEqual(child.returncode,0)
self.assertIn(b'PermissionError',child.stderr)
def test_profile_identity_mismatch_rejected_before_claim(self):
# Actual installed failure: docker exec UID0 creates cache0700, shim
# drops child to UID10000. Ownership probe requires no paid CLI call.
import runner as r
from unittest.mock import patch
with tempfile.TemporaryDirectory() as d:
home=Path(d)
for profile in r.PROFILES:
(home/'profiles'/profile).mkdir(parents=True)
class NeverClaim:
def request(self, *args):
raise AssertionError('Identity preflight must precede HTTP/claim')
with patch.object(r.os, 'getuid', return_value=os.getuid()+1):
with self.assertRaises(r.RunnerError) as caught:
r.run(home,TASK,api=NeverClaim(),command=[sys.executable])
self.assertEqual(caught.exception.stage,'preflight')
self.assertEqual(caught.exception.code,'PROFILE_IDENTITY_MISMATCH')
def test_safe_stage_diagnostic_does_not_reflect_exception(self):
import runner as r
with tempfile.TemporaryDirectory() as d:
home=Path(d)
for profile in r.PROFILES:
(home/'profiles'/profile).mkdir(parents=True)
class BrokenHTTP:
def request(self,*args):
raise OSError('secret-provider-token-and-body')
with self.assertRaises(r.RunnerError) as caught:
r.run(home,TASK,api=BrokenHTTP(),command=[sys.executable])
self.assertEqual(caught.exception.stage,'status')
self.assertEqual(caught.exception.code,'OS_ERROR')
self.assertNotIn('secret',str(caught.exception))
def test_brief_rejects_unknown_keys_and_invented_sources(self):
import runner as r
b={'researchDate':'2026-01-01','inputAvailability':[{'input':'analytics','status':'UNAVAILABLE','details':'not available'}],'findings':[{'sourceUrl':'https://www.avma.org/resources','sourceDate':'2026-01-01','finding':'protocol fixture'}],'candidates':[{'title':x,'score':60,'rationale':'protocol fixture','sourceUrls':['https://www.avma.org/resources']} for x in ['A','B','C']]}
self.assertEqual(r.validate_brief(b),b)
with self.assertRaisesRegex(r.RunnerError,'retrieved'):
r.validate_sources(b,[])
failed=[{'retrievalEvents':[{'type':'tool_use','name':'web_extract','input':{'urls':['https://www.avma.org/resources']}},{'type':'tool_result','name':'web_extract','output':json.dumps({'results':[{'url':'https://www.avma.org/resources','error':'403','content':''}]})}]}]
with self.assertRaisesRegex(r.RunnerError,'retrieved'): r.validate_sources(b,failed)
b['qaPassed']=True
with self.assertRaises(r.RunnerError): r.validate_brief(b)
def test_heartbeat_loss_kills_process_group(self):
import runner as r
import sys
import time
with tempfile.TemporaryDirectory() as d:
cwd=Path(d)
script=cwd/'hang.py'
script.write_text("import subprocess,time,os\np=subprocess.Popen(['sleep','60'])\nopen('child','w').write(str(p.pid))\ntime.sleep(60)\n")
def pulse():
if (cwd/'child').exists(): raise r.RunnerError('HTTP heartbeat rejected')
started=time.monotonic()
with self.assertRaisesRegex(r.RunnerError,'heartbeat rejected'):
r.execute([sys.executable,str(script)],'test',dict(os.environ),cwd,pulse,time.monotonic()+5,heartbeat_seconds=.2)
self.assertLess(time.monotonic()-started,2)
pid=int((cwd/'child').read_text())
stat_path=Path('/proc')/str(pid)/'stat'
# Container PID1 may reap asynchronously: a zombie cannot keep running.
self.assertTrue(not stat_path.exists() or stat_path.read_text().split()[2]=='Z')
def test_wall_clock_kills_hung_provider(self):
import runner as r
import sys
import time
with tempfile.TemporaryDirectory() as d:
with self.assertRaisesRegex(r.RunnerError,'wall clock'):
r.execute([sys.executable,'-c','import time;time.sleep(60)'],'test',dict(os.environ),Path(d),lambda:None,time.monotonic()+.3,heartbeat_seconds=.1)
def test_terminal_protocol_rejects_failed_or_contaminated_output(self):
import runner as r
import sys
import time
with tempfile.TemporaryDirectory() as d:
for events in [ [{'type':'result','exit_code':1,'text':'looks fine'}], [{'type':'result','exit_code':0,'text':'{}'},{'type':'result','exit_code':0,'text':'{}'}], [{'type':'tool_use','name':'terminal'},{'type':'result','exit_code':0,'text':'{}'}] ]:
with self.subTest(events=events),self.assertRaises(r.RunnerError):
r.execute([sys.executable,'-c','print('+repr('\n'.join(json.dumps(x) for x in events))+')'],'test',dict(os.environ),Path(d),lambda:None,time.monotonic()+3)
if __name__=='__main__': unittest.main()