Files
canina/infra/hermes-editorial/tests/test_runner.py
T

126 lines
8.4 KiB
Python

"""Disposable HTTP/process protocol fixtures, NOT research or model QA evidence."""
import importlib.util
import json
import os
from pathlib import Path
import tempfile
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
ROOT = Path(__file__).resolve().parents[1]
TASK = '565ace1f-8cfc-4447-b978-66c36e7b767b'
class RunnerTests(unittest.TestCase):
def test_ready_claim_research_synthesis_result(self):
self.assertTrue((ROOT/'runner.py').exists(), 'actual runner missing')
spec = importlib.util.spec_from_file_location('runner', ROOT/'runner.py')
r = importlib.util.module_from_spec(spec); spec.loader.exec_module(r)
with tempfile.TemporaryDirectory() as d:
home = Path(d)
(home/'editorial-v4/credentials').mkdir(parents=True)
token = home/'editorial-v4/credentials/editorial-pmo-token'
token.write_text('a'*64); token.chmod(0o400)
for profile in r.PROFILES:
(home/'profiles'/profile).mkdir(parents=True)
fixture = home/'fixture.py'
fixture.write_text('''import json,os,sys
q=sys.stdin.read()
with open(os.environ['HERMES_HOME']+'/called','w') as f:f.write(q)
print(json.dumps({'type':'tool_use','name':'web_extract','input':{'urls':['https://www.avma.org/resources']}}))
print(json.dumps({'type':'tool_result','name':'web_extract','is_error':False,'output':json.dumps({'results':[{'url':'https://www.avma.org/resources','content':'test-only retrieval boundary'}]})}))
brief={'researchDate':'2026-01-01','inputAvailability':[{'input':'test','status':'PARTIAL','details':'fixture only'}],'findings':[{'sourceUrl':'https://www.avma.org/resources','sourceDate':'2026-01-01','finding':'fixture only'}],'candidates':[{'title':x,'score':60,'rationale':'fixture only','sourceUrls':['https://www.avma.org/resources']} for x in ['A','B','C']]}
print(json.dumps({'type':'result','exit_code':0,'text':json.dumps(brief)}))
''')
requests = []
mode = {'state':'READY','heartbeat':200}
class Handler(BaseHTTPRequestHandler):
def log_message(self,*a): pass
def do_GET(self): self.respond({'taskId':TASK,'state':mode['state'],'operationMode':'PREPARE','budgetLimitCents':20,'metadata':{'scope':'owner topic','acceptanceCriteria':['three candidates'],'environment':'staging','commitSha':'a'*40}})
def do_POST(self):
body=json.loads(self.rfile.read(int(self.headers['Content-Length'])))
requests.append((self.path,body,self.headers.get('Authorization')))
if self.path.endswith('editorial-claim'): self.respond({'lease':{'taskId':TASK,'runId':TASK,'leaseToken':TASK,'commitSha':'a'*40,'environment':'staging','maxCostCents':20}})
elif self.path.endswith('heartbeat'): self.respond({},mode['heartbeat'])
else: self.respond({'state':'VERIFYING','qaStatus':'PENDING_INDEPENDENT_QA','retainedReservationCents':20,'costStatus':'upper-bound-reserved'})
def respond(self,body,code=200):
self.send_response(code);self.end_headers();self.wfile.write(json.dumps(body).encode())
server=ThreadingHTTPServer(('127.0.0.1',0),Handler)
thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
try:
api=r.API(home, endpoint='http://127.0.0.1:'+str(server.server_port))
import sys
result=r.run(home,TASK,api=api,command=[sys.executable,str(fixture)],heartbeat_seconds=.1,process_seconds=2,total_seconds=10)
self.assertEqual(result['retainedReservationCents'],20)
self.assertEqual([p for p,_,_ in requests].count('/editorial-result'),1)
self.assertTrue(all(auth=='Bearer '+'a'*64 for _,_,auth in requests))
for profile in r.PROFILES: self.assertTrue((home/'profiles'/profile/'called').exists())
self.assertNotIn('costCents',requests[-1][1])
mode['heartbeat']=403; requests.clear()
with self.assertRaises(r.RunnerError):
r.run(home,TASK,api=api,command=[sys.executable,str(fixture)])
self.assertNotIn('/editorial-result',[p for p,_,_ in requests])
mode['heartbeat']=200
# Reject an actual heartbeat HTTP request while a child is running.
requests.clear()
def failing_pulse():
mode['heartbeat']=403
api.request('heartbeat',{'taskId':TASK,'runId':TASK,'leaseToken':TASK})
import time
with self.assertRaises(r.RunnerError):
r.execute([sys.executable,'-c','import time;time.sleep(60)'],'test',dict(os.environ),home,failing_pulse,time.monotonic()+5,heartbeat_seconds=.1)
self.assertEqual(requests[-1][0],'/heartbeat')
mode['state']='GATE1_PENDING'; requests.clear()
with self.assertRaises(r.RunnerError): r.run(home,TASK,api=api,command=[sys.executable,str(fixture)])
self.assertEqual(requests,[])
finally: server.shutdown();server.server_close();thread.join()
def test_brief_rejects_unknown_keys_and_invented_sources(self):
import runner as r
b={'researchDate':'2026-01-01','inputAvailability':[{'input':'analytics','status':'UNAVAILABLE','details':'not available'}],'findings':[{'sourceUrl':'https://www.avma.org/resources','sourceDate':'2026-01-01','finding':'protocol fixture'}],'candidates':[{'title':x,'score':60,'rationale':'protocol fixture','sourceUrls':['https://www.avma.org/resources']} for x in ['A','B','C']]}
self.assertEqual(r.validate_brief(b),b)
with self.assertRaisesRegex(r.RunnerError,'retrieved'):
r.validate_sources(b,[])
failed=[{'retrievalEvents':[{'type':'tool_use','name':'web_extract','input':{'urls':['https://www.avma.org/resources']}},{'type':'tool_result','name':'web_extract','output':json.dumps({'results':[{'url':'https://www.avma.org/resources','error':'403','content':''}]})}]}]
with self.assertRaisesRegex(r.RunnerError,'retrieved'): r.validate_sources(b,failed)
b['qaPassed']=True
with self.assertRaises(r.RunnerError): r.validate_brief(b)
def test_heartbeat_loss_kills_process_group(self):
import runner as r
import sys
import time
with tempfile.TemporaryDirectory() as d:
cwd=Path(d)
script=cwd/'hang.py'
script.write_text("import subprocess,time,os\np=subprocess.Popen(['sleep','60'])\nopen('child','w').write(str(p.pid))\ntime.sleep(60)\n")
def pulse():
if (cwd/'child').exists(): raise r.RunnerError('HTTP heartbeat rejected')
started=time.monotonic()
with self.assertRaisesRegex(r.RunnerError,'heartbeat rejected'):
r.execute([sys.executable,str(script)],'test',dict(os.environ),cwd,pulse,time.monotonic()+5,heartbeat_seconds=.2)
self.assertLess(time.monotonic()-started,2)
pid=int((cwd/'child').read_text())
stat_path=Path('/proc')/str(pid)/'stat'
# Container PID1 may reap asynchronously: a zombie cannot keep running.
self.assertTrue(not stat_path.exists() or stat_path.read_text().split()[2]=='Z')
def test_wall_clock_kills_hung_provider(self):
import runner as r
import sys
import time
with tempfile.TemporaryDirectory() as d:
with self.assertRaisesRegex(r.RunnerError,'wall clock'):
r.execute([sys.executable,'-c','import time;time.sleep(60)'],'test',dict(os.environ),Path(d),lambda:None,time.monotonic()+.3,heartbeat_seconds=.1)
def test_terminal_protocol_rejects_failed_or_contaminated_output(self):
import runner as r
import sys
import time
with tempfile.TemporaryDirectory() as d:
for events in [ [{'type':'result','exit_code':1,'text':'looks fine'}], [{'type':'result','exit_code':0,'text':'{}'},{'type':'result','exit_code':0,'text':'{}'}], [{'type':'tool_use','name':'terminal'},{'type':'result','exit_code':0,'text':'{}'}] ]:
with self.subTest(events=events),self.assertRaises(r.RunnerError):
r.execute([sys.executable,'-c','print('+repr('\n'.join(json.dumps(x) for x in events))+')'],'test',dict(os.environ),Path(d),lambda:None,time.monotonic()+3)
if __name__=='__main__': unittest.main()