180 lines
11 KiB
Python
180 lines
11 KiB
Python
"""Disposable HTTP/process protocol fixtures, NOT research or model QA evidence."""
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
TASK = '565ace1f-8cfc-4447-b978-66c36e7b767b'
|
|
|
|
class RunnerTests(unittest.TestCase):
|
|
def test_ready_claim_research_synthesis_result(self):
|
|
self.assertTrue((ROOT/'runner.py').exists(), 'actual runner missing')
|
|
spec = importlib.util.spec_from_file_location('runner', ROOT/'runner.py')
|
|
r = importlib.util.module_from_spec(spec); spec.loader.exec_module(r)
|
|
with tempfile.TemporaryDirectory() as d:
|
|
home = Path(d)
|
|
(home/'editorial-v4/credentials').mkdir(parents=True)
|
|
token = home/'editorial-v4/credentials/editorial-pmo-token'
|
|
token.write_text('a'*64); token.chmod(0o400)
|
|
for profile in r.PROFILES:
|
|
(home/'profiles'/profile).mkdir(parents=True)
|
|
fixture = home/'fixture.py'
|
|
fixture.write_text('''import json,os,sys
|
|
q=sys.stdin.read()
|
|
with open(os.environ['HERMES_HOME']+'/called','w') as f:f.write(q)
|
|
print(json.dumps({'type':'tool_use','name':'web_extract','input':{'urls':['https://www.avma.org/resources']}}))
|
|
print(json.dumps({'type':'tool_result','name':'web_extract','is_error':False,'output':json.dumps({'results':[{'url':'https://www.avma.org/resources','content':'test-only retrieval boundary'}]})}))
|
|
brief={'researchDate':'2026-01-01','inputAvailability':[{'input':'test','status':'PARTIAL','details':'fixture only'}],'findings':[{'sourceUrl':'https://www.avma.org/resources','sourceDate':'2026-01-01','finding':'fixture only'}],'candidates':[{'title':x,'score':60,'rationale':'fixture only','sourceUrls':['https://www.avma.org/resources']} for x in ['A','B','C']]}
|
|
print(json.dumps({'type':'result','exit_code':0,'text':json.dumps(brief)}))
|
|
''')
|
|
requests = []
|
|
mode = {'state':'READY','heartbeat':200}
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self,*a): pass
|
|
def do_GET(self): self.respond({'taskId':TASK,'state':mode['state'],'operationMode':'PREPARE','budgetLimitCents':20,'metadata':{'scope':'owner topic','acceptanceCriteria':['three candidates'],'environment':'staging','commitSha':'a'*40}})
|
|
def do_POST(self):
|
|
body=json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
requests.append((self.path,body,self.headers.get('Authorization')))
|
|
if self.path.endswith('editorial-claim'): self.respond({'lease':{'taskId':TASK,'runId':TASK,'leaseToken':TASK,'commitSha':'a'*40,'environment':'staging','maxCostCents':20,'attempt':1,'leaseUntil':'2026-10-24T00:00:30.000Z','pilotKind':'editorial-research-to-brief-v1'}})
|
|
elif self.path.endswith('heartbeat'):
|
|
self.assert_lease(body)
|
|
self.respond({'leaseUntil':'2026-10-24T00:00:30.000Z'},mode['heartbeat'])
|
|
else: self.respond({'state':'VERIFYING','qaStatus':'PENDING_INDEPENDENT_QA','retainedReservationCents':20,'costStatus':'upper-bound-reserved'})
|
|
def assert_lease(self,body):
|
|
if set(body) != {'taskId','runId','leaseToken'}:
|
|
raise AssertionError('Deployed LeaseDto forbids extra keys')
|
|
def respond(self,body,code=200):
|
|
self.send_response(code);self.end_headers();self.wfile.write(json.dumps(body).encode())
|
|
server=ThreadingHTTPServer(('127.0.0.1',0),Handler)
|
|
thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
|
|
try:
|
|
api=r.API(home, endpoint='http://127.0.0.1:'+str(server.server_port))
|
|
import sys
|
|
result=r.run(home,TASK,api=api,command=[sys.executable,str(fixture)],heartbeat_seconds=.1,process_seconds=2,total_seconds=10)
|
|
self.assertEqual(result['retainedReservationCents'],20)
|
|
self.assertEqual([p for p,_,_ in requests].count('/editorial-result'),1)
|
|
self.assertTrue(all(auth=='Bearer '+'a'*64 for _,_,auth in requests))
|
|
for profile in r.PROFILES: self.assertTrue((home/'profiles'/profile/'called').exists())
|
|
self.assertNotIn('costCents',requests[-1][1])
|
|
mode['heartbeat']=403; requests.clear()
|
|
with self.assertRaises(r.RunnerError):
|
|
r.run(home,TASK,api=api,command=[sys.executable,str(fixture)])
|
|
self.assertNotIn('/editorial-result',[p for p,_,_ in requests])
|
|
mode['heartbeat']=200
|
|
# Reject an actual heartbeat HTTP request while a child is running.
|
|
requests.clear()
|
|
def failing_pulse():
|
|
mode['heartbeat']=403
|
|
api.request('heartbeat',{'taskId':TASK,'runId':TASK,'leaseToken':TASK})
|
|
import time
|
|
with self.assertRaises(r.RunnerError):
|
|
r.execute([sys.executable,'-c','import time;time.sleep(60)'],'test',dict(os.environ),home,failing_pulse,time.monotonic()+5,heartbeat_seconds=.1)
|
|
self.assertEqual(requests[-1][0],'/heartbeat')
|
|
mode['state']='GATE1_PENDING'; requests.clear()
|
|
with self.assertRaises(r.RunnerError): r.run(home,TASK,api=api,command=[sys.executable,str(fixture)])
|
|
self.assertEqual(requests,[])
|
|
finally: server.shutdown();server.server_close();thread.join()
|
|
|
|
@unittest.skipUnless(os.getuid()==0, 'requires disposable UID-drop seam')
|
|
def test_actual_uid_drop_cannot_enter_root_private_cache(self):
|
|
# Minimal real process repro of installed shim/cache ownership boundary;
|
|
# never calls Hermes, a provider, a live endpoint or the installed runner.
|
|
import subprocess
|
|
with tempfile.TemporaryDirectory() as d:
|
|
cache=Path(d)/'cache';cache.mkdir(mode=0o700)
|
|
os.chmod(d,0o755)
|
|
child=subprocess.run([sys.executable,'-c',
|
|
'import os,sys; os.setgid(10000); os.setuid(10000); os.chdir(sys.argv[1])',str(cache)],
|
|
capture_output=True)
|
|
self.assertNotEqual(child.returncode,0)
|
|
self.assertIn(b'PermissionError',child.stderr)
|
|
|
|
def test_profile_identity_mismatch_rejected_before_claim(self):
|
|
# Actual installed failure: docker exec UID0 creates cache0700, shim
|
|
# drops child to UID10000. Ownership probe requires no paid CLI call.
|
|
import runner as r
|
|
from unittest.mock import patch
|
|
with tempfile.TemporaryDirectory() as d:
|
|
home=Path(d)
|
|
for profile in r.PROFILES:
|
|
(home/'profiles'/profile).mkdir(parents=True)
|
|
class NeverClaim:
|
|
def request(self, *args):
|
|
raise AssertionError('Identity preflight must precede HTTP/claim')
|
|
with patch.object(r.os, 'getuid', return_value=os.getuid()+1):
|
|
with self.assertRaises(r.RunnerError) as caught:
|
|
r.run(home,TASK,api=NeverClaim(),command=[sys.executable])
|
|
self.assertEqual(caught.exception.stage,'preflight')
|
|
self.assertEqual(caught.exception.code,'PROFILE_IDENTITY_MISMATCH')
|
|
|
|
def test_safe_stage_diagnostic_does_not_reflect_exception(self):
|
|
import runner as r
|
|
with tempfile.TemporaryDirectory() as d:
|
|
home=Path(d)
|
|
for profile in r.PROFILES:
|
|
(home/'profiles'/profile).mkdir(parents=True)
|
|
class BrokenHTTP:
|
|
def request(self,*args):
|
|
raise OSError('secret-provider-token-and-body')
|
|
with self.assertRaises(r.RunnerError) as caught:
|
|
r.run(home,TASK,api=BrokenHTTP(),command=[sys.executable])
|
|
self.assertEqual(caught.exception.stage,'status')
|
|
self.assertEqual(caught.exception.code,'OS_ERROR')
|
|
self.assertNotIn('secret',str(caught.exception))
|
|
|
|
def test_brief_rejects_unknown_keys_and_invented_sources(self):
|
|
import runner as r
|
|
b={'researchDate':'2026-01-01','inputAvailability':[{'input':'analytics','status':'UNAVAILABLE','details':'not available'}],'findings':[{'sourceUrl':'https://www.avma.org/resources','sourceDate':'2026-01-01','finding':'protocol fixture'}],'candidates':[{'title':x,'score':60,'rationale':'protocol fixture','sourceUrls':['https://www.avma.org/resources']} for x in ['A','B','C']]}
|
|
self.assertEqual(r.validate_brief(b),b)
|
|
with self.assertRaisesRegex(r.RunnerError,'retrieved'):
|
|
r.validate_sources(b,[])
|
|
failed=[{'retrievalEvents':[{'type':'tool_use','name':'web_extract','input':{'urls':['https://www.avma.org/resources']}},{'type':'tool_result','name':'web_extract','output':json.dumps({'results':[{'url':'https://www.avma.org/resources','error':'403','content':''}]})}]}]
|
|
with self.assertRaisesRegex(r.RunnerError,'retrieved'): r.validate_sources(b,failed)
|
|
b['qaPassed']=True
|
|
with self.assertRaises(r.RunnerError): r.validate_brief(b)
|
|
|
|
def test_heartbeat_loss_kills_process_group(self):
|
|
import runner as r
|
|
import sys
|
|
import time
|
|
with tempfile.TemporaryDirectory() as d:
|
|
cwd=Path(d)
|
|
script=cwd/'hang.py'
|
|
script.write_text("import subprocess,time,os\np=subprocess.Popen(['sleep','60'])\nopen('child','w').write(str(p.pid))\ntime.sleep(60)\n")
|
|
def pulse():
|
|
if (cwd/'child').exists(): raise r.RunnerError('HTTP heartbeat rejected')
|
|
started=time.monotonic()
|
|
with self.assertRaisesRegex(r.RunnerError,'heartbeat rejected'):
|
|
r.execute([sys.executable,str(script)],'test',dict(os.environ),cwd,pulse,None,heartbeat_seconds=.2)
|
|
self.assertLess(time.monotonic()-started,2)
|
|
pid=int((cwd/'child').read_text())
|
|
stat_path=Path('/proc')/str(pid)/'stat'
|
|
# Container PID1 may reap asynchronously: a zombie cannot keep running.
|
|
self.assertTrue(not stat_path.exists() or stat_path.read_text().split()[2]=='Z')
|
|
|
|
def test_wall_clock_kills_hung_provider(self):
|
|
import runner as r
|
|
import sys
|
|
import time
|
|
with tempfile.TemporaryDirectory() as d:
|
|
with self.assertRaisesRegex(r.RunnerError,'wall clock'):
|
|
r.execute([sys.executable,'-c','import time;time.sleep(60)'],'test',dict(os.environ),Path(d),lambda:None,time.monotonic()+.3,heartbeat_seconds=.1)
|
|
|
|
def test_terminal_protocol_rejects_failed_or_contaminated_output(self):
|
|
import runner as r
|
|
import sys
|
|
import time
|
|
with tempfile.TemporaryDirectory() as d:
|
|
for events in [ [{'type':'result','exit_code':1,'text':'looks fine'}], [{'type':'result','exit_code':0,'text':'{}'},{'type':'result','exit_code':0,'text':'{}'}], [{'type':'tool_use','name':'terminal'},{'type':'result','exit_code':0,'text':'{}'}] ]:
|
|
with self.subTest(events=events),self.assertRaises(r.RunnerError):
|
|
r.execute([sys.executable,'-c','print('+repr('\n'.join(json.dumps(x) for x in events))+')'],'test',dict(os.environ),Path(d),lambda:None,time.monotonic()+3)
|
|
|
|
if __name__=='__main__': unittest.main()
|