96 lines
5.0 KiB
Python
96 lines
5.0 KiB
Python
"""Synthetic processes, never provider/QA evidence."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
import runner as r
|
|
|
|
|
|
class DeadlineEvidenceTests(unittest.TestCase):
|
|
def test_timeout_retains_safe_session_and_counts_not_payload(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
cwd = Path(d)
|
|
evidence = cwd/'evidence.json'
|
|
events = [
|
|
{'type':'system','subtype':'init','session_id':'20261024_132234_a57b57','model':'secret'},
|
|
{'type':'tool_use','name':'web_extract','input':{'secret':'token'}},
|
|
{'type':'tool_result','name':'web_extract','output':'secret body','duration_ms':970},
|
|
{'type':'text','text':'secret answer'},
|
|
]
|
|
script = 'import time; print('+repr('\n'.join(json.dumps(x) for x in events))+',flush=True); time.sleep(60)'
|
|
started = time.monotonic()
|
|
with self.assertRaises(r.RunnerError) as caught:
|
|
r.execute([sys.executable,'-c',script],'secret prompt',dict(os.environ),cwd,
|
|
lambda:None,time.monotonic()+2,process_seconds=.25,
|
|
heartbeat_seconds=.05,evidence_path=evidence)
|
|
self.assertEqual(caught.exception.code,'PROFILE_TIMEOUT')
|
|
self.assertLess(time.monotonic()-started,1.5)
|
|
data = json.loads(evidence.read_text())
|
|
self.assertEqual(data['sessionId'],'20261024_132234_a57b57')
|
|
self.assertEqual(data['toolUseCount'],1)
|
|
self.assertEqual(data['toolResultCount'],1)
|
|
self.assertEqual(data['terminalResultCount'],0)
|
|
self.assertEqual(data['lastEventType'],'text')
|
|
self.assertEqual(data['outcome'],'PROFILE_TIMEOUT')
|
|
self.assertNotIn('secret',evidence.read_text())
|
|
self.assertEqual(evidence.stat().st_mode & 0o777,0o600)
|
|
|
|
def test_bringup_has_no_implicit_profile_or_total_deadline(self):
|
|
import inspect
|
|
self.assertIsNone(inspect.signature(r.run).parameters['process_seconds'].default)
|
|
self.assertIsNone(inspect.signature(r.run).parameters['total_seconds'].default)
|
|
self.assertIsNone(inspect.signature(r.execute).parameters['deadline'].default)
|
|
source = inspect.getsource(r.run)
|
|
self.assertIn("'--max-turns','0','--run-budget','0'", source)
|
|
self.assertNotIn('cli_budget', source)
|
|
|
|
def test_installed_emitter_timeout_evidence_contract(self):
|
|
import contextlib
|
|
import importlib.util
|
|
import io
|
|
path = Path(os.environ.get('HERMES_STREAM_JSON_SOURCE', '/opt/hermes/hermes_cli/stream_json.py'))
|
|
if not path.exists(): self.skipTest('installed emitter unavailable')
|
|
spec = importlib.util.spec_from_file_location('deadline_emitter', path)
|
|
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
|
|
stream = io.StringIO()
|
|
with contextlib.redirect_stdout(stream):
|
|
emitter = module.StreamJsonEmitter(model='synthetic', session_id='20261024_132234_a57b57')
|
|
emitter.on_tool_progress('tool.started','web_extract',args={'urls':['https://example.test']},tool_call_id='synthetic')
|
|
emitter.on_tool_progress('tool.completed','web_extract',result='synthetic private',tool_call_id='synthetic')
|
|
emitter.on_text_delta('synthetic partial answer')
|
|
with tempfile.TemporaryDirectory() as d:
|
|
cwd = Path(d)
|
|
evidence = cwd/'evidence.json'
|
|
source = 'import sys,time; sys.stdout.write('+repr(stream.getvalue())+'); sys.stdout.flush(); time.sleep(60)'
|
|
with self.assertRaises(r.RunnerError):
|
|
r.execute([sys.executable,'-c',source],'',dict(os.environ),cwd,lambda:None,
|
|
time.monotonic()+.25,heartbeat_seconds=.05,evidence_path=evidence)
|
|
data = json.loads(evidence.read_text())
|
|
self.assertEqual(data['sessionId'],'20261024_132234_a57b57')
|
|
self.assertEqual(data['toolResultCount'],1)
|
|
self.assertEqual(data['lastEventType'],'text')
|
|
self.assertEqual(data['outcome'],'PROFILE_TIMEOUT')
|
|
self.assertNotIn('synthetic',evidence.read_text())
|
|
|
|
def test_invalid_stream_cannot_inject_diagnostic_fields(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
cwd = Path(d)
|
|
evidence = cwd/'evidence.json'
|
|
payload = json.dumps({'type':'secret','session_id':'secret','outcome':'QA_PASS'})
|
|
with self.assertRaises(r.RunnerError):
|
|
r.execute([sys.executable,'-c','print('+repr(payload)+')'],'',dict(os.environ),cwd,
|
|
lambda:None,time.monotonic()+2,evidence_path=evidence,heartbeat_seconds=.05)
|
|
data = json.loads(evidence.read_text())
|
|
self.assertIsNone(data['sessionId'])
|
|
self.assertIsNone(data['lastEventType'])
|
|
self.assertNotIn('secret',evidence.read_text())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|