Files
canina/infra/hermes-editorial/tests/test_deadline_evidence.py
T
2026-10-24 16:26:50 +00:00

96 lines
5.0 KiB
Python

"""Synthetic processes, never provider/QA evidence."""
import json
import os
from pathlib import Path
import sys
import tempfile
import time
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import runner as r
class DeadlineEvidenceTests(unittest.TestCase):
def test_timeout_retains_safe_session_and_counts_not_payload(self):
with tempfile.TemporaryDirectory() as d:
cwd = Path(d)
evidence = cwd/'evidence.json'
events = [
{'type':'system','subtype':'init','session_id':'20261024_132234_a57b57','model':'secret'},
{'type':'tool_use','name':'web_extract','input':{'secret':'token'}},
{'type':'tool_result','name':'web_extract','output':'secret body','duration_ms':970},
{'type':'text','text':'secret answer'},
]
script = 'import time; print('+repr('\n'.join(json.dumps(x) for x in events))+',flush=True); time.sleep(60)'
started = time.monotonic()
with self.assertRaises(r.RunnerError) as caught:
r.execute([sys.executable,'-c',script],'secret prompt',dict(os.environ),cwd,
lambda:None,time.monotonic()+2,process_seconds=.25,
heartbeat_seconds=.05,evidence_path=evidence)
self.assertEqual(caught.exception.code,'PROFILE_TIMEOUT')
self.assertLess(time.monotonic()-started,1.5)
data = json.loads(evidence.read_text())
self.assertEqual(data['sessionId'],'20261024_132234_a57b57')
self.assertEqual(data['toolUseCount'],1)
self.assertEqual(data['toolResultCount'],1)
self.assertEqual(data['terminalResultCount'],0)
self.assertEqual(data['lastEventType'],'text')
self.assertEqual(data['outcome'],'PROFILE_TIMEOUT')
self.assertNotIn('secret',evidence.read_text())
self.assertEqual(evidence.stat().st_mode & 0o777,0o600)
def test_bringup_has_no_implicit_profile_or_total_deadline(self):
import inspect
self.assertIsNone(inspect.signature(r.run).parameters['process_seconds'].default)
self.assertIsNone(inspect.signature(r.run).parameters['total_seconds'].default)
self.assertIsNone(inspect.signature(r.execute).parameters['deadline'].default)
source = inspect.getsource(r.run)
self.assertIn("'--max-turns','0','--run-budget','0'", source)
self.assertNotIn('cli_budget', source)
def test_installed_emitter_timeout_evidence_contract(self):
import contextlib
import importlib.util
import io
path = Path(os.environ.get('HERMES_STREAM_JSON_SOURCE', '/opt/hermes/hermes_cli/stream_json.py'))
if not path.exists(): self.skipTest('installed emitter unavailable')
spec = importlib.util.spec_from_file_location('deadline_emitter', path)
module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module)
stream = io.StringIO()
with contextlib.redirect_stdout(stream):
emitter = module.StreamJsonEmitter(model='synthetic', session_id='20261024_132234_a57b57')
emitter.on_tool_progress('tool.started','web_extract',args={'urls':['https://example.test']},tool_call_id='synthetic')
emitter.on_tool_progress('tool.completed','web_extract',result='synthetic private',tool_call_id='synthetic')
emitter.on_text_delta('synthetic partial answer')
with tempfile.TemporaryDirectory() as d:
cwd = Path(d)
evidence = cwd/'evidence.json'
source = 'import sys,time; sys.stdout.write('+repr(stream.getvalue())+'); sys.stdout.flush(); time.sleep(60)'
with self.assertRaises(r.RunnerError):
r.execute([sys.executable,'-c',source],'',dict(os.environ),cwd,lambda:None,
time.monotonic()+.25,heartbeat_seconds=.05,evidence_path=evidence)
data = json.loads(evidence.read_text())
self.assertEqual(data['sessionId'],'20261024_132234_a57b57')
self.assertEqual(data['toolResultCount'],1)
self.assertEqual(data['lastEventType'],'text')
self.assertEqual(data['outcome'],'PROFILE_TIMEOUT')
self.assertNotIn('synthetic',evidence.read_text())
def test_invalid_stream_cannot_inject_diagnostic_fields(self):
with tempfile.TemporaryDirectory() as d:
cwd = Path(d)
evidence = cwd/'evidence.json'
payload = json.dumps({'type':'secret','session_id':'secret','outcome':'QA_PASS'})
with self.assertRaises(r.RunnerError):
r.execute([sys.executable,'-c','print('+repr(payload)+')'],'',dict(os.environ),cwd,
lambda:None,time.monotonic()+2,evidence_path=evidence,heartbeat_seconds=.05)
data = json.loads(evidence.read_text())
self.assertIsNone(data['sessionId'])
self.assertIsNone(data['lastEventType'])
self.assertNotIn('secret',evidence.read_text())
if __name__ == '__main__':
unittest.main()