feat(corporate): add dormant manual source runtime and independent QA adapter

This commit is contained in:
Canina Automation committed 2026-10-28 12:48:29 +00:00
1 parent cc365bcda2
commit d80845af0d
10 files changed
+567

No files matched your search

+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
"""Explicit local integration pilot. Requires disposable migrated PostgreSQL :55441."""
import json
import os
from pathlib import Path
import secrets
import subprocess
import sys
import time
import urllib.request
import corporate_runtime as r
def main():
root = Path(__file__).resolve().parent.parent
scratch = Path(os.environ['CORPORATE_PILOT_DIR']).resolve()
if not str(scratch).startswith('/opt/data/cache/scratch/'):
raise ValueError('Scratch pilot output required')
scratch.mkdir(parents=True, exist_ok=True, mode=0o700)
env = dict(os.environ, CORPORATE_API_KEY=secrets.token_hex(32), CORPORATE_QA_API_KEY=secrets.token_hex(32),
CORPORATE_TEST_DATABASE_URL='postgresql://hermes@127.0.0.1:55441/corporate_test')
server = subprocess.Popen(['node', str(root/'scripts/corporate_pilot_server.cjs')], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
base = 'http://127.0.0.1:55442/api/admin/hermes'
transcript = []
def cli(command, *args, key=None):
cmd = [sys.executable,str(root/'scripts/corporate_runtime.py'),'--api',base,'--store',str(scratch/'store')]
if key: cmd += ['--key',key]
result = subprocess.run(cmd+[command,*args],env=env,text=True,capture_output=True,timeout=90)
if result.returncode:
raise ValueError(command + ' failed: ' + result.stderr)
value = json.loads(result.stdout)
transcript.append({'command':command,'result':value})
return value
def manifest(name, value):
path=scratch/(name+'.json');path.write_text(json.dumps(value));return str(path)
try:
for _ in range(100):
try:
req = urllib.request.Request(base+'/runtime', headers={'x-api-key':env['CORPORATE_API_KEY']})
with urllib.request.urlopen(req,timeout=1): break
except Exception:
if server.poll() is not None: raise ValueError('Nest pilot host failed')
time.sleep(.1)
else: raise ValueError('Nest readiness timeout')
assert cli('status')['schedulesEnabled'] is False
insight=cli('ingest',manifest('research',{
'title':'Manual official Hermes documentation source inventory pilot','category':'TREND',
'source':'Official Hermes documentation fetched over verified HTTPS',
'opportunity':'Validate read-only source provenance, not a clinical or market claim',
'impact':'LOW','recommendedAction':'Independent exact-byte inventory QA only', 'cadence':'EVENT',
'provenance':{'method':'manual HTTPS GET + SHA256','collector':'manual CLI pilot','context':'disposable local integration, not production analytics'},
'fetchSources':[{'url':'https://hermes-agent.nousresearch.com/docs/llms.txt','title':'Official Hermes docs index'}]
}),'--hosts','hermes-agent.nousresearch.com',key='pilot-research')
replay=cli('replay-ingest',insight['requestDigest'],key='pilot-research')
assert replay['id']==insight['research']['id']
roadmap=cli('roadmap',manifest('roadmap',{
'researchId':replay['id'],'cycleName':'Manual local pilot','month':r.now()[:7],
'seasonalTheme':'No seasonal inference','strategicGoals':['Verify read-only evidence'],
'plannedArticlesCount':0,'plannedRefreshesCount':0,'plannedCatalogEnrichments':0,'isActive':False,
'goals':[{'title':'Inventory fetched official docs source','department':'CONTENT_SEO',
'acceptanceCriteria':'Exact source byte digests match persisted provenance','metric':'source_inventory_verified'}]
}),key='pilot-roadmap')
tasks=cli('fanout',roadmap['id'],key='pilot-fanout')
assert cli('fanout',roadmap['id'],key='pilot-fanout')[0]['id']==tasks[0]['id']
execution=cli('execute','--department','CONTENT_SEO',key='pilot-execution')
task_id=execution['taskId']; evidence=execution['evidence']
# Executor identity cannot self-review; exercise real backend denial.
own_env=os.environ.copy()
try:
os.environ['CORPORATE_API_KEY']=env['CORPORATE_API_KEY']
c=r.Client(base,'CORPORATE_API_KEY')
try: r.review(c,r.Store(scratch/'store'),task_id,evidence['id'],'self-review-denied')
except ValueError as e: assert '409' in str(e)
else: raise AssertionError('self-review unexpectedly allowed')
finally:
os.environ.clear();os.environ.update(own_env)
cli('qa',task_id,evidence['id'],key='pilot-independent-qa')
cli('effect',task_id,manifest('effect',{'metric':'production_engagement','availability':'UNINSTRUMENTED',
'source':'Local pilot has no production analytics instrumentation','observedAt':r.now()}),key='pilot-effect')
cli('report',manifest('report',{'day':r.now()[:10], 'summary':'Manual local source-inventory pilot only; no publishing, deployment or measured production effect.'}),key='pilot-report')
req=urllib.request.Request(base+'/tasks/'+task_id,headers={'x-api-key':env['CORPORATE_QA_API_KEY']})
with urllib.request.urlopen(req) as response: detail=json.load(response)
assert detail['status']=='DONE' and len(detail['effects'])==1 and len(detail['reviews'])==1
assert detail['evidence'][0]['sha256']==evidence['sha256']
assert detail['reviews'][0]['reviewer'] != detail['evidence'][0]['executor']
transcript.append({'persistedReadback':{'taskId':task_id,'status':detail['status'],
'logs':[x['step'] for x in detail['logs']], 'effects':detail['effects'],
'sha256':evidence['sha256'],'independentSubjects':True,'selfReviewDenied':True}})
(scratch/'transcript.json').write_text(json.dumps(transcript,indent=2))
print(json.dumps({'status':'PASS','taskId':task_id,'artifactDigest':evidence['sha256'],
'transcript':str(scratch/'transcript.json')}))
finally:
server.terminate()
try: server.communicate(timeout=10)
except subprocess.TimeoutExpired: server.kill();server.communicate()
if __name__=='__main__': main()
+24
View File
@@ -0,0 +1,24 @@
// Disposable loopback Nest feature host; test-auth only, never runtime deployment.
process.env.TS_NODE_PROJECT = require('path').resolve(__dirname, '../backend/tsconfig.json');
require('../backend/node_modules/ts-node/register/transpile-only');
const { Test } = require('../backend/node_modules/@nestjs/testing');
const { PrismaClient } = require('../backend/node_modules/@prisma/client');
const { CorporateController } = require('../backend/src/admin/hermes/corporate.controller');
const { CorporateService } = require('../backend/src/admin/hermes/corporate.service');
const { PrismaService } = require('../backend/src/prisma/prisma.service');
const { ApiKeysService } = require('../backend/src/admin/api-keys.service');
(async () => {
const u = new URL(process.env.CORPORATE_TEST_DATABASE_URL);
if (u.hostname !== '127.0.0.1' || u.port !== '55441' || u.pathname !== '/corporate_test') throw Error('disposable DB required');
const db = new PrismaClient({datasources:{db:{url:u.href}}});
const module = await Test.createTestingModule({controllers:[CorporateController], providers:[CorporateService,
{provide:PrismaService,useValue:db}, {provide:ApiKeysService,useValue:{validateKey:async key => {
const role = key === process.env.CORPORATE_API_KEY ? 'executor' : key === process.env.CORPORATE_QA_API_KEY ? 'reviewer' : null;
return role ? {id:'pilot-' + role,isApiKey:true,role:'Admin',scopes:['admin']} : null;
}}}]}).compile();
const app = module.createNestApplication({logger:false});
app.setGlobalPrefix('api');
await app.listen(55442,'127.0.0.1');
console.log('READY loopback test-auth Nest corporate feature');
for (const signal of ['SIGTERM','SIGINT']) process.on(signal,async()=>{await app.close();await db.$disconnect();process.exit(0);});
})().catch(()=>{console.error('Pilot server failed (no secrets logged)');process.exit(1);});
+312
View File
@@ -0,0 +1,312 @@
#!/usr/bin/env python3
"""Dormant manual Hermes terminal adapter. No model, scheduler or shell execution."""
import argparse
import hashlib
import http.client
import ipaddress
import json
import os
from pathlib import Path
import re
import socket
import ssl
import sys
from datetime import datetime, timezone
from urllib.parse import urlsplit, quote
import urllib.request
import urllib.error
MAX_BYTES = 2_000_000
def canonical(value):
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode()
def sha(data):
return hashlib.sha256(data).hexdigest()
def now():
return datetime.now(timezone.utc).isoformat()
class Store:
def __init__(self, root):
self.root = Path(root).resolve()
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
def path(self, digest):
if not re.fullmatch('[a-f0-9]{64}', digest):
raise ValueError('Invalid digest')
path = self.root / digest
if path.is_symlink():
raise ValueError('Symlink forbidden')
return path
def put(self, data):
digest = sha(data)
path = self.path(digest)
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
except FileExistsError:
self.get(digest)
else:
with os.fdopen(fd, 'wb') as f:
f.write(data)
return digest
def get(self, digest):
data = self.path(digest).read_bytes()
if sha(data) != digest:
raise ValueError('Artifact digest mismatch')
return data
def source_target(url, hosts):
p = urlsplit(url)
if (p.scheme != 'https' or p.hostname not in hosts or p.username or p.password
or p.port not in (None, 443) or p.fragment):
raise ValueError('Source must be allowlisted public HTTPS on port 443')
addresses = sorted({row[4][0] for row in socket.getaddrinfo(p.hostname, 443, type=socket.SOCK_STREAM)})
if not addresses or any(not ipaddress.ip_address(x).is_global for x in addresses):
raise ValueError('Non-public source address')
return p, addresses[0]
def fetch(url, hosts):
p, address = source_target(url, hosts)
# Pin resolved public address; keep hostname for TLS certificate/SNI validation.
class PinnedHTTPS(http.client.HTTPSConnection):
def connect(self):
raw = socket.create_connection((address, 443), timeout=20)
self.sock = ssl.create_default_context().wrap_socket(raw, server_hostname=p.hostname)
conn = PinnedHTTPS(p.hostname, timeout=20)
try:
conn.request('GET', p.path + ('?' + p.query if p.query else '') or '/',
headers={'User-Agent': 'Canina-Manual-Research/1.0', 'Accept-Encoding': 'identity'})
response = conn.getresponse()
if response.status != 200:
raise ValueError('Source HTTP status ' + str(response.status) + '; redirects forbidden')
data = response.read(MAX_BYTES + 1)
if not data or len(data) > MAX_BYTES:
raise ValueError('Empty or oversized source')
return data, {'url': url, 'sha256': sha(data), 'retrievedAt': now(),
'status': response.status, 'bytes': len(data),
'contentType': response.getheader('Content-Type', '')}
finally:
conn.close()
def api_target(url, allow_stage):
p = urlsplit(url)
if p.username or p.password or p.query or p.fragment or p.path.rstrip('/') != '/api/admin/hermes':
raise ValueError('Invalid API base')
if not (p.scheme == 'http' and p.hostname == '127.0.0.1'):
if not (allow_stage and p.scheme == 'https' and p.hostname):
raise ValueError('API is loopback-only unless --allow-stage-https explicitly supplied')
return url.rstrip('/')
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
raise ValueError('API redirects forbidden')
class Client:
def __init__(self, base, credential_env, allow_stage=False):
self.base = api_target(base, allow_stage)
self.secret = os.environ.get(credential_env)
if not self.secret or '\n' in self.secret or '\r' in self.secret:
raise ValueError('Required credential environment variable missing/invalid')
# Avoid ambient cookie/netrc/proxy credentials; never follow redirects.
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
def checkpoint(self, store, key, path, body):
"""Save immutable request before network mutation, enable explicit safe replay."""
return store.put(canonical({'path': path, 'body': body, 'key': key}))
def replay(self, store, digest):
saved = json.loads(store.get(digest))
path = saved['path']
if not re.fullmatch(r'tasks/[a-f0-9-]{36}/evidence', path):
raise ValueError('Only an evidence checkpoint may be replayed')
return self.call(path, saved['body'], saved['key'])
# All requests use fixed adapter routes, not source-provided paths.
def call(self, path, body=None, key=None):
headers = {'x-api-key': self.secret, 'Accept': 'application/json'}
if body is not None:
if not key:
raise ValueError('Explicit idempotency key required')
headers.update({'Idempotency-Key': key, 'Content-Type': 'application/json'})
req = urllib.request.Request(self.base + '/' + path, data=canonical(body) if body is not None else None, headers=headers)
try:
with self.opener.open(req, timeout=30) as response:
value = json.load(response)
except urllib.error.HTTPError as e:
# Do not reflect response body, URL or credentials into logs.
raise ValueError('Corporate API HTTP ' + str(e.code)) from None
return value
def research(self, research_id):
page = 1
while True:
response = self.call('market-insights?limit=100&page=' + str(page))
for item in response['data']:
if item['id'] == research_id:
return item
if not response['meta']['hasNextPage']:
raise ValueError('Research not found')
page += 1
def operation(description):
try:
op = json.loads(description)
except (ValueError, TypeError):
raise ValueError('Structured source inventory operation required') from None
if not isinstance(op, dict) or set(op) != {'operation', 'researchId'} or op['operation'] != 'source_inventory_v1':
raise ValueError('Only source_inventory_v1 is executable')
if not isinstance(op['researchId'], str) or not op['researchId']:
raise ValueError('Research ID required')
return op
def inventory(task_id, research, store):
sources = []
for source in research['sources']:
raw = store.get(source['sha256'])
sources.append({k: source[k] for k in ('url', 'sha256', 'retrievedAt')} | {'bytes': len(raw)})
if not sources:
raise ValueError('No persisted sources')
return {'operation': 'source_inventory_v1', 'taskId': task_id, 'researchId': research['id'],
'sources': sorted(sources, key=lambda s: (s['url'], s['sha256']))}
def ingest(client, store, manifest, hosts, key):
body = dict(manifest)
sources = []
for source in body.pop('fetchSources'):
raw, receipt = fetch(source['url'], hosts)
store.put(raw)
store.put(canonical(receipt))
sources.append({k: receipt[k] for k in ('url', 'sha256', 'retrievedAt')} |
{'title': source['title'], 'excerpt': raw.decode('utf-8', errors='replace')[:1500]})
body['sources'] = sources
# Persist exact immutable request before HTTP so an operator can retry without refetching.
request_digest = store.put(canonical(body))
print('Research request checkpoint: ' + request_digest, file=sys.stderr)
result = client.call('market-insights', body, key)
return {'research': result, 'requestDigest': request_digest}
def fanout(client, roadmap, key):
tasks = []
for goal in roadmap['goals']:
body = {'title': goal['title'], 'department': goal['department'], 'goalId': goal['id'],
'actionType': 'RESEARCH', 'description': json.dumps({'operation': 'source_inventory_v1', 'researchId': roadmap['researchId']})}
tasks.append(client.call('tasks', body, key + ':' + goal['id']))
return tasks
def execute(client, store, department, key):
task = client.call('claims', {'department': department, 'leaseSeconds': 300}, key + ':claim')['task']
if task is None:
return {'task': None}
# Nothing from title/description is evaluated as Python, shell, or LLM instructions.
if task['actionType'] != 'RESEARCH':
raise ValueError('Claimed task is not RESEARCH; left leased, no evidence submitted')
op = operation(task['description'])
started = now()
artifact = inventory(task['id'], client.research(op['researchId']), store)
digest = store.put(canonical(artifact))
result = {'startedAt': started, 'finishedAt': now(), 'operation': op['operation'],
'bytes': len(canonical(artifact)), 'sourceCount': len(artifact['sources']), 'sha256': digest}
receipt_digest = store.put(canonical(result))
body = {'claimToken': task['claimToken'], 'artifactUri': 'urn:sha256:' + digest, 'sha256': digest,
'summary': 'Read-only persisted source inventory; no publishing or production action',
'tool': 'python:source_inventory_v1', 'result': json.dumps(result)}
checkpoint = client.checkpoint(store, key + ':evidence', 'tasks/' + task['id'] + '/evidence', body)
print('Evidence checkpoint: ' + checkpoint, file=sys.stderr)
evidence = client.call('tasks/' + task['id'] + '/evidence', body, key + ':evidence')
return {'taskId': task['id'], 'evidence': evidence, 'toolReceiptDigest': receipt_digest}
def review(client, store, task_id, evidence_id, key):
task = client.call('tasks/' + task_id)
evidence = next(e for e in task['evidence'] if e['id'] == evidence_id)
raw = store.get(evidence['sha256'])
op = operation(task['description'])
expected = inventory(task_id, client.research(op['researchId']), store)
if raw != canonical(expected) or evidence['artifactUri'] != 'urn:sha256:' + sha(raw):
raise ValueError('Exact artifact/source inventory verification failed; no pass submitted')
return client.call('tasks/' + task_id + '/reviews', {
'evidenceId': evidence_id, 'passed': True,
'findings': 'Reopened content-addressed bytes, recomputed exact deterministic inventory from API provenance and source bytes',
'verification': {'artifactUri': evidence['artifactUri'], 'sha256': sha(raw),
'method': 'independent local bytes + persisted provenance recomputation',
'result': json.dumps({'verifiedAt': now(), 'sourceCount': len(expected['sources']), 'bytes': len(raw)})}}, key)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--api', default='http://127.0.0.1:3000/api/admin/hermes')
p.add_argument('--allow-stage-https', action='store_true')
p.add_argument('--store', required=True)
p.add_argument('--key', help='Stable operation key; retain for retries')
sub = p.add_subparsers(dest='command', required=True)
i = sub.add_parser('ingest'); i.add_argument('manifest'); i.add_argument('--hosts', required=True)
i = sub.add_parser('replay-ingest'); i.add_argument('request_digest')
i = sub.add_parser('replay-evidence'); i.add_argument('checkpoint_digest')
i = sub.add_parser('roadmap'); i.add_argument('manifest')
i = sub.add_parser('fanout'); i.add_argument('roadmap_id')
i = sub.add_parser('execute'); i.add_argument('--department', required=True)
i = sub.add_parser('qa'); i.add_argument('task_id'); i.add_argument('evidence_id')
i = sub.add_parser('effect'); i.add_argument('task_id'); i.add_argument('manifest')
i = sub.add_parser('report'); i.add_argument('manifest')
sub.add_parser('status')
a = p.parse_args()
if a.command != 'status' and not a.key:
p.error('--key required for mutations')
c = Client(a.api, 'CORPORATE_QA_API_KEY' if a.command == 'qa' else 'CORPORATE_API_KEY', a.allow_stage_https)
s = Store(a.store)
load = lambda: json.loads(Path(a.manifest).read_text())
if a.command == 'ingest': result = ingest(c, s, load(), set(a.hosts.split(',')), a.key)
elif a.command == 'replay-ingest': result = c.call('market-insights', json.loads(s.get(a.request_digest)), a.key)
elif a.command == 'replay-evidence': result = c.replay(s, a.checkpoint_digest)
elif a.command == 'roadmap':
body = load()
if body.get('isActive') is not False: raise ValueError('Manual roadmap must explicitly be inactive')
result = c.call('monthly-roadmap', body, a.key)
elif a.command == 'fanout':
page = 1
while True:
collection = c.call('monthly-roadmap?limit=100&page=' + str(page))
road = next((x for x in collection['data'] if x['id'] == a.roadmap_id), None)
if road: break
if not collection['meta']['hasNextPage']: raise ValueError('Roadmap not found')
page += 1
result = fanout(c, road, a.key)
elif a.command == 'execute': result = execute(c, s, a.department, a.key)
elif a.command == 'qa': result = review(c, s, a.task_id, a.evidence_id, a.key)
elif a.command == 'effect': result = c.call('tasks/' + a.task_id + '/effects', load(), a.key)
elif a.command == 'report': result = c.call('reports', load(), a.key)
else: result = c.call('runtime')
# Claim tokens may appear inside raw tasks: redact capability material from stdout.
def clean(v):
if isinstance(v, dict): return {k: clean(x) for k, x in v.items() if k != 'claimToken'}
if isinstance(v, list): return [clean(x) for x in v]
return v
print(json.dumps(clean(result), ensure_ascii=False))
if __name__ == '__main__':
try:
main()
except Exception as error:
# Only known validation/status messages; transport exceptions can contain secrets.
print(str(error) if isinstance(error, ValueError) and not isinstance(error, json.JSONDecodeError)
else 'Operation failed; inspect local store/API safely (no credentials logged)', file=sys.stderr)
sys.exit(1)
+48
View File
@@ -0,0 +1,48 @@
import importlib.util
import pathlib
import tempfile
import unittest
spec = importlib.util.spec_from_file_location('runtime', pathlib.Path(__file__).with_name('corporate_runtime.py'))
r = importlib.util.module_from_spec(spec)
spec.loader.exec_module(r)
class Safety(unittest.TestCase):
def test_urls_fail_closed(self):
for url in ['http://example.com/', 'https://localhost/', 'https://127.0.0.1/', 'https://example.com:444/', 'https://user:pass@example.com/', 'file:///etc/passwd']:
with self.subTest(url=url), self.assertRaises(ValueError):
r.source_target(url, {'example.com'})
def test_only_exact_operation(self):
for desc in ['rm -rf /', '{"operation":"publish"}', '{"operation":"source_inventory_v1","researchId":"x","command":"id"}']:
with self.assertRaises(ValueError):
r.operation(desc)
def test_store_digest_tamper(self):
with tempfile.TemporaryDirectory() as tmp:
store = r.Store(tmp)
digest = store.put(b'actual bytes')
self.assertEqual(store.get(digest), b'actual bytes')
(pathlib.Path(tmp) / digest).write_bytes(b'changed')
with self.assertRaises(ValueError): store.get(digest)
def test_artifact_recomputed(self):
research = {'id':'research', 'sources':[{'url':'https://example.com/', 'sha256':r.sha(b'hello'), 'retrievedAt':'now'}]}
with tempfile.TemporaryDirectory() as tmp:
store = r.Store(tmp); store.put(b'hello')
artifact = r.inventory('task', research, store)
self.assertEqual(artifact['sources'][0]['bytes'],5)
self.assertEqual(artifact, r.inventory('task',research,store))
def test_checkpoint_replays_exact_receipt(self):
client = object.__new__(r.Client)
calls = []
client.call = lambda path, body, key: calls.append((path, body, key)) or {'id': 'persisted'}
with tempfile.TemporaryDirectory() as tmp:
store = r.Store(tmp)
path = 'tasks/618a6800-d241-41e2-b093-969164c83cfa/evidence'
digest = client.checkpoint(store, 'stable-key', path, {'result': 'original timestamp'})
self.assertEqual(client.replay(store, digest), {'id': 'persisted'})
self.assertEqual(calls, [(path, {'result':'original timestamp'}, 'stable-key')])
def test_api_target(self):
with self.assertRaises(ValueError): r.api_target('http://example.com/api/admin/hermes',False)
with self.assertRaises(ValueError): r.api_target('https://example.com/api/admin/hermes',False)
self.assertEqual(r.api_target('http://127.0.0.1:1234/api/admin/hermes',False),'http://127.0.0.1:1234/api/admin/hermes')
if __name__ == '__main__': unittest.main()