feat(corporate): add dormant manual source runtime and independent QA adapter
This commit is contained in:
1 parent
cc365bcda2
commit
d80845af0d
10 files changed
+567
No files matched your search
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dormant manual Hermes terminal adapter. No model, scheduler or shell execution."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import http.client
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import socket
|
||||
import ssl
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlsplit, quote
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
MAX_BYTES = 2_000_000
|
||||
|
||||
|
||||
def canonical(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False).encode()
|
||||
|
||||
|
||||
def sha(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, root):
|
||||
self.root = Path(root).resolve()
|
||||
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
|
||||
def path(self, digest):
|
||||
if not re.fullmatch('[a-f0-9]{64}', digest):
|
||||
raise ValueError('Invalid digest')
|
||||
path = self.root / digest
|
||||
if path.is_symlink():
|
||||
raise ValueError('Symlink forbidden')
|
||||
return path
|
||||
|
||||
def put(self, data):
|
||||
digest = sha(data)
|
||||
path = self.path(digest)
|
||||
try:
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
except FileExistsError:
|
||||
self.get(digest)
|
||||
else:
|
||||
with os.fdopen(fd, 'wb') as f:
|
||||
f.write(data)
|
||||
return digest
|
||||
|
||||
def get(self, digest):
|
||||
data = self.path(digest).read_bytes()
|
||||
if sha(data) != digest:
|
||||
raise ValueError('Artifact digest mismatch')
|
||||
return data
|
||||
|
||||
|
||||
def source_target(url, hosts):
|
||||
p = urlsplit(url)
|
||||
if (p.scheme != 'https' or p.hostname not in hosts or p.username or p.password
|
||||
or p.port not in (None, 443) or p.fragment):
|
||||
raise ValueError('Source must be allowlisted public HTTPS on port 443')
|
||||
addresses = sorted({row[4][0] for row in socket.getaddrinfo(p.hostname, 443, type=socket.SOCK_STREAM)})
|
||||
if not addresses or any(not ipaddress.ip_address(x).is_global for x in addresses):
|
||||
raise ValueError('Non-public source address')
|
||||
return p, addresses[0]
|
||||
|
||||
|
||||
def fetch(url, hosts):
|
||||
p, address = source_target(url, hosts)
|
||||
# Pin resolved public address; keep hostname for TLS certificate/SNI validation.
|
||||
class PinnedHTTPS(http.client.HTTPSConnection):
|
||||
def connect(self):
|
||||
raw = socket.create_connection((address, 443), timeout=20)
|
||||
self.sock = ssl.create_default_context().wrap_socket(raw, server_hostname=p.hostname)
|
||||
conn = PinnedHTTPS(p.hostname, timeout=20)
|
||||
try:
|
||||
conn.request('GET', p.path + ('?' + p.query if p.query else '') or '/',
|
||||
headers={'User-Agent': 'Canina-Manual-Research/1.0', 'Accept-Encoding': 'identity'})
|
||||
response = conn.getresponse()
|
||||
if response.status != 200:
|
||||
raise ValueError('Source HTTP status ' + str(response.status) + '; redirects forbidden')
|
||||
data = response.read(MAX_BYTES + 1)
|
||||
if not data or len(data) > MAX_BYTES:
|
||||
raise ValueError('Empty or oversized source')
|
||||
return data, {'url': url, 'sha256': sha(data), 'retrievedAt': now(),
|
||||
'status': response.status, 'bytes': len(data),
|
||||
'contentType': response.getheader('Content-Type', '')}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def api_target(url, allow_stage):
|
||||
p = urlsplit(url)
|
||||
if p.username or p.password or p.query or p.fragment or p.path.rstrip('/') != '/api/admin/hermes':
|
||||
raise ValueError('Invalid API base')
|
||||
if not (p.scheme == 'http' and p.hostname == '127.0.0.1'):
|
||||
if not (allow_stage and p.scheme == 'https' and p.hostname):
|
||||
raise ValueError('API is loopback-only unless --allow-stage-https explicitly supplied')
|
||||
return url.rstrip('/')
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
raise ValueError('API redirects forbidden')
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, base, credential_env, allow_stage=False):
|
||||
self.base = api_target(base, allow_stage)
|
||||
self.secret = os.environ.get(credential_env)
|
||||
if not self.secret or '\n' in self.secret or '\r' in self.secret:
|
||||
raise ValueError('Required credential environment variable missing/invalid')
|
||||
# Avoid ambient cookie/netrc/proxy credentials; never follow redirects.
|
||||
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
|
||||
|
||||
def checkpoint(self, store, key, path, body):
|
||||
"""Save immutable request before network mutation, enable explicit safe replay."""
|
||||
return store.put(canonical({'path': path, 'body': body, 'key': key}))
|
||||
|
||||
def replay(self, store, digest):
|
||||
saved = json.loads(store.get(digest))
|
||||
path = saved['path']
|
||||
if not re.fullmatch(r'tasks/[a-f0-9-]{36}/evidence', path):
|
||||
raise ValueError('Only an evidence checkpoint may be replayed')
|
||||
return self.call(path, saved['body'], saved['key'])
|
||||
|
||||
# All requests use fixed adapter routes, not source-provided paths.
|
||||
|
||||
def call(self, path, body=None, key=None):
|
||||
headers = {'x-api-key': self.secret, 'Accept': 'application/json'}
|
||||
if body is not None:
|
||||
if not key:
|
||||
raise ValueError('Explicit idempotency key required')
|
||||
headers.update({'Idempotency-Key': key, 'Content-Type': 'application/json'})
|
||||
req = urllib.request.Request(self.base + '/' + path, data=canonical(body) if body is not None else None, headers=headers)
|
||||
try:
|
||||
with self.opener.open(req, timeout=30) as response:
|
||||
value = json.load(response)
|
||||
except urllib.error.HTTPError as e:
|
||||
# Do not reflect response body, URL or credentials into logs.
|
||||
raise ValueError('Corporate API HTTP ' + str(e.code)) from None
|
||||
return value
|
||||
|
||||
def research(self, research_id):
|
||||
page = 1
|
||||
while True:
|
||||
response = self.call('market-insights?limit=100&page=' + str(page))
|
||||
for item in response['data']:
|
||||
if item['id'] == research_id:
|
||||
return item
|
||||
if not response['meta']['hasNextPage']:
|
||||
raise ValueError('Research not found')
|
||||
page += 1
|
||||
|
||||
|
||||
def operation(description):
|
||||
try:
|
||||
op = json.loads(description)
|
||||
except (ValueError, TypeError):
|
||||
raise ValueError('Structured source inventory operation required') from None
|
||||
if not isinstance(op, dict) or set(op) != {'operation', 'researchId'} or op['operation'] != 'source_inventory_v1':
|
||||
raise ValueError('Only source_inventory_v1 is executable')
|
||||
if not isinstance(op['researchId'], str) or not op['researchId']:
|
||||
raise ValueError('Research ID required')
|
||||
return op
|
||||
|
||||
|
||||
def inventory(task_id, research, store):
|
||||
sources = []
|
||||
for source in research['sources']:
|
||||
raw = store.get(source['sha256'])
|
||||
sources.append({k: source[k] for k in ('url', 'sha256', 'retrievedAt')} | {'bytes': len(raw)})
|
||||
if not sources:
|
||||
raise ValueError('No persisted sources')
|
||||
return {'operation': 'source_inventory_v1', 'taskId': task_id, 'researchId': research['id'],
|
||||
'sources': sorted(sources, key=lambda s: (s['url'], s['sha256']))}
|
||||
|
||||
|
||||
def ingest(client, store, manifest, hosts, key):
|
||||
body = dict(manifest)
|
||||
sources = []
|
||||
for source in body.pop('fetchSources'):
|
||||
raw, receipt = fetch(source['url'], hosts)
|
||||
store.put(raw)
|
||||
store.put(canonical(receipt))
|
||||
sources.append({k: receipt[k] for k in ('url', 'sha256', 'retrievedAt')} |
|
||||
{'title': source['title'], 'excerpt': raw.decode('utf-8', errors='replace')[:1500]})
|
||||
body['sources'] = sources
|
||||
# Persist exact immutable request before HTTP so an operator can retry without refetching.
|
||||
request_digest = store.put(canonical(body))
|
||||
print('Research request checkpoint: ' + request_digest, file=sys.stderr)
|
||||
result = client.call('market-insights', body, key)
|
||||
return {'research': result, 'requestDigest': request_digest}
|
||||
|
||||
|
||||
def fanout(client, roadmap, key):
|
||||
tasks = []
|
||||
for goal in roadmap['goals']:
|
||||
body = {'title': goal['title'], 'department': goal['department'], 'goalId': goal['id'],
|
||||
'actionType': 'RESEARCH', 'description': json.dumps({'operation': 'source_inventory_v1', 'researchId': roadmap['researchId']})}
|
||||
tasks.append(client.call('tasks', body, key + ':' + goal['id']))
|
||||
return tasks
|
||||
|
||||
|
||||
def execute(client, store, department, key):
|
||||
task = client.call('claims', {'department': department, 'leaseSeconds': 300}, key + ':claim')['task']
|
||||
if task is None:
|
||||
return {'task': None}
|
||||
# Nothing from title/description is evaluated as Python, shell, or LLM instructions.
|
||||
if task['actionType'] != 'RESEARCH':
|
||||
raise ValueError('Claimed task is not RESEARCH; left leased, no evidence submitted')
|
||||
op = operation(task['description'])
|
||||
started = now()
|
||||
artifact = inventory(task['id'], client.research(op['researchId']), store)
|
||||
digest = store.put(canonical(artifact))
|
||||
result = {'startedAt': started, 'finishedAt': now(), 'operation': op['operation'],
|
||||
'bytes': len(canonical(artifact)), 'sourceCount': len(artifact['sources']), 'sha256': digest}
|
||||
receipt_digest = store.put(canonical(result))
|
||||
body = {'claimToken': task['claimToken'], 'artifactUri': 'urn:sha256:' + digest, 'sha256': digest,
|
||||
'summary': 'Read-only persisted source inventory; no publishing or production action',
|
||||
'tool': 'python:source_inventory_v1', 'result': json.dumps(result)}
|
||||
checkpoint = client.checkpoint(store, key + ':evidence', 'tasks/' + task['id'] + '/evidence', body)
|
||||
print('Evidence checkpoint: ' + checkpoint, file=sys.stderr)
|
||||
evidence = client.call('tasks/' + task['id'] + '/evidence', body, key + ':evidence')
|
||||
return {'taskId': task['id'], 'evidence': evidence, 'toolReceiptDigest': receipt_digest}
|
||||
|
||||
|
||||
def review(client, store, task_id, evidence_id, key):
|
||||
task = client.call('tasks/' + task_id)
|
||||
evidence = next(e for e in task['evidence'] if e['id'] == evidence_id)
|
||||
raw = store.get(evidence['sha256'])
|
||||
op = operation(task['description'])
|
||||
expected = inventory(task_id, client.research(op['researchId']), store)
|
||||
if raw != canonical(expected) or evidence['artifactUri'] != 'urn:sha256:' + sha(raw):
|
||||
raise ValueError('Exact artifact/source inventory verification failed; no pass submitted')
|
||||
return client.call('tasks/' + task_id + '/reviews', {
|
||||
'evidenceId': evidence_id, 'passed': True,
|
||||
'findings': 'Reopened content-addressed bytes, recomputed exact deterministic inventory from API provenance and source bytes',
|
||||
'verification': {'artifactUri': evidence['artifactUri'], 'sha256': sha(raw),
|
||||
'method': 'independent local bytes + persisted provenance recomputation',
|
||||
'result': json.dumps({'verifiedAt': now(), 'sourceCount': len(expected['sources']), 'bytes': len(raw)})}}, key)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--api', default='http://127.0.0.1:3000/api/admin/hermes')
|
||||
p.add_argument('--allow-stage-https', action='store_true')
|
||||
p.add_argument('--store', required=True)
|
||||
p.add_argument('--key', help='Stable operation key; retain for retries')
|
||||
sub = p.add_subparsers(dest='command', required=True)
|
||||
i = sub.add_parser('ingest'); i.add_argument('manifest'); i.add_argument('--hosts', required=True)
|
||||
i = sub.add_parser('replay-ingest'); i.add_argument('request_digest')
|
||||
i = sub.add_parser('replay-evidence'); i.add_argument('checkpoint_digest')
|
||||
i = sub.add_parser('roadmap'); i.add_argument('manifest')
|
||||
i = sub.add_parser('fanout'); i.add_argument('roadmap_id')
|
||||
i = sub.add_parser('execute'); i.add_argument('--department', required=True)
|
||||
i = sub.add_parser('qa'); i.add_argument('task_id'); i.add_argument('evidence_id')
|
||||
i = sub.add_parser('effect'); i.add_argument('task_id'); i.add_argument('manifest')
|
||||
i = sub.add_parser('report'); i.add_argument('manifest')
|
||||
sub.add_parser('status')
|
||||
a = p.parse_args()
|
||||
if a.command != 'status' and not a.key:
|
||||
p.error('--key required for mutations')
|
||||
c = Client(a.api, 'CORPORATE_QA_API_KEY' if a.command == 'qa' else 'CORPORATE_API_KEY', a.allow_stage_https)
|
||||
s = Store(a.store)
|
||||
load = lambda: json.loads(Path(a.manifest).read_text())
|
||||
if a.command == 'ingest': result = ingest(c, s, load(), set(a.hosts.split(',')), a.key)
|
||||
elif a.command == 'replay-ingest': result = c.call('market-insights', json.loads(s.get(a.request_digest)), a.key)
|
||||
elif a.command == 'replay-evidence': result = c.replay(s, a.checkpoint_digest)
|
||||
elif a.command == 'roadmap':
|
||||
body = load()
|
||||
if body.get('isActive') is not False: raise ValueError('Manual roadmap must explicitly be inactive')
|
||||
result = c.call('monthly-roadmap', body, a.key)
|
||||
elif a.command == 'fanout':
|
||||
page = 1
|
||||
while True:
|
||||
collection = c.call('monthly-roadmap?limit=100&page=' + str(page))
|
||||
road = next((x for x in collection['data'] if x['id'] == a.roadmap_id), None)
|
||||
if road: break
|
||||
if not collection['meta']['hasNextPage']: raise ValueError('Roadmap not found')
|
||||
page += 1
|
||||
result = fanout(c, road, a.key)
|
||||
elif a.command == 'execute': result = execute(c, s, a.department, a.key)
|
||||
elif a.command == 'qa': result = review(c, s, a.task_id, a.evidence_id, a.key)
|
||||
elif a.command == 'effect': result = c.call('tasks/' + a.task_id + '/effects', load(), a.key)
|
||||
elif a.command == 'report': result = c.call('reports', load(), a.key)
|
||||
else: result = c.call('runtime')
|
||||
# Claim tokens may appear inside raw tasks: redact capability material from stdout.
|
||||
def clean(v):
|
||||
if isinstance(v, dict): return {k: clean(x) for k, x in v.items() if k != 'claimToken'}
|
||||
if isinstance(v, list): return [clean(x) for x in v]
|
||||
return v
|
||||
print(json.dumps(clean(result), ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except Exception as error:
|
||||
# Only known validation/status messages; transport exceptions can contain secrets.
|
||||
print(str(error) if isinstance(error, ValueError) and not isinstance(error, json.JSONDecodeError)
|
||||
else 'Operation failed; inspect local store/API safely (no credentials logged)', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in new issue
Block a user