feat(editorial): wire serialized Stage runtime install and PMO proposal client
This commit is contained in:
1 parent
8869ce2583
commit
f7cf2c8f5d
8 files changed
+387
-4
No files matched your search
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Serialized Stage CI only; stream reviewed bundle into actual Hermes identity."""
|
||||
import base64
|
||||
import copy
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
PROFILES=('pmo','canina-competitor-worker','canina-seo-worker','canina-content-worker','canina-vet-review-worker','canina-qa-worker')
|
||||
FILES={'bundle.json','installer.py','client.py'} | {'payload/editorial-v4/status.json'} | {f'payload/profiles/{p}/skills/canina-editorial-v4/{f}' for p in PROFILES for f in ('SKILL.md','references/policy.md','references/role.json')}
|
||||
|
||||
def safe(path):
|
||||
if not path.is_absolute() or '..' in path.parts: raise ValueError('Canonical path required')
|
||||
for p in (path,*path.parents):
|
||||
if p.is_symlink(): raise ValueError('Symlink refused')
|
||||
return path
|
||||
|
||||
def private(path):
|
||||
safe(path)
|
||||
s=path.lstat()
|
||||
if not stat.S_ISREG(s.st_mode) or s.st_nlink!=1 or s.st_uid!=os.getuid() or stat.S_IMODE(s.st_mode)!=0o400 or not 1<=s.st_size<=1048576:
|
||||
raise ValueError('Private runtime file required')
|
||||
return path.read_bytes()
|
||||
|
||||
def atomic(path, data, uid=None, gid=None):
|
||||
safe(path)
|
||||
fd,name=tempfile.mkstemp(prefix='.editorial-',dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd,'wb') as f:
|
||||
f.write(data);f.flush();os.fsync(f.fileno())
|
||||
if uid is not None: os.fchown(f.fileno(),uid,gid)
|
||||
os.chmod(name,0o400);os.replace(name,path)
|
||||
finally:
|
||||
if os.path.exists(name):os.unlink(name)
|
||||
|
||||
def package(root, sha):
|
||||
if not re.fullmatch('[0-9a-f]{40}',sha):raise ValueError('Full deployment SHA required')
|
||||
result={}
|
||||
for name in FILES:
|
||||
p=safe(root/name)
|
||||
if p.stat().st_size>65536:raise ValueError('Package exceeds bound')
|
||||
result[name]=base64.b64encode(p.read_bytes()).decode()
|
||||
return {'commitSha':sha,'files':result}
|
||||
|
||||
def install(home, payload):
|
||||
safe(home)
|
||||
if set(payload)!={'commitSha','files'} or set(payload['files'])!=FILES or not re.fullmatch('[0-9a-f]{40}',payload['commitSha']):
|
||||
raise ValueError('Unreviewed package')
|
||||
# Fixed private package path; no archive extraction or arbitrary destinations.
|
||||
root=safe(home/'editorial-v4/runtime-package')
|
||||
root.mkdir(mode=0o700,parents=True,exist_ok=True)
|
||||
if root.stat().st_uid!=os.getuid() or stat.S_IMODE(root.stat().st_mode)!=0o700:raise ValueError('Private package required')
|
||||
for name,value in payload['files'].items():
|
||||
path=safe(root/name);path.parent.mkdir(mode=0o700,parents=True,exist_ok=True)
|
||||
data=base64.b64decode(value,validate=True)
|
||||
if len(data)>65536:raise ValueError('Package exceeds bound')
|
||||
atomic(path,data)
|
||||
spec=importlib.util.spec_from_file_location('editorial_installer',root/'installer.py')
|
||||
installer=importlib.util.module_from_spec(spec);spec.loader.exec_module(installer)
|
||||
installer.execute('install',home)
|
||||
report=installer.execute('verify',home)
|
||||
# Confirm all six skill discovery paths, without activating any scheduler.
|
||||
for profile in PROFILES:
|
||||
skill=home/f'profiles/{profile}/skills/canina-editorial-v4/SKILL.md'
|
||||
if not skill.is_file() or 'name: canina-editorial-v4' not in skill.read_text():raise ValueError('Skill discovery failed')
|
||||
credentials=safe(home/'credentials');credentials.mkdir(mode=0o700,exist_ok=True)
|
||||
st=credentials.stat()
|
||||
if st.st_uid!=os.getuid() or stat.S_IMODE(st.st_mode) not in (0o700,0o755):raise ValueError('Runtime-owned non-writable credentials directory required')
|
||||
token=safe(credentials/'editorial-pmo-token')
|
||||
if token.exists():
|
||||
value=private(token).decode('ascii')
|
||||
if not re.fullmatch('[A-Za-z0-9_-]{64}',value):raise ValueError('Invalid PMO credential')
|
||||
else:
|
||||
value=secrets.token_urlsafe(48);atomic(token,value.encode())
|
||||
atomic(home/'editorial-v4/deployment.json',json.dumps({'commitSha':payload['commitSha']}).encode())
|
||||
return {'files':report['files'],'profiles':list(PROFILES),'credentialSha256':hashlib.sha256(value.encode()).hexdigest()}
|
||||
|
||||
def add_grant(config,digest):
|
||||
if set(config)!={'grants','workerIds','stageCiIds','artifactBaseUrl'} or not re.fullmatch('[0-9a-f]{64}',digest):raise ValueError('Invalid grants')
|
||||
out=copy.deepcopy(config)
|
||||
desired={'id':'pmo','role':'WORKFLOW_WORKER','credentialSha256':digest}
|
||||
existing=[g for g in out['grants'] if g['id']=='pmo']
|
||||
if existing:
|
||||
if existing!=[desired] or out['workerIds'].count('pmo')!=1 or 'pmo' in out['stageCiIds']:raise ValueError('Conflicting PMO grant')
|
||||
return out
|
||||
if any(g['credentialSha256']==digest for g in out['grants']) or 'pmo' in out['workerIds']+out['stageCiIds'] or len(out['grants'])>=32:raise ValueError('Ambiguous PMO grant')
|
||||
out['grants'].append(desired);out['workerIds'].append('pmo')
|
||||
return out
|
||||
|
||||
def provision(sha, uid, gid):
|
||||
gate_path=Path(__file__).resolve().with_name('provision-stage-runtime.py')
|
||||
spec=importlib.util.spec_from_file_location('stage_gate',gate_path)
|
||||
gate=importlib.util.module_from_spec(spec);spec.loader.exec_module(gate)
|
||||
gate.ci_gate()
|
||||
if sha!=os.environ.get('STAGE_PROVISION_APPROVED_SHA'):raise ValueError('Deployment SHA mismatch')
|
||||
def docker(*args, data=None):
|
||||
return subprocess.check_output(['docker',*args],input=data,text=True,timeout=60)
|
||||
h=json.loads(docker('inspect','canina-hermes'))[0]
|
||||
if not h['State']['Running'] or h['HostConfig'].get('UsernsMode') not in ('','default'):raise ValueError('Reviewed running Hermes required')
|
||||
env=dict(v.split('=',1) for v in h['Config']['Env'] if '=' in v)
|
||||
home=Path(env.get('HERMES_HOME',''));safe(home)
|
||||
if not any(home==Path(v['Destination']) or Path(v['Destination']) in home.parents for v in h['Mounts'] if v['Type'] in ('bind','volume')):raise ValueError('Persistent home required')
|
||||
# No --user override: Docker executes as configured runtime identity, not host UID.
|
||||
source=Path(__file__).read_text()
|
||||
payload=package(Path(__file__).resolve().parents[1]/'infra/hermes-editorial',sha)
|
||||
code="import json,sys; p=json.load(sys.stdin); ns={'__name__':'editorial_stream'}; exec(compile(p['source'],'<ci-editorial>','exec'),ns); print(json.dumps(ns['install'](ns['Path'](p['home']),p['payload'])))"
|
||||
result=json.loads(docker('exec','-i','canina-hermes','python3','-B','-c',code,data=json.dumps({'source':source,'home':str(home),'payload':payload})))
|
||||
for profile in PROFILES:
|
||||
discovered=docker('exec','-e','HERMES_HOME='+str(home/'profiles'/profile),'canina-hermes','/opt/hermes/bin/hermes','skills','list')
|
||||
if 'canina-editorial-v4' not in discovered:raise ValueError('Hermes CLI skill discovery failed')
|
||||
path=Path('/opt/services/canina/stage-runtime/secrets/prepare_worker_grants');safe(path)
|
||||
st=path.lstat()
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_uid!=uid or st.st_gid!=gid or st.st_nlink!=1 or stat.S_IMODE(st.st_mode)!=0o400 or not 1<=st.st_size<=16384:raise ValueError('Unsafe backend grant file')
|
||||
config=json.loads(path.read_bytes());updated=add_grant(config,result['credentialSha256'])
|
||||
if updated!=config:atomic(path,json.dumps(updated).encode(),uid,gid)
|
||||
print('Editorial: 19 files and six profile skill paths verified; dedicated PMO grant preserved; no workflow activated')
|
||||
@@ -89,7 +89,7 @@ def validate(root, uid, gid):
|
||||
if not re.fullmatch('[A-Za-z0-9_-]{64}', (root / 'secrets' / name).read_text()):
|
||||
raise ValueError('Invalid dedicated credential')
|
||||
config = json.loads((root / 'secrets/prepare_worker_grants').read_text())
|
||||
if set(config) != {'grants', 'workerIds', 'stageCiIds', 'artifactBaseUrl'} or len(config['grants']) != 2:
|
||||
if set(config) != {'grants', 'workerIds', 'stageCiIds', 'artifactBaseUrl'} or len(config['grants']) not in (2, 3):
|
||||
raise ValueError('Unexpected worker config')
|
||||
for index, name in enumerate(['prepare_worker_token', 'stage_ci_token']):
|
||||
grant = config['grants'][index]
|
||||
@@ -99,7 +99,12 @@ def validate(root, uid, gid):
|
||||
raise ValueError('Invalid service credential')
|
||||
if hashlib.sha256((root / 'credentials' / name).read_bytes()).hexdigest() != grant['credentialSha256']:
|
||||
raise ValueError('Credential digest mismatch')
|
||||
if config['workerIds'] != ['stage-prepare-worker'] or config['stageCiIds'] != ['stage-evidence-ci'] or config['artifactBaseUrl'] != BASE_URL or [g['role'] for g in config['grants']] != ['WORKFLOW_WORKER', 'STAGE_CI']:
|
||||
editorial = len(config['grants']) == 3
|
||||
if editorial:
|
||||
grant = config['grants'][2]
|
||||
if set(grant) != {'id', 'role', 'credentialSha256'} or grant['id'] != 'pmo' or grant['role'] != 'WORKFLOW_WORKER' or not re.fullmatch('[0-9a-f]{64}', grant['credentialSha256']) or grant['credentialSha256'] in [g['credentialSha256'] for g in config['grants'][:2]]:
|
||||
raise ValueError('Unexpected PMO grant')
|
||||
if config['workerIds'] != ['stage-prepare-worker'] + (['pmo'] if editorial else []) or config['stageCiIds'] != ['stage-evidence-ci'] or config['artifactBaseUrl'] != BASE_URL or [g['role'] for g in config['grants'][:2]] != ['WORKFLOW_WORKER', 'STAGE_CI']:
|
||||
raise ValueError('Unexpected grants')
|
||||
if json.loads((root / 'secrets/hermes_runtime_grants').read_text()) != {}:
|
||||
raise ValueError('Runtime grants require separate verified-ID review')
|
||||
|
||||
@@ -5,6 +5,8 @@ stage_runtime_provision() {
|
||||
[ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" = true ]
|
||||
[ "${STAGE_PROVISION_ENABLED:-}" = true ]
|
||||
export DEPLOY_SHA STAGE_PROVISION_APPROVED_SHA="$DEPLOY_SHA" CI=true
|
||||
PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s infra/hermes-editorial/tests
|
||||
PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s scripts -p test_editorial_provision.py
|
||||
PYTHONDONTWRITEBYTECODE=1 python3 - <<'PY'
|
||||
import os, json, subprocess, importlib.util, resource, secrets, re, stat
|
||||
from pathlib import Path
|
||||
@@ -63,6 +65,9 @@ rows=json.loads(docker('exec','-w','/app','canino_backend_stage','node','-e',que
|
||||
if not 1<=len(rows)<=1000 or any(not isinstance(r,dict) or set(r)!={'id'} or not isinstance(r['id'],str) or not re.fullmatch('[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}',r['id']) for r in rows):
|
||||
raise ValueError('Verified admin subjects required')
|
||||
m.provision(root,uid,gid)
|
||||
spec=importlib.util.spec_from_file_location('editorial','scripts/provision-editorial-runtime.py')
|
||||
editorial=importlib.util.module_from_spec(spec);spec.loader.exec_module(editorial)
|
||||
editorial.provision(os.environ['DEPLOY_SHA'],uid,gid)
|
||||
# Leave the validated deny-all sentinel unchanged for repeat provisioning.
|
||||
capture=os.environ['DEPLOY_SHA']+'-'+secrets.token_hex(8)
|
||||
dest=root/'snapshots'/capture
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT=Path(__file__).resolve().parent
|
||||
BUNDLE = ROOT/'hermes-editorial' if (ROOT/'hermes-editorial').exists() else ROOT.parent/'infra/hermes-editorial'
|
||||
|
||||
def load(name, path):
|
||||
spec=importlib.util.spec_from_file_location(name,path)
|
||||
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m
|
||||
|
||||
class ProvisionTests(unittest.TestCase):
|
||||
def test_runtime_install_and_credential_are_idempotent_and_private(self):
|
||||
self.assertTrue((ROOT/'provision-editorial-runtime.py').exists(), 'CI editorial wiring missing')
|
||||
m=load('editorial',ROOT/'provision-editorial-runtime.py')
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
home=Path(t); sentinel=home/'config.yaml';sentinel.write_text('unchanged')
|
||||
payload=m.package(BUNDLE,'a'*40)
|
||||
a=m.install(home,payload);b=m.install(home,payload)
|
||||
self.assertEqual(a,b);self.assertEqual(sentinel.read_text(),'unchanged')
|
||||
self.assertEqual((home/'credentials/editorial-pmo-token').stat().st_mode & 0o777,0o400)
|
||||
self.assertEqual(a['files'],19)
|
||||
grants={'grants':[{'id':'ci','role':'STAGE_CI','credentialSha256':'c'*64},{'id':'worker','role':'WORKFLOW_WORKER','credentialSha256':'d'*64}], 'workerIds':['worker'],'stageCiIds':['ci'],'artifactBaseUrl':'https://stageapi.canina.ir/api/internal/prepare-worker/artifacts'}
|
||||
updated=m.add_grant(grants,a['credentialSha256'])
|
||||
self.assertEqual(updated['grants'][:2],grants['grants']);self.assertEqual(len(updated['grants']),3)
|
||||
self.assertEqual(m.add_grant(updated,a['credentialSha256']),updated)
|
||||
with self.assertRaises(ValueError):m.add_grant(updated,'e'*64)
|
||||
self.assertEqual(grants['workerIds'],['worker'])
|
||||
(home/'credentials/editorial-pmo-token').chmod(0o644)
|
||||
with self.assertRaises(ValueError):m.install(home,payload)
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -28,16 +28,18 @@ class WiringTests(unittest.TestCase):
|
||||
db.execute("INSERT INTO sessions VALUES ('cli',1,2,3,0.1,'PRIVATE TITLE')");db.commit();db.close()
|
||||
helper=repo/'scripts/rollout-stage.sh'
|
||||
helper.write_text(helper.read_text().replace('/opt/services/canina/stage-runtime',str(root)))
|
||||
editorial=repo/'scripts/provision-editorial-runtime.py'
|
||||
editorial.write_text(editorial.read_text().replace('/opt/services/canina/stage-runtime',str(root)))
|
||||
def git(*args):return subprocess.check_output(['git',*args],cwd=repo,text=True).strip()
|
||||
git('init','-q');git('config','user.email','fixture@example.invalid');git('config','user.name','Fixture');git('add','.');git('commit','-qm','fixture')
|
||||
sha=git('rev-parse','HEAD')
|
||||
subject='12345678-1234-1234-1234-123456789abc'
|
||||
fixture={'backend':{'Config':{'User':'node'},'HostConfig':{'UsernsMode':''},'State':{'Running':True,'Pid':os.getpid()}},'hermes':{'Config':{'Env':['HERMES_HOME=/home/runtime']},'Mounts':[{'Type':'bind','Destination':'/home','Source':str(base)}]}}
|
||||
fixture={'backend':{'Config':{'User':'node'},'HostConfig':{'UsernsMode':''},'State':{'Running':True,'Pid':os.getpid()}},'hermes':{'State':{'Running':True},'HostConfig':{'UsernsMode':''},'Config':{'Env':['HERMES_HOME=/home/runtime']},'Mounts':[{'Type':'bind','Destination':'/home','Source':str(base)}]}}
|
||||
# /home/runtime resolves to base/runtime; point home at the SQLite fixture.
|
||||
fixture['hermes']['Mounts'][0]['Source']=str(home)
|
||||
fixture['hermes']['Config']['Env']=['HERMES_HOME=/home']
|
||||
(base/'inspect.json').write_text(json.dumps(fixture))
|
||||
(tools/'docker').write_text('#!/usr/bin/env python3\nimport os,sys,json\na=sys.argv[1:]\nif a[0]=="compose": os.execv("/usr/bin/docker",["docker",*a])\nf=json.load(open(os.environ["INSPECT"]))\nif a[0]=="inspect": print(json.dumps([f["backend" if a[1]=="canino_backend_stage" else "hermes"]]))\nelif a[0]=="exec":\n assert "SET TRANSACTION READ ONLY" in a[-1] and "role::text" in a[-1]\n print(json.dumps([{ "id":os.environ["SUBJECT"]}]))\nelse: raise RuntimeError(a)\n');(tools/'docker').chmod(0o755)
|
||||
(tools/'docker').write_text('#!/usr/bin/env python3\nimport os,sys,json,subprocess\na=sys.argv[1:]\nif a[0]=="compose": os.execv("/usr/bin/docker",["docker",*a])\nf=json.load(open(os.environ["INSPECT"]))\nif a[0]=="inspect": print(json.dumps([f["backend" if a[1]=="canino_backend_stage" else "hermes"]]))\nelif a[0]=="exec" and "-i" in a:\n p=json.load(sys.stdin); p["home"]=f["hermes"]["Mounts"][0]["Source"]; subprocess.run([sys.executable,"-B","-c",a[-1]],input=json.dumps(p),text=True,check=True)\nelif a[0]=="exec" and "skills" in a:\n print("canina-editorial-v4")\nelif a[0]=="exec":\n assert "SET TRANSACTION READ ONLY" in a[-1] and "role::text" in a[-1]\n print(json.dumps([{ "id":os.environ["SUBJECT"]}]))\nelse: raise RuntimeError(a)\n');(tools/'docker').chmod(0o755)
|
||||
env=dict(os.environ,PATH=str(tools)+':'+os.environ['PATH'],INSPECT=str(base/'inspect.json'),SUBJECT=subject,GITHUB_ACTIONS='true',STAGE_PROVISION_ENABLED='true',PYTHONDONTWRITEBYTECODE='1')
|
||||
command='set -euo pipefail; source scripts/rollout-stage.sh; DEPLOY_SHA='+sha+'; unset STAGE_PROVISION_APPROVED_SHA; stage_runtime_provision'
|
||||
# DEPLOY_SHA is deliberately shell-local: helper must export it.
|
||||
|
||||
Reference in new issue
Block a user