Wire opt-in Stage runtime provisioning before isolated render; verify snapshot and Compose integration

This commit is contained in:
devops committed 2026-10-23 18:15:47 +00:00
1 parent bf84e5da43
commit 105ab07fbd
3 files changed
+205 -1

No files matched your search

+12 -1
View File
@@ -29,8 +29,16 @@ git cat-file -e "${DEPLOY_SHA}^{commit}"
git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop
git checkout --detach "$DEPLOY_SHA"
[ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ]
# Provision against the pristine approved SHA, before any tracked render edits.
source scripts/rollout-stage.sh
if [ "${STAGE_PROVISION_ENABLED:-false}" = true ]; then
stage_runtime_provision
elif [ -e /opt/services/canina/stage-runtime ]; then
echo 'ERROR: existing runtime requires explicit Stage CI opt-in on every deploy' >&2
exit 2
fi
# Keep operational files out of status/accidental commits in this disposable clone.
printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n' >> .git/info/exclude
printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n.stage-runtime.json\n.stage-runtime-merged.json\n' >> .git/info/exclude
# Runtime secrets/config must never enter the frontend Docker build context.
printf '\n.support-chat.env\n.revalidation.env\n.stage-*.json\n' >> .dockerignore
# Only these backend/frontend runtime env files are required. Do not copy the
@@ -92,6 +100,9 @@ for service in config['services'].values():
Path('.stage-compose.json').write_text(json.dumps(config))
Path('.stage-base.json').unlink()
PY
if [ "${STAGE_PROVISION_ENABLED:-false}" = true ]; then
stage_runtime_render
fi
# Read-only checks: missing external storage must fail, never provision blank DB.
docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null
docker network inspect traefik_public >/dev/null
+95
View File
@@ -1,9 +1,104 @@
#!/bin/bash
# Sourced by the CI/CD deploy entrypoint, never a standalone deploy command.
# Invoked only by the authenticated Stage CI entrypoint under its rollout lock.
stage_runtime_provision() {
[ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" = true ]
[ "${STAGE_PROVISION_ENABLED:-}" = true ]
export DEPLOY_SHA STAGE_PROVISION_APPROVED_SHA="$DEPLOY_SHA" CI=true
python3 - <<'PY'
import os, json, subprocess, importlib.util, resource, secrets, re, stat
from pathlib import Path
resource.setrlimit(resource.RLIMIT_AS, (1536*1024*1024, 1536*1024*1024))
resource.setrlimit(resource.RLIMIT_CPU, (60, 60))
resource.setrlimit(resource.RLIMIT_FSIZE, (64*1024*1024, 64*1024*1024))
def docker(*args):
return subprocess.check_output(['docker', *args], text=True, timeout=45)
spec=importlib.util.spec_from_file_location('provision','scripts/provision-stage-runtime.py')
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
m.ci_gate()
def inspect(name): return json.loads(docker('inspect',name))[0]
b=inspect('canino_backend_stage')
if not b['State'].get('Running') or b['State'].get('Pid',0) <= 0:
raise ValueError('Running backend required')
if b['Config']['User'] not in ('node','1000','1000:1000') or b['HostConfig'].get('UsernsMode') not in ('','default'):
raise ValueError('Unreviewed runtime identity/userns')
def mapped(kind):
for row in Path('/proc/%s/%s_map'%(b['State']['Pid'],kind)).read_text().splitlines():
inside,outside,count=map(int,row.split())
if inside <= 1000 < inside+count: return outside+1000-inside
raise ValueError('Unmapped runtime identity')
uid,gid=mapped('uid'),mapped('gid')
if min(uid,gid)<=0: raise ValueError('Root identity forbidden')
h=inspect('canina-hermes')
env=dict(e.split('=',1) for e in h['Config']['Env'] if '=' in e)
home=Path(env.get('HERMES_HOME',''))
if not home.is_absolute() or '..' in home.parts: raise ValueError('Explicit Hermes home required')
mounts=[v for v in h['Mounts'] if home==Path(v['Destination']) or Path(v['Destination']) in home.parents]
if not mounts: raise ValueError('Home mount required')
mount=max(mounts,key=lambda v:len(v['Destination']))
if mount.get('Type') not in ('bind','volume'):
raise ValueError('Filesystem home mount required')
m.safe_path(Path(mount['Source']))
db=Path(mount['Source'])/home.relative_to(mount['Destination'])/'state.db'
m.safe_path(db.parent)
for p in [db,Path(str(db)+'-wal'),Path(str(db)+'-shm')]:
if p.exists() or p.is_symlink():
st=p.lstat()
if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1:
raise ValueError('Unsafe SQLite source')
size=sum(p.stat().st_size for p in [db,Path(str(db)+'-wal')] if p.exists())
if not 0<size<=512*1024*1024: raise ValueError('Source exceeds 512 MiB cap')
root=Path('/opt/services/canina/stage-runtime')
query="""const {PrismaClient}=require('@prisma/client');const p=new PrismaClient();
(async()=>{try{const rows=await p.$transaction(async tx=>{
await tx.$executeRawUnsafe('SET TRANSACTION READ ONLY');
return tx.$queryRawUnsafe(\"SELECT id::text FROM users WHERE lower(replace(replace(role::text, '_', ''), '-', '')) IN ('admin','superadmin') ORDER BY id LIMIT 1001\");
});process.stdout.write(JSON.stringify(rows));}catch{process.exitCode=1;}finally{await p.$disconnect();}})();"""
rows=json.loads(docker('exec','-w','/app','canino_backend_stage','node','-e',query))
if not 1<=len(rows)<=1000 or any(not isinstance(r,dict) or set(r)!={'id'} or not isinstance(r['id'],str) or not re.fullmatch('[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}',r['id']) for r in rows):
raise ValueError('Verified admin subjects required')
m.provision(root,uid,gid)
# Leave the validated deny-all sentinel unchanged for repeat provisioning.
capture=os.environ['DEPLOY_SHA']+'-'+secrets.token_hex(8)
dest=root/'snapshots'/capture
m.export_snapshot(db,dest,'default',uid,gid)
grants=root/'snapshots'/(capture+'-grants.json')
m.write_private(grants,json.dumps({r['id']:['default'] for r in rows}),uid,gid)
Path('.stage-runtime.json').write_text(json.dumps({'snapshot':str(dest),'grants':str(grants)}))
print('Stage projected snapshot captured; existing admin grants verified; no executor started')
PY
}
stage_runtime_render() {
export STAGE_RUNTIME_SERVE_FILE="$TARGET_DIR/infra/hermes-runtime/serve.py"
export STAGE_RUNTIME_SNAPSHOT
STAGE_RUNTIME_SNAPSHOT=$(python3 -c 'import json;print(json.load(open(".stage-runtime.json"))["snapshot"])')
command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" \
-f "$TARGET_DIR/.stage-compose.json" -f infra/hermes-runtime/compose.stage-provision.yml \
--profile stage-runtime-inventory config --format json > .stage-runtime-merged.json
python3 - <<'PY'
import json
from pathlib import Path
p=Path('.stage-runtime-merged.json');c=json.loads(p.read_text())
b=c['services']['backend_stage']
b.setdefault('environment',{})['CANINA_RUNTIME_READ_ENABLED']='true'
b.setdefault('networks',{})['traefik_public']=None
b.setdefault('labels',{})['traefik.docker.network']='traefik_public'
c['secrets']['hermes_runtime_grants']['file']=json.loads(Path('.stage-runtime.json').read_text())['grants']
assert c['networks']['hermes_inventory_backend']['internal'] is True
assert c['networks']['traefik_public']['external'] is True
assert set(c['services'])=={'db_stage','redis_stage','frontend_stage','backend_stage','runtime-inventory'}
Path('.stage-compose.json').write_text(json.dumps(c));p.unlink()
PY
}
stage_rollout() {
# Maintenance is deliberate: neither old nor new app serves during DDL.
docker compose stop frontend_stage backend_stage
docker compose run --rm --no-deps --entrypoint ./node_modules/.bin/prisma backend_stage migrate deploy
if [ "${STAGE_PROVISION_ENABLED:-false}" = true ]; then
docker compose up -d --no-deps runtime-inventory
fi
docker compose up -d --no-deps backend_stage
if ! docker compose exec -T backend_stage node -e "$(< scripts/rollout-health.cjs)"; then
docker compose stop frontend_stage backend_stage
@@ -0,0 +1,98 @@
"""CI runtime integration: execute extracted deploy boundary, never live apply."""
import pathlib, subprocess, tempfile, unittest, os
ROOT = pathlib.Path(__file__).resolve().parents[2]
class WiringTests(unittest.TestCase):
def test_clean_provision_precedes_all_rendered_edits(self):
s=(ROOT/'scripts/deploy.sh').read_text()
self.assertIn('stage_runtime_provision',s)
self.assertLess(s.index('stage_runtime_provision'),s.index('>> .dockerignore'))
self.assertIn('CANINA_RUNTIME_READ_ENABLED', (ROOT/'scripts/rollout-stage.sh').read_text())
def test_inventory_started_before_backend(self):
with tempfile.TemporaryDirectory() as d:
p=pathlib.Path(d)/'docker'; p.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$TRACE"\n');p.chmod(0o755)
r=subprocess.run(['bash','-c','set -e; source scripts/rollout-stage.sh; STAGE_PROVISION_ENABLED=true; stage_rollout'],cwd=ROOT,env=dict(os.environ,PATH=d+':'+os.environ['PATH'],TRACE=d+'/trace'),capture_output=True,text=True)
self.assertEqual(r.returncode,0,r.stderr)
trace=pathlib.Path(d+'/trace').read_text()
self.assertIn('up -d --no-deps runtime-inventory',trace)
self.assertLess(trace.index('up -d --no-deps runtime-inventory'),trace.index('up -d --no-deps backend_stage'))
def test_provision_and_real_compose_render_repeat(self):
import json, shutil, sqlite3
with tempfile.TemporaryDirectory() as directory:
base=pathlib.Path(directory); repo=base/'repo';repo.mkdir()
shutil.copytree(ROOT/'scripts',repo/'scripts',ignore=shutil.ignore_patterns('__pycache__'))
shutil.copytree(ROOT/'infra',repo/'infra')
root=base/'runtime';home=base/'home';home.mkdir();tools=base/'tools';tools.mkdir()
db=sqlite3.connect(home/'state.db')
db.execute('CREATE TABLE sessions(source,started_at,input_tokens,output_tokens,estimated_cost_usd,title)')
db.execute("INSERT INTO sessions VALUES ('cli',1,2,3,0.1,'PRIVATE TITLE')");db.commit();db.close()
helper=repo/'scripts/rollout-stage.sh'
helper.write_text(helper.read_text().replace('/opt/services/canina/stage-runtime',str(root)))
def git(*args):return subprocess.check_output(['git',*args],cwd=repo,text=True).strip()
git('init','-q');git('config','user.email','fixture@example.invalid');git('config','user.name','Fixture');git('add','.');git('commit','-qm','fixture')
sha=git('rev-parse','HEAD')
subject='12345678-1234-1234-1234-123456789abc'
fixture={'backend':{'Config':{'User':'node'},'HostConfig':{'UsernsMode':''},'State':{'Running':True,'Pid':os.getpid()}},'hermes':{'Config':{'Env':['HERMES_HOME=/home/runtime']},'Mounts':[{'Type':'bind','Destination':'/home','Source':str(base)}]}}
# /home/runtime resolves to base/runtime; point home at the SQLite fixture.
fixture['hermes']['Mounts'][0]['Source']=str(home)
fixture['hermes']['Config']['Env']=['HERMES_HOME=/home']
(base/'inspect.json').write_text(json.dumps(fixture))
(tools/'docker').write_text('#!/usr/bin/env python3\nimport os,sys,json\na=sys.argv[1:]\nif a[0]=="compose": os.execv("/usr/bin/docker",["docker",*a])\nf=json.load(open(os.environ["INSPECT"]))\nif a[0]=="inspect": print(json.dumps([f["backend" if a[1]=="canino_backend_stage" else "hermes"]]))\nelif a[0]=="exec":\n assert "SET TRANSACTION READ ONLY" in a[-1] and "role::text" in a[-1]\n print(json.dumps([{ "id":os.environ["SUBJECT"]}]))\nelse: raise RuntimeError(a)\n');(tools/'docker').chmod(0o755)
env=dict(os.environ,PATH=str(tools)+':'+os.environ['PATH'],INSPECT=str(base/'inspect.json'),SUBJECT=subject,GITHUB_ACTIONS='true',STAGE_PROVISION_ENABLED='true',PYTHONDONTWRITEBYTECODE='1')
command='set -euo pipefail; source scripts/rollout-stage.sh; DEPLOY_SHA='+sha+'; unset STAGE_PROVISION_APPROVED_SHA; stage_runtime_provision'
# DEPLOY_SHA is deliberately shell-local: helper must export it.
captures=[]
for _ in range(2):
r=subprocess.run(['bash','-c',command],cwd=repo,env=env,capture_output=True,text=True)
self.assertEqual(r.returncode,0,r.stderr)
capture=json.loads((repo/'.stage-runtime.json').read_text());captures.append(capture)
self.assertEqual(json.loads(pathlib.Path(capture['grants']).read_text()),{subject:['default']})
self.assertNotIn('PRIVATE TITLE',(pathlib.Path(capture['snapshot'])/'inventory.json').read_text())
self.assertEqual(json.loads((root/'secrets/hermes_runtime_grants').read_text()),{})
if len(captures)==1:
(repo/'.git/info/exclude').write_text('.stage-runtime.json\n')
secrets_before={p.name:p.read_bytes() for p in (root/'secrets').iterdir()}
self.assertNotEqual(captures[0]['snapshot'],captures[1]['snapshot'])
self.assertEqual(secrets_before,{p.name:p.read_bytes() for p in (root/'secrets').iterdir()})
before=set((root/'snapshots').iterdir())
for invalid in ['', 'not-a-uuid', '------------------------------------']:
r=subprocess.run(['bash','-c',command],cwd=repo,env=dict(env,SUBJECT=invalid),capture_output=True,text=True)
self.assertNotEqual(r.returncode,0)
self.assertIn('Verified admin subjects required',r.stderr)
self.assertEqual(before,set((root/'snapshots').iterdir()))
(home/'state.db-wal').symlink_to(home/'state.db')
r=subprocess.run(['bash','-c',command],cwd=repo,env=env,capture_output=True,text=True)
self.assertNotEqual(r.returncode,0)
self.assertIn('Unsafe SQLite source',r.stderr)
self.assertEqual(before,set((root/'snapshots').iterdir()))
(home/'state.db-wal').unlink()
# Real Compose CLI normalization and merge; no daemon/socket required.
base_config={'services':{n:{'image':'fixture:never-start','networks':{'traefik_public':None}} for n in ['db_stage','redis_stage','backend_stage','frontend_stage']},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}}
(repo/'.stage-compose.json').write_text(json.dumps(base_config))
r=subprocess.run(['bash','-c','set -euo pipefail; source scripts/rollout-stage.sh; TARGET_DIR="$PWD"; stage_runtime_render'],cwd=repo,env=env,capture_output=True,text=True)
self.assertEqual(r.returncode,0,r.stderr)
c=json.loads((repo/'.stage-compose.json').read_text());b=c['services']['backend_stage'];inventory=c['services']['runtime-inventory']
self.assertEqual(set(b['networks']),{'traefik_public','hermes_inventory_backend'})
self.assertEqual(b['labels']['traefik.docker.network'],'traefik_public')
self.assertEqual(set(inventory['networks']),{'hermes_inventory_backend'})
self.assertTrue(c['networks']['hermes_inventory_backend']['internal'])
self.assertEqual(b['environment']['CANINA_RUNTIME_READ_ENABLED'],'true')
self.assertEqual(c['secrets']['hermes_runtime_grants']['file'],captures[-1]['grants'])
self.assertEqual(inventory['mem_limit'],'1073741824')
self.assertTrue(all(m.get('read_only') for m in inventory['volumes']))
self.assertFalse(inventory.get('ports'))
self.assertEqual(inventory['user'],'1000:1000')
def test_base_bind_allowlist_rejects_before_runtime_merge(self):
import json
script=(ROOT/'scripts/deploy.sh').read_text()
validation=script.split("config = json.loads(Path('.stage-base.json').read_text())",1)[1].split("\nPY",1)[0]
code="import json\nfrom pathlib import Path\nconfig=json.loads(Path('.stage-base.json').read_text())"+validation
config={'services':{'db_stage':{},'redis_stage':{},'backend_stage':{'container_name':'canino_backend_stage','volumes':[{'type':'volume','source':'canina_stage_uploads','target':'/app/uploads'},{'type':'bind','source':'/etc','target':'/stolen'}]},'frontend_stage':{'container_name':'canino_frontend_stage'}},'volumes':{n:{'name':'stage_'+n} for n in ['canina_stage_db','canina_stage_uploads']},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}}
with tempfile.TemporaryDirectory() as directory:
p=pathlib.Path(directory);(p/'.stage-base.json').write_text(json.dumps(config))
result=subprocess.run(['python3','-c',code],cwd=p,capture_output=True,text=True)
self.assertNotEqual(result.returncode,0)
self.assertIn('unexpected Stage bind mount',result.stderr)
self.assertFalse((p/'.stage-compose.json').exists())
self.assertLess(script.index('unexpected Stage bind mount'),script.index(' stage_runtime_render'))