Wire opt-in Stage runtime provisioning before isolated render; verify snapshot and Compose integration
This commit is contained in:
1 parent
bf84e5da43
commit
105ab07fbd
3 files changed
+205
-1
No files matched your search
+12
-1
@@ -29,8 +29,16 @@ git cat-file -e "${DEPLOY_SHA}^{commit}"
|
||||
git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop
|
||||
git checkout --detach "$DEPLOY_SHA"
|
||||
[ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ]
|
||||
# Provision against the pristine approved SHA, before any tracked render edits.
|
||||
source scripts/rollout-stage.sh
|
||||
if [ "${STAGE_PROVISION_ENABLED:-false}" = true ]; then
|
||||
stage_runtime_provision
|
||||
elif [ -e /opt/services/canina/stage-runtime ]; then
|
||||
echo 'ERROR: existing runtime requires explicit Stage CI opt-in on every deploy' >&2
|
||||
exit 2
|
||||
fi
|
||||
# Keep operational files out of status/accidental commits in this disposable clone.
|
||||
printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n' >> .git/info/exclude
|
||||
printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n.stage-runtime.json\n.stage-runtime-merged.json\n' >> .git/info/exclude
|
||||
# Runtime secrets/config must never enter the frontend Docker build context.
|
||||
printf '\n.support-chat.env\n.revalidation.env\n.stage-*.json\n' >> .dockerignore
|
||||
# Only these backend/frontend runtime env files are required. Do not copy the
|
||||
@@ -92,6 +100,9 @@ for service in config['services'].values():
|
||||
Path('.stage-compose.json').write_text(json.dumps(config))
|
||||
Path('.stage-base.json').unlink()
|
||||
PY
|
||||
if [ "${STAGE_PROVISION_ENABLED:-false}" = true ]; then
|
||||
stage_runtime_render
|
||||
fi
|
||||
# Read-only checks: missing external storage must fail, never provision blank DB.
|
||||
docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null
|
||||
docker network inspect traefik_public >/dev/null
|
||||
|
||||
@@ -1,9 +1,104 @@
|
||||
#!/bin/bash
|
||||
# Sourced by the CI/CD deploy entrypoint, never a standalone deploy command.
|
||||
# Invoked only by the authenticated Stage CI entrypoint under its rollout lock.
|
||||
stage_runtime_provision() {
|
||||
[ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" = true ]
|
||||
[ "${STAGE_PROVISION_ENABLED:-}" = true ]
|
||||
export DEPLOY_SHA STAGE_PROVISION_APPROVED_SHA="$DEPLOY_SHA" CI=true
|
||||
python3 - <<'PY'
|
||||
import os, json, subprocess, importlib.util, resource, secrets, re, stat
|
||||
from pathlib import Path
|
||||
resource.setrlimit(resource.RLIMIT_AS, (1536*1024*1024, 1536*1024*1024))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (60, 60))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (64*1024*1024, 64*1024*1024))
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker', *args], text=True, timeout=45)
|
||||
spec=importlib.util.spec_from_file_location('provision','scripts/provision-stage-runtime.py')
|
||||
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
m.ci_gate()
|
||||
def inspect(name): return json.loads(docker('inspect',name))[0]
|
||||
b=inspect('canino_backend_stage')
|
||||
if not b['State'].get('Running') or b['State'].get('Pid',0) <= 0:
|
||||
raise ValueError('Running backend required')
|
||||
if b['Config']['User'] not in ('node','1000','1000:1000') or b['HostConfig'].get('UsernsMode') not in ('','default'):
|
||||
raise ValueError('Unreviewed runtime identity/userns')
|
||||
def mapped(kind):
|
||||
for row in Path('/proc/%s/%s_map'%(b['State']['Pid'],kind)).read_text().splitlines():
|
||||
inside,outside,count=map(int,row.split())
|
||||
if inside <= 1000 < inside+count: return outside+1000-inside
|
||||
raise ValueError('Unmapped runtime identity')
|
||||
uid,gid=mapped('uid'),mapped('gid')
|
||||
if min(uid,gid)<=0: raise ValueError('Root identity forbidden')
|
||||
h=inspect('canina-hermes')
|
||||
env=dict(e.split('=',1) for e in h['Config']['Env'] if '=' in e)
|
||||
home=Path(env.get('HERMES_HOME',''))
|
||||
if not home.is_absolute() or '..' in home.parts: raise ValueError('Explicit Hermes home required')
|
||||
mounts=[v for v in h['Mounts'] if home==Path(v['Destination']) or Path(v['Destination']) in home.parents]
|
||||
if not mounts: raise ValueError('Home mount required')
|
||||
mount=max(mounts,key=lambda v:len(v['Destination']))
|
||||
if mount.get('Type') not in ('bind','volume'):
|
||||
raise ValueError('Filesystem home mount required')
|
||||
m.safe_path(Path(mount['Source']))
|
||||
db=Path(mount['Source'])/home.relative_to(mount['Destination'])/'state.db'
|
||||
m.safe_path(db.parent)
|
||||
for p in [db,Path(str(db)+'-wal'),Path(str(db)+'-shm')]:
|
||||
if p.exists() or p.is_symlink():
|
||||
st=p.lstat()
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1:
|
||||
raise ValueError('Unsafe SQLite source')
|
||||
size=sum(p.stat().st_size for p in [db,Path(str(db)+'-wal')] if p.exists())
|
||||
if not 0<size<=512*1024*1024: raise ValueError('Source exceeds 512 MiB cap')
|
||||
root=Path('/opt/services/canina/stage-runtime')
|
||||
query="""const {PrismaClient}=require('@prisma/client');const p=new PrismaClient();
|
||||
(async()=>{try{const rows=await p.$transaction(async tx=>{
|
||||
await tx.$executeRawUnsafe('SET TRANSACTION READ ONLY');
|
||||
return tx.$queryRawUnsafe(\"SELECT id::text FROM users WHERE lower(replace(replace(role::text, '_', ''), '-', '')) IN ('admin','superadmin') ORDER BY id LIMIT 1001\");
|
||||
});process.stdout.write(JSON.stringify(rows));}catch{process.exitCode=1;}finally{await p.$disconnect();}})();"""
|
||||
rows=json.loads(docker('exec','-w','/app','canino_backend_stage','node','-e',query))
|
||||
if not 1<=len(rows)<=1000 or any(not isinstance(r,dict) or set(r)!={'id'} or not isinstance(r['id'],str) or not re.fullmatch('[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}',r['id']) for r in rows):
|
||||
raise ValueError('Verified admin subjects required')
|
||||
m.provision(root,uid,gid)
|
||||
# Leave the validated deny-all sentinel unchanged for repeat provisioning.
|
||||
capture=os.environ['DEPLOY_SHA']+'-'+secrets.token_hex(8)
|
||||
dest=root/'snapshots'/capture
|
||||
m.export_snapshot(db,dest,'default',uid,gid)
|
||||
grants=root/'snapshots'/(capture+'-grants.json')
|
||||
m.write_private(grants,json.dumps({r['id']:['default'] for r in rows}),uid,gid)
|
||||
Path('.stage-runtime.json').write_text(json.dumps({'snapshot':str(dest),'grants':str(grants)}))
|
||||
print('Stage projected snapshot captured; existing admin grants verified; no executor started')
|
||||
PY
|
||||
}
|
||||
|
||||
stage_runtime_render() {
|
||||
export STAGE_RUNTIME_SERVE_FILE="$TARGET_DIR/infra/hermes-runtime/serve.py"
|
||||
export STAGE_RUNTIME_SNAPSHOT
|
||||
STAGE_RUNTIME_SNAPSHOT=$(python3 -c 'import json;print(json.load(open(".stage-runtime.json"))["snapshot"])')
|
||||
command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" \
|
||||
-f "$TARGET_DIR/.stage-compose.json" -f infra/hermes-runtime/compose.stage-provision.yml \
|
||||
--profile stage-runtime-inventory config --format json > .stage-runtime-merged.json
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
p=Path('.stage-runtime-merged.json');c=json.loads(p.read_text())
|
||||
b=c['services']['backend_stage']
|
||||
b.setdefault('environment',{})['CANINA_RUNTIME_READ_ENABLED']='true'
|
||||
b.setdefault('networks',{})['traefik_public']=None
|
||||
b.setdefault('labels',{})['traefik.docker.network']='traefik_public'
|
||||
c['secrets']['hermes_runtime_grants']['file']=json.loads(Path('.stage-runtime.json').read_text())['grants']
|
||||
assert c['networks']['hermes_inventory_backend']['internal'] is True
|
||||
assert c['networks']['traefik_public']['external'] is True
|
||||
assert set(c['services'])=={'db_stage','redis_stage','frontend_stage','backend_stage','runtime-inventory'}
|
||||
Path('.stage-compose.json').write_text(json.dumps(c));p.unlink()
|
||||
PY
|
||||
}
|
||||
|
||||
stage_rollout() {
|
||||
# Maintenance is deliberate: neither old nor new app serves during DDL.
|
||||
docker compose stop frontend_stage backend_stage
|
||||
docker compose run --rm --no-deps --entrypoint ./node_modules/.bin/prisma backend_stage migrate deploy
|
||||
if [ "${STAGE_PROVISION_ENABLED:-false}" = true ]; then
|
||||
docker compose up -d --no-deps runtime-inventory
|
||||
fi
|
||||
docker compose up -d --no-deps backend_stage
|
||||
if ! docker compose exec -T backend_stage node -e "$(< scripts/rollout-health.cjs)"; then
|
||||
docker compose stop frontend_stage backend_stage
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
"""CI runtime integration: execute extracted deploy boundary, never live apply."""
|
||||
import pathlib, subprocess, tempfile, unittest, os
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
class WiringTests(unittest.TestCase):
|
||||
def test_clean_provision_precedes_all_rendered_edits(self):
|
||||
s=(ROOT/'scripts/deploy.sh').read_text()
|
||||
self.assertIn('stage_runtime_provision',s)
|
||||
self.assertLess(s.index('stage_runtime_provision'),s.index('>> .dockerignore'))
|
||||
self.assertIn('CANINA_RUNTIME_READ_ENABLED', (ROOT/'scripts/rollout-stage.sh').read_text())
|
||||
def test_inventory_started_before_backend(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=pathlib.Path(d)/'docker'; p.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$TRACE"\n');p.chmod(0o755)
|
||||
r=subprocess.run(['bash','-c','set -e; source scripts/rollout-stage.sh; STAGE_PROVISION_ENABLED=true; stage_rollout'],cwd=ROOT,env=dict(os.environ,PATH=d+':'+os.environ['PATH'],TRACE=d+'/trace'),capture_output=True,text=True)
|
||||
self.assertEqual(r.returncode,0,r.stderr)
|
||||
trace=pathlib.Path(d+'/trace').read_text()
|
||||
self.assertIn('up -d --no-deps runtime-inventory',trace)
|
||||
self.assertLess(trace.index('up -d --no-deps runtime-inventory'),trace.index('up -d --no-deps backend_stage'))
|
||||
|
||||
def test_provision_and_real_compose_render_repeat(self):
|
||||
import json, shutil, sqlite3
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
base=pathlib.Path(directory); repo=base/'repo';repo.mkdir()
|
||||
shutil.copytree(ROOT/'scripts',repo/'scripts',ignore=shutil.ignore_patterns('__pycache__'))
|
||||
shutil.copytree(ROOT/'infra',repo/'infra')
|
||||
root=base/'runtime';home=base/'home';home.mkdir();tools=base/'tools';tools.mkdir()
|
||||
db=sqlite3.connect(home/'state.db')
|
||||
db.execute('CREATE TABLE sessions(source,started_at,input_tokens,output_tokens,estimated_cost_usd,title)')
|
||||
db.execute("INSERT INTO sessions VALUES ('cli',1,2,3,0.1,'PRIVATE TITLE')");db.commit();db.close()
|
||||
helper=repo/'scripts/rollout-stage.sh'
|
||||
helper.write_text(helper.read_text().replace('/opt/services/canina/stage-runtime',str(root)))
|
||||
def git(*args):return subprocess.check_output(['git',*args],cwd=repo,text=True).strip()
|
||||
git('init','-q');git('config','user.email','fixture@example.invalid');git('config','user.name','Fixture');git('add','.');git('commit','-qm','fixture')
|
||||
sha=git('rev-parse','HEAD')
|
||||
subject='12345678-1234-1234-1234-123456789abc'
|
||||
fixture={'backend':{'Config':{'User':'node'},'HostConfig':{'UsernsMode':''},'State':{'Running':True,'Pid':os.getpid()}},'hermes':{'Config':{'Env':['HERMES_HOME=/home/runtime']},'Mounts':[{'Type':'bind','Destination':'/home','Source':str(base)}]}}
|
||||
# /home/runtime resolves to base/runtime; point home at the SQLite fixture.
|
||||
fixture['hermes']['Mounts'][0]['Source']=str(home)
|
||||
fixture['hermes']['Config']['Env']=['HERMES_HOME=/home']
|
||||
(base/'inspect.json').write_text(json.dumps(fixture))
|
||||
(tools/'docker').write_text('#!/usr/bin/env python3\nimport os,sys,json\na=sys.argv[1:]\nif a[0]=="compose": os.execv("/usr/bin/docker",["docker",*a])\nf=json.load(open(os.environ["INSPECT"]))\nif a[0]=="inspect": print(json.dumps([f["backend" if a[1]=="canino_backend_stage" else "hermes"]]))\nelif a[0]=="exec":\n assert "SET TRANSACTION READ ONLY" in a[-1] and "role::text" in a[-1]\n print(json.dumps([{ "id":os.environ["SUBJECT"]}]))\nelse: raise RuntimeError(a)\n');(tools/'docker').chmod(0o755)
|
||||
env=dict(os.environ,PATH=str(tools)+':'+os.environ['PATH'],INSPECT=str(base/'inspect.json'),SUBJECT=subject,GITHUB_ACTIONS='true',STAGE_PROVISION_ENABLED='true',PYTHONDONTWRITEBYTECODE='1')
|
||||
command='set -euo pipefail; source scripts/rollout-stage.sh; DEPLOY_SHA='+sha+'; unset STAGE_PROVISION_APPROVED_SHA; stage_runtime_provision'
|
||||
# DEPLOY_SHA is deliberately shell-local: helper must export it.
|
||||
captures=[]
|
||||
for _ in range(2):
|
||||
r=subprocess.run(['bash','-c',command],cwd=repo,env=env,capture_output=True,text=True)
|
||||
self.assertEqual(r.returncode,0,r.stderr)
|
||||
capture=json.loads((repo/'.stage-runtime.json').read_text());captures.append(capture)
|
||||
self.assertEqual(json.loads(pathlib.Path(capture['grants']).read_text()),{subject:['default']})
|
||||
self.assertNotIn('PRIVATE TITLE',(pathlib.Path(capture['snapshot'])/'inventory.json').read_text())
|
||||
self.assertEqual(json.loads((root/'secrets/hermes_runtime_grants').read_text()),{})
|
||||
if len(captures)==1:
|
||||
(repo/'.git/info/exclude').write_text('.stage-runtime.json\n')
|
||||
secrets_before={p.name:p.read_bytes() for p in (root/'secrets').iterdir()}
|
||||
self.assertNotEqual(captures[0]['snapshot'],captures[1]['snapshot'])
|
||||
self.assertEqual(secrets_before,{p.name:p.read_bytes() for p in (root/'secrets').iterdir()})
|
||||
before=set((root/'snapshots').iterdir())
|
||||
for invalid in ['', 'not-a-uuid', '------------------------------------']:
|
||||
r=subprocess.run(['bash','-c',command],cwd=repo,env=dict(env,SUBJECT=invalid),capture_output=True,text=True)
|
||||
self.assertNotEqual(r.returncode,0)
|
||||
self.assertIn('Verified admin subjects required',r.stderr)
|
||||
self.assertEqual(before,set((root/'snapshots').iterdir()))
|
||||
(home/'state.db-wal').symlink_to(home/'state.db')
|
||||
r=subprocess.run(['bash','-c',command],cwd=repo,env=env,capture_output=True,text=True)
|
||||
self.assertNotEqual(r.returncode,0)
|
||||
self.assertIn('Unsafe SQLite source',r.stderr)
|
||||
self.assertEqual(before,set((root/'snapshots').iterdir()))
|
||||
(home/'state.db-wal').unlink()
|
||||
# Real Compose CLI normalization and merge; no daemon/socket required.
|
||||
base_config={'services':{n:{'image':'fixture:never-start','networks':{'traefik_public':None}} for n in ['db_stage','redis_stage','backend_stage','frontend_stage']},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}}
|
||||
(repo/'.stage-compose.json').write_text(json.dumps(base_config))
|
||||
r=subprocess.run(['bash','-c','set -euo pipefail; source scripts/rollout-stage.sh; TARGET_DIR="$PWD"; stage_runtime_render'],cwd=repo,env=env,capture_output=True,text=True)
|
||||
self.assertEqual(r.returncode,0,r.stderr)
|
||||
c=json.loads((repo/'.stage-compose.json').read_text());b=c['services']['backend_stage'];inventory=c['services']['runtime-inventory']
|
||||
self.assertEqual(set(b['networks']),{'traefik_public','hermes_inventory_backend'})
|
||||
self.assertEqual(b['labels']['traefik.docker.network'],'traefik_public')
|
||||
self.assertEqual(set(inventory['networks']),{'hermes_inventory_backend'})
|
||||
self.assertTrue(c['networks']['hermes_inventory_backend']['internal'])
|
||||
self.assertEqual(b['environment']['CANINA_RUNTIME_READ_ENABLED'],'true')
|
||||
self.assertEqual(c['secrets']['hermes_runtime_grants']['file'],captures[-1]['grants'])
|
||||
self.assertEqual(inventory['mem_limit'],'1073741824')
|
||||
self.assertTrue(all(m.get('read_only') for m in inventory['volumes']))
|
||||
self.assertFalse(inventory.get('ports'))
|
||||
self.assertEqual(inventory['user'],'1000:1000')
|
||||
|
||||
def test_base_bind_allowlist_rejects_before_runtime_merge(self):
|
||||
import json
|
||||
script=(ROOT/'scripts/deploy.sh').read_text()
|
||||
validation=script.split("config = json.loads(Path('.stage-base.json').read_text())",1)[1].split("\nPY",1)[0]
|
||||
code="import json\nfrom pathlib import Path\nconfig=json.loads(Path('.stage-base.json').read_text())"+validation
|
||||
config={'services':{'db_stage':{},'redis_stage':{},'backend_stage':{'container_name':'canino_backend_stage','volumes':[{'type':'volume','source':'canina_stage_uploads','target':'/app/uploads'},{'type':'bind','source':'/etc','target':'/stolen'}]},'frontend_stage':{'container_name':'canino_frontend_stage'}},'volumes':{n:{'name':'stage_'+n} for n in ['canina_stage_db','canina_stage_uploads']},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}}
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
p=pathlib.Path(directory);(p/'.stage-base.json').write_text(json.dumps(config))
|
||||
result=subprocess.run(['python3','-c',code],cwd=p,capture_output=True,text=True)
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
self.assertIn('unexpected Stage bind mount',result.stderr)
|
||||
self.assertFalse((p/'.stage-compose.json').exists())
|
||||
self.assertLess(script.index('unexpected Stage bind mount'),script.index(' stage_runtime_render'))
|
||||
Reference in new issue
Block a user