diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 810f82fe..a0547b2d 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -29,8 +29,16 @@ git cat-file -e "${DEPLOY_SHA}^{commit}" git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop git checkout --detach "$DEPLOY_SHA" [ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ] +# Provision against the pristine approved SHA, before any tracked render edits. +source scripts/rollout-stage.sh +if [ "${STAGE_PROVISION_ENABLED:-false}" = true ]; then + stage_runtime_provision +elif [ -e /opt/services/canina/stage-runtime ]; then + echo 'ERROR: existing runtime requires explicit Stage CI opt-in on every deploy' >&2 + exit 2 +fi # Keep operational files out of status/accidental commits in this disposable clone. -printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n' >> .git/info/exclude +printf '\n.support-chat.env\n.revalidation.env\n.stage-base.json\n.stage-compose.json\n.stage-runtime.json\n.stage-runtime-merged.json\n' >> .git/info/exclude # Runtime secrets/config must never enter the frontend Docker build context. printf '\n.support-chat.env\n.revalidation.env\n.stage-*.json\n' >> .dockerignore # Only these backend/frontend runtime env files are required. Do not copy the @@ -92,6 +100,9 @@ for service in config['services'].values(): Path('.stage-compose.json').write_text(json.dumps(config)) Path('.stage-base.json').unlink() PY +if [ "${STAGE_PROVISION_ENABLED:-false}" = true ]; then + stage_runtime_render +fi # Read-only checks: missing external storage must fail, never provision blank DB. docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null docker network inspect traefik_public >/dev/null diff --git a/scripts/rollout-stage.sh b/scripts/rollout-stage.sh index 05823b43..871c5056 100644 --- a/scripts/rollout-stage.sh +++ b/scripts/rollout-stage.sh @@ -1,9 +1,104 @@ #!/bin/bash # Sourced by the CI/CD deploy entrypoint, never a standalone deploy command. +# Invoked only by the authenticated Stage CI entrypoint under its rollout lock. +stage_runtime_provision() { + [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" = true ] + [ "${STAGE_PROVISION_ENABLED:-}" = true ] + export DEPLOY_SHA STAGE_PROVISION_APPROVED_SHA="$DEPLOY_SHA" CI=true + python3 - <<'PY' +import os, json, subprocess, importlib.util, resource, secrets, re, stat +from pathlib import Path +resource.setrlimit(resource.RLIMIT_AS, (1536*1024*1024, 1536*1024*1024)) +resource.setrlimit(resource.RLIMIT_CPU, (60, 60)) +resource.setrlimit(resource.RLIMIT_FSIZE, (64*1024*1024, 64*1024*1024)) +def docker(*args): + return subprocess.check_output(['docker', *args], text=True, timeout=45) +spec=importlib.util.spec_from_file_location('provision','scripts/provision-stage-runtime.py') +m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m) +m.ci_gate() +def inspect(name): return json.loads(docker('inspect',name))[0] +b=inspect('canino_backend_stage') +if not b['State'].get('Running') or b['State'].get('Pid',0) <= 0: + raise ValueError('Running backend required') +if b['Config']['User'] not in ('node','1000','1000:1000') or b['HostConfig'].get('UsernsMode') not in ('','default'): + raise ValueError('Unreviewed runtime identity/userns') +def mapped(kind): + for row in Path('/proc/%s/%s_map'%(b['State']['Pid'],kind)).read_text().splitlines(): + inside,outside,count=map(int,row.split()) + if inside <= 1000 < inside+count: return outside+1000-inside + raise ValueError('Unmapped runtime identity') +uid,gid=mapped('uid'),mapped('gid') +if min(uid,gid)<=0: raise ValueError('Root identity forbidden') +h=inspect('canina-hermes') +env=dict(e.split('=',1) for e in h['Config']['Env'] if '=' in e) +home=Path(env.get('HERMES_HOME','')) +if not home.is_absolute() or '..' in home.parts: raise ValueError('Explicit Hermes home required') +mounts=[v for v in h['Mounts'] if home==Path(v['Destination']) or Path(v['Destination']) in home.parents] +if not mounts: raise ValueError('Home mount required') +mount=max(mounts,key=lambda v:len(v['Destination'])) +if mount.get('Type') not in ('bind','volume'): + raise ValueError('Filesystem home mount required') +m.safe_path(Path(mount['Source'])) +db=Path(mount['Source'])/home.relative_to(mount['Destination'])/'state.db' +m.safe_path(db.parent) +for p in [db,Path(str(db)+'-wal'),Path(str(db)+'-shm')]: + if p.exists() or p.is_symlink(): + st=p.lstat() + if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1: + raise ValueError('Unsafe SQLite source') +size=sum(p.stat().st_size for p in [db,Path(str(db)+'-wal')] if p.exists()) +if not 0{try{const rows=await p.$transaction(async tx=>{ +await tx.$executeRawUnsafe('SET TRANSACTION READ ONLY'); +return tx.$queryRawUnsafe(\"SELECT id::text FROM users WHERE lower(replace(replace(role::text, '_', ''), '-', '')) IN ('admin','superadmin') ORDER BY id LIMIT 1001\"); +});process.stdout.write(JSON.stringify(rows));}catch{process.exitCode=1;}finally{await p.$disconnect();}})();""" +rows=json.loads(docker('exec','-w','/app','canino_backend_stage','node','-e',query)) +if not 1<=len(rows)<=1000 or any(not isinstance(r,dict) or set(r)!={'id'} or not isinstance(r['id'],str) or not re.fullmatch('[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}',r['id']) for r in rows): + raise ValueError('Verified admin subjects required') +m.provision(root,uid,gid) +# Leave the validated deny-all sentinel unchanged for repeat provisioning. +capture=os.environ['DEPLOY_SHA']+'-'+secrets.token_hex(8) +dest=root/'snapshots'/capture +m.export_snapshot(db,dest,'default',uid,gid) +grants=root/'snapshots'/(capture+'-grants.json') +m.write_private(grants,json.dumps({r['id']:['default'] for r in rows}),uid,gid) +Path('.stage-runtime.json').write_text(json.dumps({'snapshot':str(dest),'grants':str(grants)})) +print('Stage projected snapshot captured; existing admin grants verified; no executor started') +PY +} + +stage_runtime_render() { + export STAGE_RUNTIME_SERVE_FILE="$TARGET_DIR/infra/hermes-runtime/serve.py" + export STAGE_RUNTIME_SNAPSHOT + STAGE_RUNTIME_SNAPSHOT=$(python3 -c 'import json;print(json.load(open(".stage-runtime.json"))["snapshot"])') + command docker compose -p stage --env-file /dev/null --project-directory "$TARGET_DIR" \ + -f "$TARGET_DIR/.stage-compose.json" -f infra/hermes-runtime/compose.stage-provision.yml \ + --profile stage-runtime-inventory config --format json > .stage-runtime-merged.json + python3 - <<'PY' +import json +from pathlib import Path +p=Path('.stage-runtime-merged.json');c=json.loads(p.read_text()) +b=c['services']['backend_stage'] +b.setdefault('environment',{})['CANINA_RUNTIME_READ_ENABLED']='true' +b.setdefault('networks',{})['traefik_public']=None +b.setdefault('labels',{})['traefik.docker.network']='traefik_public' +c['secrets']['hermes_runtime_grants']['file']=json.loads(Path('.stage-runtime.json').read_text())['grants'] +assert c['networks']['hermes_inventory_backend']['internal'] is True +assert c['networks']['traefik_public']['external'] is True +assert set(c['services'])=={'db_stage','redis_stage','frontend_stage','backend_stage','runtime-inventory'} +Path('.stage-compose.json').write_text(json.dumps(c));p.unlink() +PY +} + stage_rollout() { # Maintenance is deliberate: neither old nor new app serves during DDL. docker compose stop frontend_stage backend_stage docker compose run --rm --no-deps --entrypoint ./node_modules/.bin/prisma backend_stage migrate deploy + if [ "${STAGE_PROVISION_ENABLED:-false}" = true ]; then + docker compose up -d --no-deps runtime-inventory + fi docker compose up -d --no-deps backend_stage if ! docker compose exec -T backend_stage node -e "$(< scripts/rollout-health.cjs)"; then docker compose stop frontend_stage backend_stage diff --git a/scripts/tests/test_stage_provision_wiring.py b/scripts/tests/test_stage_provision_wiring.py new file mode 100644 index 00000000..cebb3c38 --- /dev/null +++ b/scripts/tests/test_stage_provision_wiring.py @@ -0,0 +1,98 @@ +"""CI runtime integration: execute extracted deploy boundary, never live apply.""" +import pathlib, subprocess, tempfile, unittest, os +ROOT = pathlib.Path(__file__).resolve().parents[2] +class WiringTests(unittest.TestCase): + def test_clean_provision_precedes_all_rendered_edits(self): + s=(ROOT/'scripts/deploy.sh').read_text() + self.assertIn('stage_runtime_provision',s) + self.assertLess(s.index('stage_runtime_provision'),s.index('>> .dockerignore')) + self.assertIn('CANINA_RUNTIME_READ_ENABLED', (ROOT/'scripts/rollout-stage.sh').read_text()) + def test_inventory_started_before_backend(self): + with tempfile.TemporaryDirectory() as d: + p=pathlib.Path(d)/'docker'; p.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$TRACE"\n');p.chmod(0o755) + r=subprocess.run(['bash','-c','set -e; source scripts/rollout-stage.sh; STAGE_PROVISION_ENABLED=true; stage_rollout'],cwd=ROOT,env=dict(os.environ,PATH=d+':'+os.environ['PATH'],TRACE=d+'/trace'),capture_output=True,text=True) + self.assertEqual(r.returncode,0,r.stderr) + trace=pathlib.Path(d+'/trace').read_text() + self.assertIn('up -d --no-deps runtime-inventory',trace) + self.assertLess(trace.index('up -d --no-deps runtime-inventory'),trace.index('up -d --no-deps backend_stage')) + + def test_provision_and_real_compose_render_repeat(self): + import json, shutil, sqlite3 + with tempfile.TemporaryDirectory() as directory: + base=pathlib.Path(directory); repo=base/'repo';repo.mkdir() + shutil.copytree(ROOT/'scripts',repo/'scripts',ignore=shutil.ignore_patterns('__pycache__')) + shutil.copytree(ROOT/'infra',repo/'infra') + root=base/'runtime';home=base/'home';home.mkdir();tools=base/'tools';tools.mkdir() + db=sqlite3.connect(home/'state.db') + db.execute('CREATE TABLE sessions(source,started_at,input_tokens,output_tokens,estimated_cost_usd,title)') + db.execute("INSERT INTO sessions VALUES ('cli',1,2,3,0.1,'PRIVATE TITLE')");db.commit();db.close() + helper=repo/'scripts/rollout-stage.sh' + helper.write_text(helper.read_text().replace('/opt/services/canina/stage-runtime',str(root))) + def git(*args):return subprocess.check_output(['git',*args],cwd=repo,text=True).strip() + git('init','-q');git('config','user.email','fixture@example.invalid');git('config','user.name','Fixture');git('add','.');git('commit','-qm','fixture') + sha=git('rev-parse','HEAD') + subject='12345678-1234-1234-1234-123456789abc' + fixture={'backend':{'Config':{'User':'node'},'HostConfig':{'UsernsMode':''},'State':{'Running':True,'Pid':os.getpid()}},'hermes':{'Config':{'Env':['HERMES_HOME=/home/runtime']},'Mounts':[{'Type':'bind','Destination':'/home','Source':str(base)}]}} + # /home/runtime resolves to base/runtime; point home at the SQLite fixture. + fixture['hermes']['Mounts'][0]['Source']=str(home) + fixture['hermes']['Config']['Env']=['HERMES_HOME=/home'] + (base/'inspect.json').write_text(json.dumps(fixture)) + (tools/'docker').write_text('#!/usr/bin/env python3\nimport os,sys,json\na=sys.argv[1:]\nif a[0]=="compose": os.execv("/usr/bin/docker",["docker",*a])\nf=json.load(open(os.environ["INSPECT"]))\nif a[0]=="inspect": print(json.dumps([f["backend" if a[1]=="canino_backend_stage" else "hermes"]]))\nelif a[0]=="exec":\n assert "SET TRANSACTION READ ONLY" in a[-1] and "role::text" in a[-1]\n print(json.dumps([{ "id":os.environ["SUBJECT"]}]))\nelse: raise RuntimeError(a)\n');(tools/'docker').chmod(0o755) + env=dict(os.environ,PATH=str(tools)+':'+os.environ['PATH'],INSPECT=str(base/'inspect.json'),SUBJECT=subject,GITHUB_ACTIONS='true',STAGE_PROVISION_ENABLED='true',PYTHONDONTWRITEBYTECODE='1') + command='set -euo pipefail; source scripts/rollout-stage.sh; DEPLOY_SHA='+sha+'; unset STAGE_PROVISION_APPROVED_SHA; stage_runtime_provision' + # DEPLOY_SHA is deliberately shell-local: helper must export it. + captures=[] + for _ in range(2): + r=subprocess.run(['bash','-c',command],cwd=repo,env=env,capture_output=True,text=True) + self.assertEqual(r.returncode,0,r.stderr) + capture=json.loads((repo/'.stage-runtime.json').read_text());captures.append(capture) + self.assertEqual(json.loads(pathlib.Path(capture['grants']).read_text()),{subject:['default']}) + self.assertNotIn('PRIVATE TITLE',(pathlib.Path(capture['snapshot'])/'inventory.json').read_text()) + self.assertEqual(json.loads((root/'secrets/hermes_runtime_grants').read_text()),{}) + if len(captures)==1: + (repo/'.git/info/exclude').write_text('.stage-runtime.json\n') + secrets_before={p.name:p.read_bytes() for p in (root/'secrets').iterdir()} + self.assertNotEqual(captures[0]['snapshot'],captures[1]['snapshot']) + self.assertEqual(secrets_before,{p.name:p.read_bytes() for p in (root/'secrets').iterdir()}) + before=set((root/'snapshots').iterdir()) + for invalid in ['', 'not-a-uuid', '------------------------------------']: + r=subprocess.run(['bash','-c',command],cwd=repo,env=dict(env,SUBJECT=invalid),capture_output=True,text=True) + self.assertNotEqual(r.returncode,0) + self.assertIn('Verified admin subjects required',r.stderr) + self.assertEqual(before,set((root/'snapshots').iterdir())) + (home/'state.db-wal').symlink_to(home/'state.db') + r=subprocess.run(['bash','-c',command],cwd=repo,env=env,capture_output=True,text=True) + self.assertNotEqual(r.returncode,0) + self.assertIn('Unsafe SQLite source',r.stderr) + self.assertEqual(before,set((root/'snapshots').iterdir())) + (home/'state.db-wal').unlink() + # Real Compose CLI normalization and merge; no daemon/socket required. + base_config={'services':{n:{'image':'fixture:never-start','networks':{'traefik_public':None}} for n in ['db_stage','redis_stage','backend_stage','frontend_stage']},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}} + (repo/'.stage-compose.json').write_text(json.dumps(base_config)) + r=subprocess.run(['bash','-c','set -euo pipefail; source scripts/rollout-stage.sh; TARGET_DIR="$PWD"; stage_runtime_render'],cwd=repo,env=env,capture_output=True,text=True) + self.assertEqual(r.returncode,0,r.stderr) + c=json.loads((repo/'.stage-compose.json').read_text());b=c['services']['backend_stage'];inventory=c['services']['runtime-inventory'] + self.assertEqual(set(b['networks']),{'traefik_public','hermes_inventory_backend'}) + self.assertEqual(b['labels']['traefik.docker.network'],'traefik_public') + self.assertEqual(set(inventory['networks']),{'hermes_inventory_backend'}) + self.assertTrue(c['networks']['hermes_inventory_backend']['internal']) + self.assertEqual(b['environment']['CANINA_RUNTIME_READ_ENABLED'],'true') + self.assertEqual(c['secrets']['hermes_runtime_grants']['file'],captures[-1]['grants']) + self.assertEqual(inventory['mem_limit'],'1073741824') + self.assertTrue(all(m.get('read_only') for m in inventory['volumes'])) + self.assertFalse(inventory.get('ports')) + self.assertEqual(inventory['user'],'1000:1000') + + def test_base_bind_allowlist_rejects_before_runtime_merge(self): + import json + script=(ROOT/'scripts/deploy.sh').read_text() + validation=script.split("config = json.loads(Path('.stage-base.json').read_text())",1)[1].split("\nPY",1)[0] + code="import json\nfrom pathlib import Path\nconfig=json.loads(Path('.stage-base.json').read_text())"+validation + config={'services':{'db_stage':{},'redis_stage':{},'backend_stage':{'container_name':'canino_backend_stage','volumes':[{'type':'volume','source':'canina_stage_uploads','target':'/app/uploads'},{'type':'bind','source':'/etc','target':'/stolen'}]},'frontend_stage':{'container_name':'canino_frontend_stage'}},'volumes':{n:{'name':'stage_'+n} for n in ['canina_stage_db','canina_stage_uploads']},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}} + with tempfile.TemporaryDirectory() as directory: + p=pathlib.Path(directory);(p/'.stage-base.json').write_text(json.dumps(config)) + result=subprocess.run(['python3','-c',code],cwd=p,capture_output=True,text=True) + self.assertNotEqual(result.returncode,0) + self.assertIn('unexpected Stage bind mount',result.stderr) + self.assertFalse((p/'.stage-compose.json').exists()) + self.assertLess(script.index('unexpected Stage bind mount'),script.index(' stage_runtime_render'))