fix(deploy): gate Stage v4 rollout on schema-first maintenance and acceptance

This commit is contained in:
devops committed 2026-10-22 19:18:50 +00:00
1 parent ac065fa911
commit 795bbb33cf
7 files changed
+179 -43

No files matched your search

+11 -43
View File
@@ -1,22 +1,13 @@
#!/bin/bash
set -e
# Explicit no-data-write stage rollout. Run only after testing and pushing HEAD.
if [ "${2:-}" = "stage-backend-only" ]; then
[ "${1:-}" = "develop" ] || exit 2
cd /opt/services/canina/stage
exec 9>/opt/services/canina/stage-backend-deploy.lock
flock -n 9 || { echo "Stage deployment already in progress"; exit 1; }
[ "$(git branch --show-current)" = develop ]
git fetch origin develop
[ "$(git rev-parse HEAD)" = "$(git rev-parse origin/develop)" ]
git diff --quiet -- backend scripts/compose.stage.yml scripts/deploy.sh
# Read tracked compose directly; preserve generated config and nginx.
docker compose -p stage -f scripts/compose.stage.yml --project-directory . build backend_stage
docker compose -p stage -f scripts/compose.stage.yml --project-directory . up -d --no-deps --remove-orphans backend_stage
exit 0
set -euo pipefail
# CI/CD-only Stage rollout; Production awaits separate reviewed authorization.
if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [ "${1:-}" != develop ] || [ "$#" != 1 ]; then
echo 'ERROR: refusing deployment outside CI/CD Stage rollout' >&2
exit 2
fi
# END ROLLOUT GUARD
exec 9>/opt/services/canina/stage-backend-deploy.lock
flock -n 9 || { echo 'Stage deployment already in progress' >&2; exit 1; }
export DOCKER_BUILDKIT=1
export COMPOSE_DOCKER_CLI_BUILD=1
@@ -125,32 +116,9 @@ docker compose --progress plain build \
--build-arg VITE_API_URL="${API_URL}" \
${NPM_REGISTRY_ARG}
echo "Building support-chat bot image from committed source..."
BOT_IMAGE="canina-support-bot:${BRANCH}"
DOCKER_BUILDKIT=0 docker build --pull=false -f scripts/support-chat/Dockerfile -t "${BOT_IMAGE}" .
BOT_ENV="${TARGET_DIR}/.support-bot.runtime.env"
docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' canina-bot > "${BOT_ENV}"
chmod 600 "${BOT_ENV}"
echo "Starting containers..."
docker compose up -d --remove-orphans
echo "Starting support-chat bot container..."
docker rm -f canina-bot >/dev/null 2>&1 || true
docker run -d --name canina-bot --restart unless-stopped --env-file "${BOT_ENV}" -e HERMES_HOME=/opt/data -e HERMES_RUNTIME_DIR=/opt/hermes/tools -e API_SERVER_HOST=0.0.0.0 -e API_SERVER_PORT=8642 --network traefik_public -v canina_bot_data:/opt/data "${BOT_IMAGE}" gateway run
docker network connect --ip 172.22.0.15 vpn_net canina-bot
rm -f "${BOT_ENV}"
echo "Waiting for backend to start..."
sleep 15
# Apply committed Prisma migrations as part of the CI/CD deployment.
# This is intentionally migrate deploy (not db push): it is repeatable,
# fail-closed, and applies only migrations already reviewed and committed.
echo "Applying committed database migrations..."
if [ "$BRANCH" = "main" ]; then
docker exec -e PRISMA_HIDE_UPDATE_MESSAGE=true "${CONTAINER_PREFIX}" npx prisma migrate resolve --applied 20261013210253_add_admin_modules >/dev/null 2>&1 || true
fi
docker exec -e PRISMA_HIDE_UPDATE_MESSAGE=true "${CONTAINER_PREFIX}" npx prisma migrate deploy
# Code-only deployment: seed and destructive data changes are not performed here.
# Never recreate the shared Production support bot during a Stage rollout.
source scripts/rollout-stage.sh
stage_rollout
# Auto-cleanup old docker builder cache to keep disk healthy (keeps latest 6GB)
echo "Cleaning up old docker build cache..."
+24
View File
@@ -0,0 +1,24 @@
// No response bodies or credentials are logged by rollout acceptance.
async function accept(base) {
const r = await fetch(`${base}/api/products?limit=1`, {signal: AbortSignal.timeout(5000), redirect: 'error'});
if (r.status !== 200) throw new Error('catalog status');
const b = await r.json();
if (!Array.isArray(b.data) || !Number.isInteger(b.meta?.total) || b.meta.total < 0) throw new Error('catalog shape');
}
async function main() {
const {PrismaClient} = require('@prisma/client');
const db = new PrismaClient();
try {
const names = ['tasks','runs','artifacts','attestations','dependencies','approval_decisions','events','budget_ledger','kill_switches'];
for (const name of names) await db.$queryRawUnsafe(`SELECT * FROM orchestration_${name} LIMIT 0`);
const rows = await db.$queryRawUnsafe("SELECT 1 FROM _prisma_migrations WHERE migration_name = '20261022000000_add_orchestration_v4_core' AND finished_at IS NOT NULL AND rolled_back_at IS NULL");
if (rows.length !== 1) throw new Error('migration receipt');
for (let n=0; n<30; n++) {
try { await accept('http://127.0.0.1:3000'); console.log('Backend schema/catalog acceptance passed'); return; }
catch { if(n===29) throw new Error('catalog acceptance'); await new Promise(r=>setTimeout(r,2000)); }
}
} finally { await db.$disconnect(); }
}
module.exports = {accept};
// node -e is used inside the built backend; imports in tests do not execute main.
if (!module.parent) main().catch(()=>{console.error('Backend acceptance failed (schema, migration receipt or catalog)');process.exitCode=1;});
+14
View File
@@ -0,0 +1,14 @@
#!/bin/bash
# Sourced by the CI/CD deploy entrypoint, never a standalone deploy command.
stage_rollout() {
# Maintenance is deliberate: neither old nor new app serves during DDL.
docker compose stop frontend_stage backend_stage
docker compose run --rm --no-deps --entrypoint ./node_modules/.bin/prisma backend_stage migrate deploy
docker compose up -d --no-deps backend_stage
if ! docker compose exec -T backend_stage node -e "$(< scripts/rollout-health.cjs)"; then
docker compose stop frontend_stage backend_stage
echo 'ERROR: backend acceptance failed; maintenance retained' >&2
return 1
fi
docker compose up -d --no-deps frontend_stage
}
+38
View File
@@ -0,0 +1,38 @@
import subprocess as sp, tempfile, pathlib, os, time, json
ROOT=pathlib.Path('/opt/services/canina/worktrees/orchestration-v4-rollout')
name='canina-v4-rollout-rehearsal'
def run(args,**kw): return sp.run(args,check=True,**kw)
def sql(q): return sp.check_output(['docker','exec','-i',name,'psql','-U','rehearsal','-d','rehearsal','-v','ON_ERROR_STOP=1','-At'],input=q.encode()).decode()
with tempfile.TemporaryDirectory(prefix='canina-rollout-') as d:
os.chmod(d,0o700)
schema=pathlib.Path(d)/'schema.sql'; history=pathlib.Path(d)/'history.sql'
for file,args in [(schema,['--schema-only','--no-owner','--no-privileges']),(history,['--data-only','--table=public._prisma_migrations','--column-inserts'])]:
with file.open('wb') as f:
run(['docker','exec','canino_db_stage','sh','-c','exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"','sh',*args],stdout=f)
print('Captured schema-only Stage dump plus _prisma_migrations history only')
run(['docker','run','-d','--name',name,'--network','none','-e','POSTGRES_USER=rehearsal','-e','POSTGRES_DB=rehearsal','-e','POSTGRES_HOST_AUTH_METHOD=trust','postgres:16-alpine'],stdout=sp.DEVNULL)
try:
for _ in range(30):
if sp.run(['docker','exec',name,'pg_isready','-U','rehearsal'],stdout=sp.DEVNULL,stderr=sp.DEVNULL).returncode==0: break
time.sleep(1)
for file in (schema,history):
with file.open('rb') as f: run(['docker','exec','-i',name,'psql','-U','rehearsal','-d','rehearsal','-v','ON_ERROR_STOP=1'],stdin=f,stdout=sp.DEVNULL)
before=sql("SELECT table_name,column_name,data_type,is_nullable,column_default FROM information_schema.columns WHERE table_schema='public' AND table_name <> '_prisma_migrations' ORDER BY 1,2;")
tables=sql("SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename <> '_prisma_migrations' ORDER BY 1;").splitlines()
for table in tables:
assert sql(f'SELECT count(*) FROM "{table}";').strip()=='0',table
print(f'Restored {len(tables)} legacy tables; all business tables empty')
cmd=['docker','run','--rm','--network','container:'+name,'-v',str(ROOT/'backend/prisma')+':/app/prisma:ro','-v','/opt/services/canina/worktrees/orchestration-v4-core/backend/node_modules:/app/node_modules:ro','-w','/app','-e','DATABASE_URL=postgresql://rehearsal@127.0.0.1:5432/rehearsal?schema=public','node:22-bookworm','./node_modules/.bin/prisma','migrate','deploy']
run(cmd); run(cmd)
query = "const {PrismaClient}=require('@prisma/client');const p=new PrismaClient();(async()=>{await p.product.findMany({take:1,include:{ingredientList:true,symptoms:true}});await p.supportConversation.findMany({take:1});await p.orchestrationTask.findMany({take:1,include:{runs:true,events:true,budgetLedger:true}});console.log('PASS: actual Prisma legacy catalog/support and v4 relation queries');})().catch(()=>{console.error('Prisma compatibility failed');process.exitCode=1}).finally(()=>p.$disconnect());"
run(cmd[:-3]+['node','-e',query])
after=sql("SELECT table_name,column_name,data_type,is_nullable,column_default FROM information_schema.columns WHERE table_schema='public' AND table_name <> '_prisma_migrations' AND table_name NOT LIKE 'orchestration_%' ORDER BY 1,2;")
assert before==after,'Legacy column definitions changed'
for table in tables: sql(f'SELECT * FROM "{table}" LIMIT 0;')
v4=sql("SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename LIKE 'orchestration_%' ORDER BY 1;").splitlines()
assert len(v4)==9,v4
for table in v4: sql(f'SELECT * FROM "{table}" LIMIT 0;')
print(f'PASS: unchanged legacy column definitions; {len(tables)} legacy queries; {len(v4)} v4 tables; migrate deploy repeatable')
finally:
run(['docker','rm','-fv',name],stdout=sp.DEVNULL)
print('Disposable database/container removed; no live DDL performed')
+11
View File
@@ -0,0 +1,11 @@
const {test}=require('node:test');
const assert=require('node:assert/strict');
const http=require('node:http');
const {accept}=require('../rollout-health.cjs');
for(const [name,status,body,pass] of [['bad status',503,'{}',false],['fake html',200,'<html>ok</html>',false],['wrong envelope',200,'{}',false],['valid empty catalog',200,'{"data":[],"meta":{"total":0}}',true]]) {
test(name,async()=>{
const s=http.createServer((q,r)=>{r.writeHead(status,{'content-type':'application/json'});r.end(body)});
await new Promise(r=>s.listen(0,'127.0.0.1',r));
try { const p=accept(`http://127.0.0.1:${s.address().port}`); if(pass) await p; else await assert.rejects(p); } finally {await new Promise(r=>s.close(r));}
});
}
+37
View File
@@ -0,0 +1,37 @@
import os, pathlib, subprocess, tempfile, unittest
ROOT = pathlib.Path(__file__).resolve().parents[2]
class RolloutTests(unittest.TestCase):
def test_non_ci_and_production_refused_before_docker(self):
script=(ROOT/'scripts/deploy.sh').read_text()
self.assertIn('CI/CD-only Stage rollout',script)
guard=script.split('# END ROLLOUT GUARD')[0]
for args,env in [(['develop'],{}),(['develop','stage-backend-only'],{'GITHUB_ACTIONS':'true'}),(['main'],{'GITHUB_ACTIONS':'true'})]:
r=subprocess.run(['bash','-c',guard,'deploy.sh',*args],cwd=ROOT,env=dict(os.environ,**env),capture_output=True,text=True)
self.assertNotEqual(r.returncode,0)
self.assertIn('refusing',r.stderr)
def test_migration_failure_never_starts_apps(self):
with tempfile.TemporaryDirectory() as d:
docker = pathlib.Path(d)/'docker'
docker.write_text('#!/bin/bash\nprintf "%s\\n" "$*" >> "$TRACE"\nif [[ "$*" == *"run --rm"* ]]; then exit 42; fi\n')
docker.chmod(0o755)
env = dict(os.environ, PATH=d+':'+os.environ['PATH'], TRACE=d+'/trace')
result = subprocess.run(['bash','-c', 'set -e; source scripts/rollout-stage.sh; stage_rollout; echo SUCCESS'], cwd=ROOT,env=env,capture_output=True,text=True)
self.assertNotEqual(result.returncode,0)
trace=pathlib.Path(d+'/trace').read_text() if pathlib.Path(d+'/trace').exists() else ''
self.assertIn('stop frontend_stage backend_stage',trace)
self.assertIn('run --rm --no-deps --entrypoint',trace)
self.assertNotIn('up -d',trace)
self.assertNotIn('SUCCESS',result.stdout)
def test_health_failure_stops_backend_and_does_not_publish_frontend(self):
with tempfile.TemporaryDirectory() as d:
docker = pathlib.Path(d)/'docker'
docker.write_text('#!/bin/bash\nprintf "%s\\n" "$*" >> "$TRACE"\nif [[ "$*" == *"exec -T backend_stage node"* ]]; then exit 43; fi\n')
docker.chmod(0o755)
result = subprocess.run(['bash','-c','set -e; source scripts/rollout-stage.sh; stage_rollout; echo SUCCESS'],cwd=ROOT,env=dict(os.environ,PATH=d+':'+os.environ['PATH'],TRACE=d+'/trace'),capture_output=True,text=True)
self.assertNotEqual(result.returncode,0)
trace=pathlib.Path(d+'/trace').read_text()
self.assertIn('stop frontend_stage backend_stage',trace)
self.assertIn('exec -T backend_stage node',trace)
self.assertNotIn('up -d --no-deps frontend_stage',trace)
self.assertNotIn('SUCCESS',result.stdout)
if __name__=='__main__': unittest.main()