fix(deploy): gate Stage v4 rollout on schema-first maintenance and acceptance
This commit is contained in:
1 parent
ac065fa911
commit
795bbb33cf
7 files changed
+179
-43
No files matched your search
+11
-43
@@ -1,22 +1,13 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Explicit no-data-write stage rollout. Run only after testing and pushing HEAD.
|
||||
if [ "${2:-}" = "stage-backend-only" ]; then
|
||||
[ "${1:-}" = "develop" ] || exit 2
|
||||
cd /opt/services/canina/stage
|
||||
exec 9>/opt/services/canina/stage-backend-deploy.lock
|
||||
flock -n 9 || { echo "Stage deployment already in progress"; exit 1; }
|
||||
[ "$(git branch --show-current)" = develop ]
|
||||
git fetch origin develop
|
||||
[ "$(git rev-parse HEAD)" = "$(git rev-parse origin/develop)" ]
|
||||
git diff --quiet -- backend scripts/compose.stage.yml scripts/deploy.sh
|
||||
# Read tracked compose directly; preserve generated config and nginx.
|
||||
docker compose -p stage -f scripts/compose.stage.yml --project-directory . build backend_stage
|
||||
docker compose -p stage -f scripts/compose.stage.yml --project-directory . up -d --no-deps --remove-orphans backend_stage
|
||||
exit 0
|
||||
set -euo pipefail
|
||||
# CI/CD-only Stage rollout; Production awaits separate reviewed authorization.
|
||||
if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [ "${1:-}" != develop ] || [ "$#" != 1 ]; then
|
||||
echo 'ERROR: refusing deployment outside CI/CD Stage rollout' >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# END ROLLOUT GUARD
|
||||
exec 9>/opt/services/canina/stage-backend-deploy.lock
|
||||
flock -n 9 || { echo 'Stage deployment already in progress' >&2; exit 1; }
|
||||
export DOCKER_BUILDKIT=1
|
||||
export COMPOSE_DOCKER_CLI_BUILD=1
|
||||
|
||||
@@ -125,32 +116,9 @@ docker compose --progress plain build \
|
||||
--build-arg VITE_API_URL="${API_URL}" \
|
||||
${NPM_REGISTRY_ARG}
|
||||
|
||||
echo "Building support-chat bot image from committed source..."
|
||||
BOT_IMAGE="canina-support-bot:${BRANCH}"
|
||||
DOCKER_BUILDKIT=0 docker build --pull=false -f scripts/support-chat/Dockerfile -t "${BOT_IMAGE}" .
|
||||
BOT_ENV="${TARGET_DIR}/.support-bot.runtime.env"
|
||||
docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' canina-bot > "${BOT_ENV}"
|
||||
chmod 600 "${BOT_ENV}"
|
||||
echo "Starting containers..."
|
||||
docker compose up -d --remove-orphans
|
||||
echo "Starting support-chat bot container..."
|
||||
docker rm -f canina-bot >/dev/null 2>&1 || true
|
||||
docker run -d --name canina-bot --restart unless-stopped --env-file "${BOT_ENV}" -e HERMES_HOME=/opt/data -e HERMES_RUNTIME_DIR=/opt/hermes/tools -e API_SERVER_HOST=0.0.0.0 -e API_SERVER_PORT=8642 --network traefik_public -v canina_bot_data:/opt/data "${BOT_IMAGE}" gateway run
|
||||
docker network connect --ip 172.22.0.15 vpn_net canina-bot
|
||||
rm -f "${BOT_ENV}"
|
||||
echo "Waiting for backend to start..."
|
||||
sleep 15
|
||||
|
||||
# Apply committed Prisma migrations as part of the CI/CD deployment.
|
||||
# This is intentionally migrate deploy (not db push): it is repeatable,
|
||||
# fail-closed, and applies only migrations already reviewed and committed.
|
||||
echo "Applying committed database migrations..."
|
||||
if [ "$BRANCH" = "main" ]; then
|
||||
docker exec -e PRISMA_HIDE_UPDATE_MESSAGE=true "${CONTAINER_PREFIX}" npx prisma migrate resolve --applied 20261013210253_add_admin_modules >/dev/null 2>&1 || true
|
||||
fi
|
||||
docker exec -e PRISMA_HIDE_UPDATE_MESSAGE=true "${CONTAINER_PREFIX}" npx prisma migrate deploy
|
||||
|
||||
# Code-only deployment: seed and destructive data changes are not performed here.
|
||||
# Never recreate the shared Production support bot during a Stage rollout.
|
||||
source scripts/rollout-stage.sh
|
||||
stage_rollout
|
||||
|
||||
# Auto-cleanup old docker builder cache to keep disk healthy (keeps latest 6GB)
|
||||
echo "Cleaning up old docker build cache..."
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
// No response bodies or credentials are logged by rollout acceptance.
|
||||
async function accept(base) {
|
||||
const r = await fetch(`${base}/api/products?limit=1`, {signal: AbortSignal.timeout(5000), redirect: 'error'});
|
||||
if (r.status !== 200) throw new Error('catalog status');
|
||||
const b = await r.json();
|
||||
if (!Array.isArray(b.data) || !Number.isInteger(b.meta?.total) || b.meta.total < 0) throw new Error('catalog shape');
|
||||
}
|
||||
async function main() {
|
||||
const {PrismaClient} = require('@prisma/client');
|
||||
const db = new PrismaClient();
|
||||
try {
|
||||
const names = ['tasks','runs','artifacts','attestations','dependencies','approval_decisions','events','budget_ledger','kill_switches'];
|
||||
for (const name of names) await db.$queryRawUnsafe(`SELECT * FROM orchestration_${name} LIMIT 0`);
|
||||
const rows = await db.$queryRawUnsafe("SELECT 1 FROM _prisma_migrations WHERE migration_name = '20261022000000_add_orchestration_v4_core' AND finished_at IS NOT NULL AND rolled_back_at IS NULL");
|
||||
if (rows.length !== 1) throw new Error('migration receipt');
|
||||
for (let n=0; n<30; n++) {
|
||||
try { await accept('http://127.0.0.1:3000'); console.log('Backend schema/catalog acceptance passed'); return; }
|
||||
catch { if(n===29) throw new Error('catalog acceptance'); await new Promise(r=>setTimeout(r,2000)); }
|
||||
}
|
||||
} finally { await db.$disconnect(); }
|
||||
}
|
||||
module.exports = {accept};
|
||||
// node -e is used inside the built backend; imports in tests do not execute main.
|
||||
if (!module.parent) main().catch(()=>{console.error('Backend acceptance failed (schema, migration receipt or catalog)');process.exitCode=1;});
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
# Sourced by the CI/CD deploy entrypoint, never a standalone deploy command.
|
||||
stage_rollout() {
|
||||
# Maintenance is deliberate: neither old nor new app serves during DDL.
|
||||
docker compose stop frontend_stage backend_stage
|
||||
docker compose run --rm --no-deps --entrypoint ./node_modules/.bin/prisma backend_stage migrate deploy
|
||||
docker compose up -d --no-deps backend_stage
|
||||
if ! docker compose exec -T backend_stage node -e "$(< scripts/rollout-health.cjs)"; then
|
||||
docker compose stop frontend_stage backend_stage
|
||||
echo 'ERROR: backend acceptance failed; maintenance retained' >&2
|
||||
return 1
|
||||
fi
|
||||
docker compose up -d --no-deps frontend_stage
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import subprocess as sp, tempfile, pathlib, os, time, json
|
||||
ROOT=pathlib.Path('/opt/services/canina/worktrees/orchestration-v4-rollout')
|
||||
name='canina-v4-rollout-rehearsal'
|
||||
def run(args,**kw): return sp.run(args,check=True,**kw)
|
||||
def sql(q): return sp.check_output(['docker','exec','-i',name,'psql','-U','rehearsal','-d','rehearsal','-v','ON_ERROR_STOP=1','-At'],input=q.encode()).decode()
|
||||
with tempfile.TemporaryDirectory(prefix='canina-rollout-') as d:
|
||||
os.chmod(d,0o700)
|
||||
schema=pathlib.Path(d)/'schema.sql'; history=pathlib.Path(d)/'history.sql'
|
||||
for file,args in [(schema,['--schema-only','--no-owner','--no-privileges']),(history,['--data-only','--table=public._prisma_migrations','--column-inserts'])]:
|
||||
with file.open('wb') as f:
|
||||
run(['docker','exec','canino_db_stage','sh','-c','exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"','sh',*args],stdout=f)
|
||||
print('Captured schema-only Stage dump plus _prisma_migrations history only')
|
||||
run(['docker','run','-d','--name',name,'--network','none','-e','POSTGRES_USER=rehearsal','-e','POSTGRES_DB=rehearsal','-e','POSTGRES_HOST_AUTH_METHOD=trust','postgres:16-alpine'],stdout=sp.DEVNULL)
|
||||
try:
|
||||
for _ in range(30):
|
||||
if sp.run(['docker','exec',name,'pg_isready','-U','rehearsal'],stdout=sp.DEVNULL,stderr=sp.DEVNULL).returncode==0: break
|
||||
time.sleep(1)
|
||||
for file in (schema,history):
|
||||
with file.open('rb') as f: run(['docker','exec','-i',name,'psql','-U','rehearsal','-d','rehearsal','-v','ON_ERROR_STOP=1'],stdin=f,stdout=sp.DEVNULL)
|
||||
before=sql("SELECT table_name,column_name,data_type,is_nullable,column_default FROM information_schema.columns WHERE table_schema='public' AND table_name <> '_prisma_migrations' ORDER BY 1,2;")
|
||||
tables=sql("SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename <> '_prisma_migrations' ORDER BY 1;").splitlines()
|
||||
for table in tables:
|
||||
assert sql(f'SELECT count(*) FROM "{table}";').strip()=='0',table
|
||||
print(f'Restored {len(tables)} legacy tables; all business tables empty')
|
||||
cmd=['docker','run','--rm','--network','container:'+name,'-v',str(ROOT/'backend/prisma')+':/app/prisma:ro','-v','/opt/services/canina/worktrees/orchestration-v4-core/backend/node_modules:/app/node_modules:ro','-w','/app','-e','DATABASE_URL=postgresql://rehearsal@127.0.0.1:5432/rehearsal?schema=public','node:22-bookworm','./node_modules/.bin/prisma','migrate','deploy']
|
||||
run(cmd); run(cmd)
|
||||
query = "const {PrismaClient}=require('@prisma/client');const p=new PrismaClient();(async()=>{await p.product.findMany({take:1,include:{ingredientList:true,symptoms:true}});await p.supportConversation.findMany({take:1});await p.orchestrationTask.findMany({take:1,include:{runs:true,events:true,budgetLedger:true}});console.log('PASS: actual Prisma legacy catalog/support and v4 relation queries');})().catch(()=>{console.error('Prisma compatibility failed');process.exitCode=1}).finally(()=>p.$disconnect());"
|
||||
run(cmd[:-3]+['node','-e',query])
|
||||
after=sql("SELECT table_name,column_name,data_type,is_nullable,column_default FROM information_schema.columns WHERE table_schema='public' AND table_name <> '_prisma_migrations' AND table_name NOT LIKE 'orchestration_%' ORDER BY 1,2;")
|
||||
assert before==after,'Legacy column definitions changed'
|
||||
for table in tables: sql(f'SELECT * FROM "{table}" LIMIT 0;')
|
||||
v4=sql("SELECT tablename FROM pg_tables WHERE schemaname='public' AND tablename LIKE 'orchestration_%' ORDER BY 1;").splitlines()
|
||||
assert len(v4)==9,v4
|
||||
for table in v4: sql(f'SELECT * FROM "{table}" LIMIT 0;')
|
||||
print(f'PASS: unchanged legacy column definitions; {len(tables)} legacy queries; {len(v4)} v4 tables; migrate deploy repeatable')
|
||||
finally:
|
||||
run(['docker','rm','-fv',name],stdout=sp.DEVNULL)
|
||||
print('Disposable database/container removed; no live DDL performed')
|
||||
@@ -0,0 +1,11 @@
|
||||
const {test}=require('node:test');
|
||||
const assert=require('node:assert/strict');
|
||||
const http=require('node:http');
|
||||
const {accept}=require('../rollout-health.cjs');
|
||||
for(const [name,status,body,pass] of [['bad status',503,'{}',false],['fake html',200,'<html>ok</html>',false],['wrong envelope',200,'{}',false],['valid empty catalog',200,'{"data":[],"meta":{"total":0}}',true]]) {
|
||||
test(name,async()=>{
|
||||
const s=http.createServer((q,r)=>{r.writeHead(status,{'content-type':'application/json'});r.end(body)});
|
||||
await new Promise(r=>s.listen(0,'127.0.0.1',r));
|
||||
try { const p=accept(`http://127.0.0.1:${s.address().port}`); if(pass) await p; else await assert.rejects(p); } finally {await new Promise(r=>s.close(r));}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import os, pathlib, subprocess, tempfile, unittest
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
class RolloutTests(unittest.TestCase):
|
||||
def test_non_ci_and_production_refused_before_docker(self):
|
||||
script=(ROOT/'scripts/deploy.sh').read_text()
|
||||
self.assertIn('CI/CD-only Stage rollout',script)
|
||||
guard=script.split('# END ROLLOUT GUARD')[0]
|
||||
for args,env in [(['develop'],{}),(['develop','stage-backend-only'],{'GITHUB_ACTIONS':'true'}),(['main'],{'GITHUB_ACTIONS':'true'})]:
|
||||
r=subprocess.run(['bash','-c',guard,'deploy.sh',*args],cwd=ROOT,env=dict(os.environ,**env),capture_output=True,text=True)
|
||||
self.assertNotEqual(r.returncode,0)
|
||||
self.assertIn('refusing',r.stderr)
|
||||
def test_migration_failure_never_starts_apps(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
docker = pathlib.Path(d)/'docker'
|
||||
docker.write_text('#!/bin/bash\nprintf "%s\\n" "$*" >> "$TRACE"\nif [[ "$*" == *"run --rm"* ]]; then exit 42; fi\n')
|
||||
docker.chmod(0o755)
|
||||
env = dict(os.environ, PATH=d+':'+os.environ['PATH'], TRACE=d+'/trace')
|
||||
result = subprocess.run(['bash','-c', 'set -e; source scripts/rollout-stage.sh; stage_rollout; echo SUCCESS'], cwd=ROOT,env=env,capture_output=True,text=True)
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
trace=pathlib.Path(d+'/trace').read_text() if pathlib.Path(d+'/trace').exists() else ''
|
||||
self.assertIn('stop frontend_stage backend_stage',trace)
|
||||
self.assertIn('run --rm --no-deps --entrypoint',trace)
|
||||
self.assertNotIn('up -d',trace)
|
||||
self.assertNotIn('SUCCESS',result.stdout)
|
||||
def test_health_failure_stops_backend_and_does_not_publish_frontend(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
docker = pathlib.Path(d)/'docker'
|
||||
docker.write_text('#!/bin/bash\nprintf "%s\\n" "$*" >> "$TRACE"\nif [[ "$*" == *"exec -T backend_stage node"* ]]; then exit 43; fi\n')
|
||||
docker.chmod(0o755)
|
||||
result = subprocess.run(['bash','-c','set -e; source scripts/rollout-stage.sh; stage_rollout; echo SUCCESS'],cwd=ROOT,env=dict(os.environ,PATH=d+':'+os.environ['PATH'],TRACE=d+'/trace'),capture_output=True,text=True)
|
||||
self.assertNotEqual(result.returncode,0)
|
||||
trace=pathlib.Path(d+'/trace').read_text()
|
||||
self.assertIn('stop frontend_stage backend_stage',trace)
|
||||
self.assertIn('exec -T backend_stage node',trace)
|
||||
self.assertNotIn('up -d --no-deps frontend_stage',trace)
|
||||
self.assertNotIn('SUCCESS',result.stdout)
|
||||
if __name__=='__main__': unittest.main()
|
||||
Reference in new issue
Block a user