feat(ci): gate Stage corporate migration on durable verified backup
Deploy Canina / deploy (push) Has been cancelled
Deploy Canina / deploy (push) Has been cancelled
This commit is contained in:
1 parent
435a75a442
commit
b642833b8b
4 files changed
+341
No files matched your search
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local-only real PG transaction tests and Docker transport fault injection."""
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
spec = importlib.util.spec_from_file_location('migration', Path(__file__).with_name('corporate-stage-migrate.py'))
|
||||
assert spec is not None and spec.loader is not None
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
SQL = (m.ROOT / f'backend/prisma/migrations/{m.NAME}/migration.sql').read_bytes()
|
||||
|
||||
|
||||
class Transport(unittest.TestCase):
|
||||
def test_gate_faults_and_order(self):
|
||||
for failure in ('backup', 'restore', 'clone-apply', 'digest', None):
|
||||
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as td:
|
||||
root = Path(td)
|
||||
rel = f'backend/prisma/migrations/{m.NAME}/migration.sql'
|
||||
(root / rel).parent.mkdir(parents=True)
|
||||
(root / rel).write_bytes(SQL)
|
||||
(root / '.release-manifest.json').write_text(json.dumps({
|
||||
'sha': 'a' * 40, 'files': {rel: m.hashlib.sha256(SQL).hexdigest()}}))
|
||||
events = []
|
||||
digest_reads = 0
|
||||
|
||||
def fake(*args, data=None, timeout=600):
|
||||
nonlocal digest_reads
|
||||
if args[0] == 'inspect':
|
||||
return json.dumps({'Name': '/' + m.DB, 'State': {'Running': True},
|
||||
'Image': 'sha256:' + 'b' * 64, 'Mounts': [{'Destination':
|
||||
'/var/lib/postgresql/data', 'RW': True, 'Type': 'volume'}]}).encode()
|
||||
if args[0] == 'create':
|
||||
events.append('create'); return b'c' * 64
|
||||
if args[0] == 'rm':
|
||||
events.append('cleanup'); return b''
|
||||
if args[0] == 'exec' and args[2] == m.DB:
|
||||
cmd = args[-1]
|
||||
if 'pg_dump' in cmd:
|
||||
events.append('backup')
|
||||
if failure == 'backup': raise RuntimeError('injected')
|
||||
elif 'sha256sum' in cmd:
|
||||
digest_reads += 1
|
||||
return (('f' if failure == 'digest' and digest_reads == 2 else 'd') * 64 + ' file').encode()
|
||||
elif 'psql' in cmd:
|
||||
events.append('live')
|
||||
return b''
|
||||
if args[0] == 'exec' and 'psql' in args:
|
||||
events.append('clone-apply')
|
||||
if failure == 'clone-apply': raise RuntimeError('injected')
|
||||
return b''
|
||||
|
||||
class Source:
|
||||
stdout = io.BytesIO()
|
||||
def wait(self, timeout=None): return 0
|
||||
def poll(self): return 0
|
||||
class Result:
|
||||
returncode = 1 if failure == 'restore' else 0
|
||||
def restore(*args, **kwargs):
|
||||
events.append('restore'); return Result()
|
||||
with patch.object(m, 'ROOT', root), patch.object(m, 'docker', fake), patch.dict(
|
||||
os.environ, GITHUB_ACTIONS='true', DEPLOY_SHA='a' * 40), patch.object(
|
||||
m.subprocess, 'Popen', return_value=Source()), patch.object(m.subprocess, 'run', restore):
|
||||
if failure:
|
||||
with self.assertRaises(RuntimeError): m.run()
|
||||
self.assertNotIn('live', events)
|
||||
else:
|
||||
m.run()
|
||||
self.assertEqual(events, ['backup', 'create', 'restore', 'clone-apply', 'clone-apply', 'live', 'cleanup'])
|
||||
if 'create' in events: self.assertEqual(events[-1], 'cleanup')
|
||||
|
||||
def test_pipeline_order(self):
|
||||
source = Path(__file__).with_name('release-cache-stage.sh').read_text()
|
||||
self.assertLess(source.index('build backend_stage frontend_stage'), source.index('python3 scripts/corporate-stage-migrate.py'))
|
||||
self.assertLess(source.index('python3 scripts/corporate-stage-migrate.py'), source.index('up -d --no-deps --force-recreate'))
|
||||
|
||||
|
||||
class RealPostgres(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.server = Path(os.environ.get('PG_SERVER_BIN', '/opt/data/cache/scratch/seo-pg-bin/bin'))
|
||||
cls.client = Path(os.environ.get('PG_CLIENT_BIN', '/opt/data/cache/scratch/corporate-pg-client/usr/lib/postgresql/16/bin'))
|
||||
cls.temp = tempfile.TemporaryDirectory(prefix='cm-', dir='/opt/data/cache/scratch')
|
||||
cls.root = Path(cls.temp.name)
|
||||
cls.env = dict(os.environ, LD_LIBRARY_PATH=str(cls.server.parent / 'lib'), PGHOST=str(cls.root), PGPORT='16439', PGDATABASE='postgres')
|
||||
subprocess.run([str(cls.server / 'initdb'), '-D', str(cls.root / 'data'), '-A', 'trust', '--no-locale'], env=cls.env, check=True, stdout=subprocess.DEVNULL)
|
||||
subprocess.run([str(cls.server / 'pg_ctl'), '-D', str(cls.root / 'data'), '-l', str(cls.root / 'server.log'), '-o', f'-k {cls.root} -p 16439 -h ""', '-w', 'start'], env=cls.env, check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
subprocess.run([str(cls.server / 'pg_ctl'), '-D', str(cls.root / 'data'), '-m', 'immediate', '-w', 'stop'], env=cls.env, check=True, stdout=subprocess.DEVNULL)
|
||||
cls.temp.cleanup()
|
||||
|
||||
def query(self, sql, success=True):
|
||||
result = subprocess.run([str(self.client / 'psql'), '-X', '-At', '-v', 'ON_ERROR_STOP=1'], input=sql, env=self.env, capture_output=True)
|
||||
self.assertEqual(result.returncode == 0, success, result.stderr.decode())
|
||||
return result.stdout.decode().strip()
|
||||
|
||||
def test_real_transaction_backup_restore(self):
|
||||
self.assertIn('16.2', self.query(b'SELECT version();'))
|
||||
self.query(b'''CREATE TABLE _prisma_migrations(id varchar(36) PRIMARY KEY, checksum varchar(64) NOT NULL, finished_at timestamptz, migration_name varchar(255) NOT NULL, logs text, rolled_back_at timestamptz, started_at timestamptz NOT NULL DEFAULT now(), applied_steps_count integer NOT NULL DEFAULT 0); CREATE TABLE sentinel(id int, value text); INSERT INTO sentinel VALUES (1, 'preserved');''')
|
||||
backup = self.root / 'full.dump'
|
||||
subprocess.run([str(self.client / 'pg_dump'), '-Fc', '-f', str(backup)], env=self.env, check=True)
|
||||
self.assertGreater(backup.stat().st_size, 0)
|
||||
self.query(m.transaction(SQL + b'\nCREATE TABLE must_rollback(id int); SELECT 1/0;'), False)
|
||||
self.assertEqual(self.query(b"SELECT count(*) FROM pg_tables WHERE tablename LIKE 'corporate_%' OR tablename='must_rollback';"), '0')
|
||||
self.assertEqual(self.query(b'SELECT count(*) FROM _prisma_migrations;'), '0')
|
||||
# Hold the exact migration lock in a distinct backend; fail closed on contention.
|
||||
holder = subprocess.Popen([str(self.client / 'psql'), '-X', '-At'],
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.env)
|
||||
assert holder.stdin is not None and holder.stdout is not None
|
||||
holder.stdin.write(b'BEGIN; SELECT pg_advisory_xact_lock(72707369); SELECT 42; SELECT pg_sleep(2); COMMIT;\n')
|
||||
holder.stdin.close()
|
||||
while holder.stdout.readline().strip() != b'42':
|
||||
if holder.poll() is not None: self.fail('Lock fixture failed')
|
||||
self.query(m.transaction(SQL).replace(b"'30s'", b"'200ms'"), False)
|
||||
holder.wait(timeout=10)
|
||||
holder.stdout.close()
|
||||
assert holder.stderr is not None
|
||||
holder.stderr.close()
|
||||
self.query(m.transaction(SQL))
|
||||
self.query(m.transaction(SQL))
|
||||
self.assertEqual(self.query(b'SELECT count(*) FROM _prisma_migrations;'), '1')
|
||||
self.query(m.transaction(SQL + b'\n-- changed checksum'), False)
|
||||
self.query(b"UPDATE _prisma_migrations SET finished_at=NULL;")
|
||||
self.query(m.transaction(SQL), False)
|
||||
self.query(b"CREATE DATABASE restored;")
|
||||
subprocess.run([str(self.client / 'pg_restore'), '--exit-on-error', '--single-transaction', '-d', 'restored', str(backup)], env=self.env, check=True)
|
||||
previous = self.env
|
||||
self.env = dict(self.env, PGDATABASE='restored')
|
||||
self.assertEqual(self.query(b'SELECT value FROM sentinel;'), 'preserved')
|
||||
self.assertEqual(self.query(b'SELECT count(*) FROM _prisma_migrations;'), '0')
|
||||
self.query(m.transaction(SQL))
|
||||
self.query(m.transaction(SQL))
|
||||
self.env = previous
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,21 @@
|
||||
# Controlled corporate Stage migration
|
||||
|
||||
## Proportional dependency graph / specification (before implementation)
|
||||
|
||||
Existing graph query (`graphify query 'release-cache-stage deploy build migration Prisma' --budget 1200`) locates deploy.sh → Stage Compose and backend scripts → Prisma; release-cache-stage.sh is newer than that graph. Direct inspection establishes the exact execution edge:
|
||||
|
||||
`immutable SHA archive → runtime validation → build backend_stage + frontend_stage → [new gate] → recreate apps → readiness`
|
||||
|
||||
New gate: `CI identity → stage DB pinned container/image/volume → pg_dump custom → fsync durable DB-volume backup + digest → disposable same-image PG restore → transaction apply/replay on clone → transaction apply on Stage → app recreate`.
|
||||
|
||||
Only scripts/release-cache-stage.sh and corporate-stage-migrate* are owned. The migration SQL is read without modification. No checkout reset, production commands, cron or Hermes changes. This implementation is not a deployment authorization.
|
||||
|
||||
Read-only Stage inspection found standard public._prisma_migrations columns: id varchar, checksum varchar, finished_at timestamptz nullable, migration_name varchar, logs text nullable, rolled_back_at timestamptz nullable, started_at timestamptz default now(), applied_steps_count integer default 0. The data mount is /var/lib/postgresql/data. Credentials are expanded only by the container shell.
|
||||
|
||||
## Safety contract
|
||||
|
||||
Fail closed on non-CI invocation, invalid SHA, unexpected database mount, backup/restore/readiness errors, unresolved Prisma migrations, duplicate or mismatching migration checksum. Keep full custom-format backup under the existing Stage data volume in canina-stage-backups; sync file and directory before restore verification. Never delete backup artifacts automatically. Restore to a no-network disposable same-image database with tmpfs storage; remove only the captured owned container ID. Use a bounded transaction-level advisory lock (Prisma key 72707369), ON_ERROR_STOP and a single transaction for DDL plus standard Prisma bookkeeping. Existing successful checksum-matching entry is a no-op. DDL errors disconnect/rollback; never mark failed work as applied. Test real PostgreSQL plus fake Docker orchestration transport locally; live server interaction remains read-only.
|
||||
|
||||
## Operations
|
||||
|
||||
The release invokes `python3 scripts/corporate-stage-migrate.py` after both builds and before either app is recreated, under the existing supervised host lock. It needs Docker, Python 3 and the existing Stage DB image with PostgreSQL tools. Successful output contains backup path/digest and release SHA, never credentials. Backups contain sensitive full database content, are mode 0600 in a mode 0700 directory, and require separate operator retention policy. Restore test uses an isolated postgres superuser and restores all data/schema without source ownership or ACLs; it proves data recoverability and DDL compatibility, not source role/ACL recovery. Live access uses the source role resolved internally, never logged. Live writes during pg_dump are permitted by PostgreSQL's consistent snapshot; this additive migration does not provide a rollback of subsequent application writes. No automatic live restore is performed.
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CI-only Stage gate; never calls Prisma deploy or touches another database."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
DB = 'canino_db_stage'
|
||||
NAME = '20261028120000_corporate_orchestration'
|
||||
BACKUPS = '/var/lib/postgresql/data/canina-stage-backups'
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def docker(*args, data=None, timeout=600):
|
||||
# Do not relay stderr: tools may expose connection details or row data.
|
||||
result = subprocess.run(['docker', *args], input=data, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, timeout=timeout)
|
||||
if result.returncode:
|
||||
raise RuntimeError('Docker operation failed: ' + args[0])
|
||||
return result.stdout
|
||||
|
||||
|
||||
def stage(command, data=None):
|
||||
# Expansion occurs only inside the existing DB container. No env inspection.
|
||||
return docker('exec', '-i', DB, 'sh', '-ec', command, data=data)
|
||||
|
||||
|
||||
def transaction(sql):
|
||||
checksum = hashlib.sha256(sql).hexdigest()
|
||||
# One connection, one transaction; client disconnect rolls back on all errors.
|
||||
before = f'''\\set ON_ERROR_STOP on
|
||||
BEGIN;
|
||||
SET LOCAL lock_timeout = '30s';
|
||||
SET LOCAL statement_timeout = '300s';
|
||||
SET LOCAL search_path = public, pg_catalog;
|
||||
SELECT pg_advisory_xact_lock(72707369);
|
||||
LOCK TABLE public._prisma_migrations IN SHARE ROW EXCLUSIVE MODE;
|
||||
DO $guard$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM public._prisma_migrations
|
||||
WHERE finished_at IS NULL AND rolled_back_at IS NULL) THEN
|
||||
RAISE EXCEPTION 'Unresolved Prisma migration';
|
||||
END IF;
|
||||
IF (SELECT count(*) FROM public._prisma_migrations WHERE migration_name = '{NAME}') > 1
|
||||
OR EXISTS (SELECT 1 FROM public._prisma_migrations WHERE migration_name = '{NAME}'
|
||||
AND (checksum <> '{checksum}' OR finished_at IS NULL
|
||||
OR rolled_back_at IS NOT NULL OR applied_steps_count <> 1)) THEN
|
||||
RAISE EXCEPTION 'Corporate migration history mismatch';
|
||||
END IF;
|
||||
END $guard$;
|
||||
SELECT NOT EXISTS (SELECT 1 FROM public._prisma_migrations
|
||||
WHERE migration_name = '{NAME}') AS apply_needed \\gset
|
||||
\\if :apply_needed
|
||||
'''
|
||||
after = f'''
|
||||
INSERT INTO public._prisma_migrations
|
||||
(id, checksum, finished_at, migration_name, logs, rolled_back_at, started_at, applied_steps_count)
|
||||
VALUES ('{uuid.uuid4()}', '{checksum}', now(), '{NAME}', NULL, NULL, now(), 1);
|
||||
\\endif
|
||||
COMMIT;
|
||||
'''
|
||||
return before.encode() + sql + after.encode()
|
||||
|
||||
|
||||
def apply_stage(payload):
|
||||
stage('export PGPASSWORD="${POSTGRES_PASSWORD:-}"; exec psql -X -q -v ON_ERROR_STOP=1 '
|
||||
'-U "$POSTGRES_USER" -d "$POSTGRES_DB"', payload)
|
||||
|
||||
|
||||
def run():
|
||||
sha = os.environ.get('DEPLOY_SHA', '')
|
||||
if (os.environ.get('GITHUB_ACTIONS', os.environ.get('GITEA_ACTIONS')) != 'true'
|
||||
or not re.fullmatch('[0-9a-f]{40}', sha)):
|
||||
raise RuntimeError('CI SHA identity required')
|
||||
relative = f'backend/prisma/migrations/{NAME}/migration.sql'
|
||||
sql = (ROOT / relative).read_bytes()
|
||||
manifest = json.loads((ROOT / '.release-manifest.json').read_text())
|
||||
if manifest['sha'] != sha or manifest['files'][relative] != hashlib.sha256(sql).hexdigest():
|
||||
raise RuntimeError('Release migration provenance mismatch')
|
||||
# Fixed target, running state and exact durable mount: no configurable DB target.
|
||||
info = json.loads(docker('inspect', '--format',
|
||||
'{"Name":{{json .Name}},"State":{"Running":{{json .State.Running}}},'
|
||||
'"Image":{{json .Image}},"Mounts":{{json .Mounts}}}', DB))
|
||||
if info['Name'] != '/' + DB or not info['State']['Running']:
|
||||
raise RuntimeError('Stage database is not running')
|
||||
if not any(m['Destination'] == '/var/lib/postgresql/data'
|
||||
and m.get('RW') and m['Type'] in ('volume', 'bind') for m in info['Mounts']):
|
||||
raise RuntimeError('Stage durable data mount missing')
|
||||
image = info['Image']
|
||||
if not re.fullmatch('sha256:[0-9a-f]{64}', image):
|
||||
raise RuntimeError('Stage image identity missing')
|
||||
attempt = uuid.uuid4().hex
|
||||
backup = f'{BACKUPS}/{sha}.{attempt}.dump'
|
||||
# Never use runner filesystem as the only durable copy. Retain failed attempts.
|
||||
stage(f'umask 077; mkdir -p {BACKUPS}; chmod 700 {BACKUPS}; '
|
||||
f'test ! -e {backup}; export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; '
|
||||
f'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f {backup}; '
|
||||
f'test -s {backup}; pg_restore --list {backup} >/dev/null; '
|
||||
f'chmod 600 {backup}; sync')
|
||||
digest = stage(f'sha256sum {backup}').decode().split()[0]
|
||||
if not re.fullmatch('[0-9a-f]{64}', digest):
|
||||
raise RuntimeError('Backup digest missing')
|
||||
clone = None
|
||||
try:
|
||||
clone = docker('create', '--name', 'canina-stage-migration-verify-' + attempt,
|
||||
'--pull=never', '--network', 'none', '--tmpfs',
|
||||
'/var/lib/postgresql/data:rw', '-e', 'POSTGRES_HOST_AUTH_METHOD=trust',
|
||||
'-e', 'POSTGRES_DB=verify', image).decode().strip()
|
||||
if not re.fullmatch('[0-9a-f]{12,64}', clone):
|
||||
raise RuntimeError('Restore container identity missing')
|
||||
docker('start', clone)
|
||||
for _ in range(60):
|
||||
try:
|
||||
# TCP ensures the official image's temporary init server has ended.
|
||||
docker('exec', clone, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres',
|
||||
'-d', 'verify', timeout=10)
|
||||
break
|
||||
except RuntimeError:
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError('Restore server readiness timeout')
|
||||
# Stream full backup, never store its sensitive contents in runner logs/memory.
|
||||
source = subprocess.Popen(['docker', 'exec', DB, 'cat', backup], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
assert source.stdout is not None
|
||||
restored = subprocess.run(['docker', 'exec', '-i', clone, 'pg_restore',
|
||||
'--exit-on-error', '--single-transaction', '--no-owner',
|
||||
'--no-privileges', '-U', 'postgres', '-d', 'verify'],
|
||||
stdin=source.stdout, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=600)
|
||||
source.stdout.close()
|
||||
if source.wait(timeout=30) or restored.returncode:
|
||||
raise RuntimeError('Full backup restore failed')
|
||||
finally:
|
||||
if source.poll() is None:
|
||||
source.kill()
|
||||
source.wait()
|
||||
payload = transaction(sql)
|
||||
for _ in range(2):
|
||||
docker('exec', '-i', clone, 'psql', '-X', '-q', '-v', 'ON_ERROR_STOP=1',
|
||||
'-U', 'postgres', '-d', 'verify', data=payload)
|
||||
# Verify persisted bytes again immediately before allowing live DDL.
|
||||
if stage(f'sha256sum {backup}').decode().split()[0] != digest:
|
||||
raise RuntimeError('Durable backup changed')
|
||||
apply_stage(payload)
|
||||
print(f'Stage corporate migration verified: sha={sha} backup={backup} sha256={digest}')
|
||||
finally:
|
||||
if clone:
|
||||
docker('rm', '-f', clone, timeout=30)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
def interrupted(signum, frame):
|
||||
raise RuntimeError('Migration gate interrupted')
|
||||
|
||||
signal.signal(signal.SIGTERM, interrupted)
|
||||
signal.signal(signal.SIGINT, interrupted)
|
||||
try:
|
||||
run()
|
||||
except Exception as error:
|
||||
# Exception text from subprocess/JSON could include sensitive output.
|
||||
print('Stage corporate migration gate failed (' + type(error).__name__ +
|
||||
'); retain backup and inspect privately; apps not recreated', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -162,6 +162,10 @@ python3 scripts/verify-stage-runtime.py --base-compose "$STAGE_DIR/docker-compos
|
||||
check_stage_lock
|
||||
timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_stage frontend_stage
|
||||
check_stage_lock
|
||||
# CI-owned additive Stage migration: durable full backup + disposable restore
|
||||
# verification must succeed before either application container is recreated.
|
||||
timeout --foreground --kill-after=10s 1800 python3 scripts/corporate-stage-migrate.py
|
||||
check_stage_lock
|
||||
timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_stage frontend_stage
|
||||
for name in canino_backend_stage canino_frontend_stage; do
|
||||
# Every daemon interaction is bounded, including a hung exec/inspect.
|
||||
|
||||
Reference in new issue
Block a user