feat(ci): gate Stage corporate migration on durable verified backup
Deploy Canina / deploy (push) Has been cancelled

This commit is contained in:
Canina Automation committed 2026-10-28 12:58:56 +00:00
1 parent 435a75a442
commit b642833b8b
4 files changed
+341

No files matched your search

+145
View File
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""Local-only real PG transaction tests and Docker transport fault injection."""
import importlib.util
import io
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('migration', Path(__file__).with_name('corporate-stage-migrate.py'))
assert spec is not None and spec.loader is not None
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
SQL = (m.ROOT / f'backend/prisma/migrations/{m.NAME}/migration.sql').read_bytes()
class Transport(unittest.TestCase):
def test_gate_faults_and_order(self):
for failure in ('backup', 'restore', 'clone-apply', 'digest', None):
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as td:
root = Path(td)
rel = f'backend/prisma/migrations/{m.NAME}/migration.sql'
(root / rel).parent.mkdir(parents=True)
(root / rel).write_bytes(SQL)
(root / '.release-manifest.json').write_text(json.dumps({
'sha': 'a' * 40, 'files': {rel: m.hashlib.sha256(SQL).hexdigest()}}))
events = []
digest_reads = 0
def fake(*args, data=None, timeout=600):
nonlocal digest_reads
if args[0] == 'inspect':
return json.dumps({'Name': '/' + m.DB, 'State': {'Running': True},
'Image': 'sha256:' + 'b' * 64, 'Mounts': [{'Destination':
'/var/lib/postgresql/data', 'RW': True, 'Type': 'volume'}]}).encode()
if args[0] == 'create':
events.append('create'); return b'c' * 64
if args[0] == 'rm':
events.append('cleanup'); return b''
if args[0] == 'exec' and args[2] == m.DB:
cmd = args[-1]
if 'pg_dump' in cmd:
events.append('backup')
if failure == 'backup': raise RuntimeError('injected')
elif 'sha256sum' in cmd:
digest_reads += 1
return (('f' if failure == 'digest' and digest_reads == 2 else 'd') * 64 + ' file').encode()
elif 'psql' in cmd:
events.append('live')
return b''
if args[0] == 'exec' and 'psql' in args:
events.append('clone-apply')
if failure == 'clone-apply': raise RuntimeError('injected')
return b''
class Source:
stdout = io.BytesIO()
def wait(self, timeout=None): return 0
def poll(self): return 0
class Result:
returncode = 1 if failure == 'restore' else 0
def restore(*args, **kwargs):
events.append('restore'); return Result()
with patch.object(m, 'ROOT', root), patch.object(m, 'docker', fake), patch.dict(
os.environ, GITHUB_ACTIONS='true', DEPLOY_SHA='a' * 40), patch.object(
m.subprocess, 'Popen', return_value=Source()), patch.object(m.subprocess, 'run', restore):
if failure:
with self.assertRaises(RuntimeError): m.run()
self.assertNotIn('live', events)
else:
m.run()
self.assertEqual(events, ['backup', 'create', 'restore', 'clone-apply', 'clone-apply', 'live', 'cleanup'])
if 'create' in events: self.assertEqual(events[-1], 'cleanup')
def test_pipeline_order(self):
source = Path(__file__).with_name('release-cache-stage.sh').read_text()
self.assertLess(source.index('build backend_stage frontend_stage'), source.index('python3 scripts/corporate-stage-migrate.py'))
self.assertLess(source.index('python3 scripts/corporate-stage-migrate.py'), source.index('up -d --no-deps --force-recreate'))
class RealPostgres(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.server = Path(os.environ.get('PG_SERVER_BIN', '/opt/data/cache/scratch/seo-pg-bin/bin'))
cls.client = Path(os.environ.get('PG_CLIENT_BIN', '/opt/data/cache/scratch/corporate-pg-client/usr/lib/postgresql/16/bin'))
cls.temp = tempfile.TemporaryDirectory(prefix='cm-', dir='/opt/data/cache/scratch')
cls.root = Path(cls.temp.name)
cls.env = dict(os.environ, LD_LIBRARY_PATH=str(cls.server.parent / 'lib'), PGHOST=str(cls.root), PGPORT='16439', PGDATABASE='postgres')
subprocess.run([str(cls.server / 'initdb'), '-D', str(cls.root / 'data'), '-A', 'trust', '--no-locale'], env=cls.env, check=True, stdout=subprocess.DEVNULL)
subprocess.run([str(cls.server / 'pg_ctl'), '-D', str(cls.root / 'data'), '-l', str(cls.root / 'server.log'), '-o', f'-k {cls.root} -p 16439 -h ""', '-w', 'start'], env=cls.env, check=True, stdout=subprocess.DEVNULL)
@classmethod
def tearDownClass(cls):
subprocess.run([str(cls.server / 'pg_ctl'), '-D', str(cls.root / 'data'), '-m', 'immediate', '-w', 'stop'], env=cls.env, check=True, stdout=subprocess.DEVNULL)
cls.temp.cleanup()
def query(self, sql, success=True):
result = subprocess.run([str(self.client / 'psql'), '-X', '-At', '-v', 'ON_ERROR_STOP=1'], input=sql, env=self.env, capture_output=True)
self.assertEqual(result.returncode == 0, success, result.stderr.decode())
return result.stdout.decode().strip()
def test_real_transaction_backup_restore(self):
self.assertIn('16.2', self.query(b'SELECT version();'))
self.query(b'''CREATE TABLE _prisma_migrations(id varchar(36) PRIMARY KEY, checksum varchar(64) NOT NULL, finished_at timestamptz, migration_name varchar(255) NOT NULL, logs text, rolled_back_at timestamptz, started_at timestamptz NOT NULL DEFAULT now(), applied_steps_count integer NOT NULL DEFAULT 0); CREATE TABLE sentinel(id int, value text); INSERT INTO sentinel VALUES (1, 'preserved');''')
backup = self.root / 'full.dump'
subprocess.run([str(self.client / 'pg_dump'), '-Fc', '-f', str(backup)], env=self.env, check=True)
self.assertGreater(backup.stat().st_size, 0)
self.query(m.transaction(SQL + b'\nCREATE TABLE must_rollback(id int); SELECT 1/0;'), False)
self.assertEqual(self.query(b"SELECT count(*) FROM pg_tables WHERE tablename LIKE 'corporate_%' OR tablename='must_rollback';"), '0')
self.assertEqual(self.query(b'SELECT count(*) FROM _prisma_migrations;'), '0')
# Hold the exact migration lock in a distinct backend; fail closed on contention.
holder = subprocess.Popen([str(self.client / 'psql'), '-X', '-At'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.env)
assert holder.stdin is not None and holder.stdout is not None
holder.stdin.write(b'BEGIN; SELECT pg_advisory_xact_lock(72707369); SELECT 42; SELECT pg_sleep(2); COMMIT;\n')
holder.stdin.close()
while holder.stdout.readline().strip() != b'42':
if holder.poll() is not None: self.fail('Lock fixture failed')
self.query(m.transaction(SQL).replace(b"'30s'", b"'200ms'"), False)
holder.wait(timeout=10)
holder.stdout.close()
assert holder.stderr is not None
holder.stderr.close()
self.query(m.transaction(SQL))
self.query(m.transaction(SQL))
self.assertEqual(self.query(b'SELECT count(*) FROM _prisma_migrations;'), '1')
self.query(m.transaction(SQL + b'\n-- changed checksum'), False)
self.query(b"UPDATE _prisma_migrations SET finished_at=NULL;")
self.query(m.transaction(SQL), False)
self.query(b"CREATE DATABASE restored;")
subprocess.run([str(self.client / 'pg_restore'), '--exit-on-error', '--single-transaction', '-d', 'restored', str(backup)], env=self.env, check=True)
previous = self.env
self.env = dict(self.env, PGDATABASE='restored')
self.assertEqual(self.query(b'SELECT value FROM sentinel;'), 'preserved')
self.assertEqual(self.query(b'SELECT count(*) FROM _prisma_migrations;'), '0')
self.query(m.transaction(SQL))
self.query(m.transaction(SQL))
self.env = previous
if __name__ == '__main__':
unittest.main(verbosity=2)
+21
View File
@@ -0,0 +1,21 @@
# Controlled corporate Stage migration
## Proportional dependency graph / specification (before implementation)
Existing graph query (`graphify query 'release-cache-stage deploy build migration Prisma' --budget 1200`) locates deploy.sh → Stage Compose and backend scripts → Prisma; release-cache-stage.sh is newer than that graph. Direct inspection establishes the exact execution edge:
`immutable SHA archive → runtime validation → build backend_stage + frontend_stage → [new gate] → recreate apps → readiness`
New gate: `CI identity → stage DB pinned container/image/volume → pg_dump custom → fsync durable DB-volume backup + digest → disposable same-image PG restore → transaction apply/replay on clone → transaction apply on Stage → app recreate`.
Only scripts/release-cache-stage.sh and corporate-stage-migrate* are owned. The migration SQL is read without modification. No checkout reset, production commands, cron or Hermes changes. This implementation is not a deployment authorization.
Read-only Stage inspection found standard public._prisma_migrations columns: id varchar, checksum varchar, finished_at timestamptz nullable, migration_name varchar, logs text nullable, rolled_back_at timestamptz nullable, started_at timestamptz default now(), applied_steps_count integer default 0. The data mount is /var/lib/postgresql/data. Credentials are expanded only by the container shell.
## Safety contract
Fail closed on non-CI invocation, invalid SHA, unexpected database mount, backup/restore/readiness errors, unresolved Prisma migrations, duplicate or mismatching migration checksum. Keep full custom-format backup under the existing Stage data volume in canina-stage-backups; sync file and directory before restore verification. Never delete backup artifacts automatically. Restore to a no-network disposable same-image database with tmpfs storage; remove only the captured owned container ID. Use a bounded transaction-level advisory lock (Prisma key 72707369), ON_ERROR_STOP and a single transaction for DDL plus standard Prisma bookkeeping. Existing successful checksum-matching entry is a no-op. DDL errors disconnect/rollback; never mark failed work as applied. Test real PostgreSQL plus fake Docker orchestration transport locally; live server interaction remains read-only.
## Operations
The release invokes `python3 scripts/corporate-stage-migrate.py` after both builds and before either app is recreated, under the existing supervised host lock. It needs Docker, Python 3 and the existing Stage DB image with PostgreSQL tools. Successful output contains backup path/digest and release SHA, never credentials. Backups contain sensitive full database content, are mode 0600 in a mode 0700 directory, and require separate operator retention policy. Restore test uses an isolated postgres superuser and restores all data/schema without source ownership or ACLs; it proves data recoverability and DDL compatibility, not source role/ACL recovery. Live access uses the source role resolved internally, never logged. Live writes during pg_dump are permitted by PostgreSQL's consistent snapshot; this additive migration does not provide a rollback of subsequent application writes. No automatic live restore is performed.
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""CI-only Stage gate; never calls Prisma deploy or touches another database."""
import hashlib
import json
import os
from pathlib import Path
import re
import signal
import subprocess
import sys
import time
import uuid
DB = 'canino_db_stage'
NAME = '20261028120000_corporate_orchestration'
BACKUPS = '/var/lib/postgresql/data/canina-stage-backups'
ROOT = Path(__file__).resolve().parent.parent
def docker(*args, data=None, timeout=600):
# Do not relay stderr: tools may expose connection details or row data.
result = subprocess.run(['docker', *args], input=data, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, timeout=timeout)
if result.returncode:
raise RuntimeError('Docker operation failed: ' + args[0])
return result.stdout
def stage(command, data=None):
# Expansion occurs only inside the existing DB container. No env inspection.
return docker('exec', '-i', DB, 'sh', '-ec', command, data=data)
def transaction(sql):
checksum = hashlib.sha256(sql).hexdigest()
# One connection, one transaction; client disconnect rolls back on all errors.
before = f'''\\set ON_ERROR_STOP on
BEGIN;
SET LOCAL lock_timeout = '30s';
SET LOCAL statement_timeout = '300s';
SET LOCAL search_path = public, pg_catalog;
SELECT pg_advisory_xact_lock(72707369);
LOCK TABLE public._prisma_migrations IN SHARE ROW EXCLUSIVE MODE;
DO $guard$
BEGIN
IF EXISTS (SELECT 1 FROM public._prisma_migrations
WHERE finished_at IS NULL AND rolled_back_at IS NULL) THEN
RAISE EXCEPTION 'Unresolved Prisma migration';
END IF;
IF (SELECT count(*) FROM public._prisma_migrations WHERE migration_name = '{NAME}') > 1
OR EXISTS (SELECT 1 FROM public._prisma_migrations WHERE migration_name = '{NAME}'
AND (checksum <> '{checksum}' OR finished_at IS NULL
OR rolled_back_at IS NOT NULL OR applied_steps_count <> 1)) THEN
RAISE EXCEPTION 'Corporate migration history mismatch';
END IF;
END $guard$;
SELECT NOT EXISTS (SELECT 1 FROM public._prisma_migrations
WHERE migration_name = '{NAME}') AS apply_needed \\gset
\\if :apply_needed
'''
after = f'''
INSERT INTO public._prisma_migrations
(id, checksum, finished_at, migration_name, logs, rolled_back_at, started_at, applied_steps_count)
VALUES ('{uuid.uuid4()}', '{checksum}', now(), '{NAME}', NULL, NULL, now(), 1);
\\endif
COMMIT;
'''
return before.encode() + sql + after.encode()
def apply_stage(payload):
stage('export PGPASSWORD="${POSTGRES_PASSWORD:-}"; exec psql -X -q -v ON_ERROR_STOP=1 '
'-U "$POSTGRES_USER" -d "$POSTGRES_DB"', payload)
def run():
sha = os.environ.get('DEPLOY_SHA', '')
if (os.environ.get('GITHUB_ACTIONS', os.environ.get('GITEA_ACTIONS')) != 'true'
or not re.fullmatch('[0-9a-f]{40}', sha)):
raise RuntimeError('CI SHA identity required')
relative = f'backend/prisma/migrations/{NAME}/migration.sql'
sql = (ROOT / relative).read_bytes()
manifest = json.loads((ROOT / '.release-manifest.json').read_text())
if manifest['sha'] != sha or manifest['files'][relative] != hashlib.sha256(sql).hexdigest():
raise RuntimeError('Release migration provenance mismatch')
# Fixed target, running state and exact durable mount: no configurable DB target.
info = json.loads(docker('inspect', '--format',
'{"Name":{{json .Name}},"State":{"Running":{{json .State.Running}}},'
'"Image":{{json .Image}},"Mounts":{{json .Mounts}}}', DB))
if info['Name'] != '/' + DB or not info['State']['Running']:
raise RuntimeError('Stage database is not running')
if not any(m['Destination'] == '/var/lib/postgresql/data'
and m.get('RW') and m['Type'] in ('volume', 'bind') for m in info['Mounts']):
raise RuntimeError('Stage durable data mount missing')
image = info['Image']
if not re.fullmatch('sha256:[0-9a-f]{64}', image):
raise RuntimeError('Stage image identity missing')
attempt = uuid.uuid4().hex
backup = f'{BACKUPS}/{sha}.{attempt}.dump'
# Never use runner filesystem as the only durable copy. Retain failed attempts.
stage(f'umask 077; mkdir -p {BACKUPS}; chmod 700 {BACKUPS}; '
f'test ! -e {backup}; export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; '
f'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f {backup}; '
f'test -s {backup}; pg_restore --list {backup} >/dev/null; '
f'chmod 600 {backup}; sync')
digest = stage(f'sha256sum {backup}').decode().split()[0]
if not re.fullmatch('[0-9a-f]{64}', digest):
raise RuntimeError('Backup digest missing')
clone = None
try:
clone = docker('create', '--name', 'canina-stage-migration-verify-' + attempt,
'--pull=never', '--network', 'none', '--tmpfs',
'/var/lib/postgresql/data:rw', '-e', 'POSTGRES_HOST_AUTH_METHOD=trust',
'-e', 'POSTGRES_DB=verify', image).decode().strip()
if not re.fullmatch('[0-9a-f]{12,64}', clone):
raise RuntimeError('Restore container identity missing')
docker('start', clone)
for _ in range(60):
try:
# TCP ensures the official image's temporary init server has ended.
docker('exec', clone, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres',
'-d', 'verify', timeout=10)
break
except RuntimeError:
time.sleep(1)
else:
raise RuntimeError('Restore server readiness timeout')
# Stream full backup, never store its sensitive contents in runner logs/memory.
source = subprocess.Popen(['docker', 'exec', DB, 'cat', backup], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
try:
assert source.stdout is not None
restored = subprocess.run(['docker', 'exec', '-i', clone, 'pg_restore',
'--exit-on-error', '--single-transaction', '--no-owner',
'--no-privileges', '-U', 'postgres', '-d', 'verify'],
stdin=source.stdout, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=600)
source.stdout.close()
if source.wait(timeout=30) or restored.returncode:
raise RuntimeError('Full backup restore failed')
finally:
if source.poll() is None:
source.kill()
source.wait()
payload = transaction(sql)
for _ in range(2):
docker('exec', '-i', clone, 'psql', '-X', '-q', '-v', 'ON_ERROR_STOP=1',
'-U', 'postgres', '-d', 'verify', data=payload)
# Verify persisted bytes again immediately before allowing live DDL.
if stage(f'sha256sum {backup}').decode().split()[0] != digest:
raise RuntimeError('Durable backup changed')
apply_stage(payload)
print(f'Stage corporate migration verified: sha={sha} backup={backup} sha256={digest}')
finally:
if clone:
docker('rm', '-f', clone, timeout=30)
if __name__ == '__main__':
def interrupted(signum, frame):
raise RuntimeError('Migration gate interrupted')
signal.signal(signal.SIGTERM, interrupted)
signal.signal(signal.SIGINT, interrupted)
try:
run()
except Exception as error:
# Exception text from subprocess/JSON could include sensitive output.
print('Stage corporate migration gate failed (' + type(error).__name__ +
'); retain backup and inspect privately; apps not recreated', file=sys.stderr)
sys.exit(1)
+4
View File
@@ -162,6 +162,10 @@ python3 scripts/verify-stage-runtime.py --base-compose "$STAGE_DIR/docker-compos
check_stage_lock
timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_stage frontend_stage
check_stage_lock
# CI-owned additive Stage migration: durable full backup + disposable restore
# verification must succeed before either application container is recreated.
timeout --foreground --kill-after=10s 1800 python3 scripts/corporate-stage-migrate.py
check_stage_lock
timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_stage frontend_stage
for name in canino_backend_stage canino_frontend_stage; do
# Every daemon interaction is bounded, including a hung exec/inspect.