diff --git a/scripts/corporate-stage-migrate-test.py b/scripts/corporate-stage-migrate-test.py new file mode 100644 index 00000000..e27db81a --- /dev/null +++ b/scripts/corporate-stage-migrate-test.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Local-only real PG transaction tests and Docker transport fault injection.""" +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('migration', Path(__file__).with_name('corporate-stage-migrate.py')) +assert spec is not None and spec.loader is not None +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) +SQL = (m.ROOT / f'backend/prisma/migrations/{m.NAME}/migration.sql').read_bytes() + + +class Transport(unittest.TestCase): + def test_gate_faults_and_order(self): + for failure in ('backup', 'restore', 'clone-apply', 'digest', None): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as td: + root = Path(td) + rel = f'backend/prisma/migrations/{m.NAME}/migration.sql' + (root / rel).parent.mkdir(parents=True) + (root / rel).write_bytes(SQL) + (root / '.release-manifest.json').write_text(json.dumps({ + 'sha': 'a' * 40, 'files': {rel: m.hashlib.sha256(SQL).hexdigest()}})) + events = [] + digest_reads = 0 + + def fake(*args, data=None, timeout=600): + nonlocal digest_reads + if args[0] == 'inspect': + return json.dumps({'Name': '/' + m.DB, 'State': {'Running': True}, + 'Image': 'sha256:' + 'b' * 64, 'Mounts': [{'Destination': + '/var/lib/postgresql/data', 'RW': True, 'Type': 'volume'}]}).encode() + if args[0] == 'create': + events.append('create'); return b'c' * 64 + if args[0] == 'rm': + events.append('cleanup'); return b'' + if args[0] == 'exec' and args[2] == m.DB: + cmd = args[-1] + if 'pg_dump' in cmd: + events.append('backup') + if failure == 'backup': raise RuntimeError('injected') + elif 'sha256sum' in cmd: + digest_reads += 1 + return (('f' if failure == 'digest' and digest_reads == 2 else 'd') * 64 + ' file').encode() + elif 'psql' in cmd: + events.append('live') + return b'' + if args[0] == 'exec' and 'psql' in args: + events.append('clone-apply') + if failure == 'clone-apply': raise RuntimeError('injected') + return b'' + + class Source: + stdout = io.BytesIO() + def wait(self, timeout=None): return 0 + def poll(self): return 0 + class Result: + returncode = 1 if failure == 'restore' else 0 + def restore(*args, **kwargs): + events.append('restore'); return Result() + with patch.object(m, 'ROOT', root), patch.object(m, 'docker', fake), patch.dict( + os.environ, GITHUB_ACTIONS='true', DEPLOY_SHA='a' * 40), patch.object( + m.subprocess, 'Popen', return_value=Source()), patch.object(m.subprocess, 'run', restore): + if failure: + with self.assertRaises(RuntimeError): m.run() + self.assertNotIn('live', events) + else: + m.run() + self.assertEqual(events, ['backup', 'create', 'restore', 'clone-apply', 'clone-apply', 'live', 'cleanup']) + if 'create' in events: self.assertEqual(events[-1], 'cleanup') + + def test_pipeline_order(self): + source = Path(__file__).with_name('release-cache-stage.sh').read_text() + self.assertLess(source.index('build backend_stage frontend_stage'), source.index('python3 scripts/corporate-stage-migrate.py')) + self.assertLess(source.index('python3 scripts/corporate-stage-migrate.py'), source.index('up -d --no-deps --force-recreate')) + + +class RealPostgres(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.server = Path(os.environ.get('PG_SERVER_BIN', '/opt/data/cache/scratch/seo-pg-bin/bin')) + cls.client = Path(os.environ.get('PG_CLIENT_BIN', '/opt/data/cache/scratch/corporate-pg-client/usr/lib/postgresql/16/bin')) + cls.temp = tempfile.TemporaryDirectory(prefix='cm-', dir='/opt/data/cache/scratch') + cls.root = Path(cls.temp.name) + cls.env = dict(os.environ, LD_LIBRARY_PATH=str(cls.server.parent / 'lib'), PGHOST=str(cls.root), PGPORT='16439', PGDATABASE='postgres') + subprocess.run([str(cls.server / 'initdb'), '-D', str(cls.root / 'data'), '-A', 'trust', '--no-locale'], env=cls.env, check=True, stdout=subprocess.DEVNULL) + subprocess.run([str(cls.server / 'pg_ctl'), '-D', str(cls.root / 'data'), '-l', str(cls.root / 'server.log'), '-o', f'-k {cls.root} -p 16439 -h ""', '-w', 'start'], env=cls.env, check=True, stdout=subprocess.DEVNULL) + + @classmethod + def tearDownClass(cls): + subprocess.run([str(cls.server / 'pg_ctl'), '-D', str(cls.root / 'data'), '-m', 'immediate', '-w', 'stop'], env=cls.env, check=True, stdout=subprocess.DEVNULL) + cls.temp.cleanup() + + def query(self, sql, success=True): + result = subprocess.run([str(self.client / 'psql'), '-X', '-At', '-v', 'ON_ERROR_STOP=1'], input=sql, env=self.env, capture_output=True) + self.assertEqual(result.returncode == 0, success, result.stderr.decode()) + return result.stdout.decode().strip() + + def test_real_transaction_backup_restore(self): + self.assertIn('16.2', self.query(b'SELECT version();')) + self.query(b'''CREATE TABLE _prisma_migrations(id varchar(36) PRIMARY KEY, checksum varchar(64) NOT NULL, finished_at timestamptz, migration_name varchar(255) NOT NULL, logs text, rolled_back_at timestamptz, started_at timestamptz NOT NULL DEFAULT now(), applied_steps_count integer NOT NULL DEFAULT 0); CREATE TABLE sentinel(id int, value text); INSERT INTO sentinel VALUES (1, 'preserved');''') + backup = self.root / 'full.dump' + subprocess.run([str(self.client / 'pg_dump'), '-Fc', '-f', str(backup)], env=self.env, check=True) + self.assertGreater(backup.stat().st_size, 0) + self.query(m.transaction(SQL + b'\nCREATE TABLE must_rollback(id int); SELECT 1/0;'), False) + self.assertEqual(self.query(b"SELECT count(*) FROM pg_tables WHERE tablename LIKE 'corporate_%' OR tablename='must_rollback';"), '0') + self.assertEqual(self.query(b'SELECT count(*) FROM _prisma_migrations;'), '0') + # Hold the exact migration lock in a distinct backend; fail closed on contention. + holder = subprocess.Popen([str(self.client / 'psql'), '-X', '-At'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.env) + assert holder.stdin is not None and holder.stdout is not None + holder.stdin.write(b'BEGIN; SELECT pg_advisory_xact_lock(72707369); SELECT 42; SELECT pg_sleep(2); COMMIT;\n') + holder.stdin.close() + while holder.stdout.readline().strip() != b'42': + if holder.poll() is not None: self.fail('Lock fixture failed') + self.query(m.transaction(SQL).replace(b"'30s'", b"'200ms'"), False) + holder.wait(timeout=10) + holder.stdout.close() + assert holder.stderr is not None + holder.stderr.close() + self.query(m.transaction(SQL)) + self.query(m.transaction(SQL)) + self.assertEqual(self.query(b'SELECT count(*) FROM _prisma_migrations;'), '1') + self.query(m.transaction(SQL + b'\n-- changed checksum'), False) + self.query(b"UPDATE _prisma_migrations SET finished_at=NULL;") + self.query(m.transaction(SQL), False) + self.query(b"CREATE DATABASE restored;") + subprocess.run([str(self.client / 'pg_restore'), '--exit-on-error', '--single-transaction', '-d', 'restored', str(backup)], env=self.env, check=True) + previous = self.env + self.env = dict(self.env, PGDATABASE='restored') + self.assertEqual(self.query(b'SELECT value FROM sentinel;'), 'preserved') + self.assertEqual(self.query(b'SELECT count(*) FROM _prisma_migrations;'), '0') + self.query(m.transaction(SQL)) + self.query(m.transaction(SQL)) + self.env = previous + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/scripts/corporate-stage-migrate.md b/scripts/corporate-stage-migrate.md new file mode 100644 index 00000000..42e700ce --- /dev/null +++ b/scripts/corporate-stage-migrate.md @@ -0,0 +1,21 @@ +# Controlled corporate Stage migration + +## Proportional dependency graph / specification (before implementation) + +Existing graph query (`graphify query 'release-cache-stage deploy build migration Prisma' --budget 1200`) locates deploy.sh → Stage Compose and backend scripts → Prisma; release-cache-stage.sh is newer than that graph. Direct inspection establishes the exact execution edge: + +`immutable SHA archive → runtime validation → build backend_stage + frontend_stage → [new gate] → recreate apps → readiness` + +New gate: `CI identity → stage DB pinned container/image/volume → pg_dump custom → fsync durable DB-volume backup + digest → disposable same-image PG restore → transaction apply/replay on clone → transaction apply on Stage → app recreate`. + +Only scripts/release-cache-stage.sh and corporate-stage-migrate* are owned. The migration SQL is read without modification. No checkout reset, production commands, cron or Hermes changes. This implementation is not a deployment authorization. + +Read-only Stage inspection found standard public._prisma_migrations columns: id varchar, checksum varchar, finished_at timestamptz nullable, migration_name varchar, logs text nullable, rolled_back_at timestamptz nullable, started_at timestamptz default now(), applied_steps_count integer default 0. The data mount is /var/lib/postgresql/data. Credentials are expanded only by the container shell. + +## Safety contract + +Fail closed on non-CI invocation, invalid SHA, unexpected database mount, backup/restore/readiness errors, unresolved Prisma migrations, duplicate or mismatching migration checksum. Keep full custom-format backup under the existing Stage data volume in canina-stage-backups; sync file and directory before restore verification. Never delete backup artifacts automatically. Restore to a no-network disposable same-image database with tmpfs storage; remove only the captured owned container ID. Use a bounded transaction-level advisory lock (Prisma key 72707369), ON_ERROR_STOP and a single transaction for DDL plus standard Prisma bookkeeping. Existing successful checksum-matching entry is a no-op. DDL errors disconnect/rollback; never mark failed work as applied. Test real PostgreSQL plus fake Docker orchestration transport locally; live server interaction remains read-only. + +## Operations + +The release invokes `python3 scripts/corporate-stage-migrate.py` after both builds and before either app is recreated, under the existing supervised host lock. It needs Docker, Python 3 and the existing Stage DB image with PostgreSQL tools. Successful output contains backup path/digest and release SHA, never credentials. Backups contain sensitive full database content, are mode 0600 in a mode 0700 directory, and require separate operator retention policy. Restore test uses an isolated postgres superuser and restores all data/schema without source ownership or ACLs; it proves data recoverability and DDL compatibility, not source role/ACL recovery. Live access uses the source role resolved internally, never logged. Live writes during pg_dump are permitted by PostgreSQL's consistent snapshot; this additive migration does not provide a rollback of subsequent application writes. No automatic live restore is performed. diff --git a/scripts/corporate-stage-migrate.py b/scripts/corporate-stage-migrate.py new file mode 100644 index 00000000..95453444 --- /dev/null +++ b/scripts/corporate-stage-migrate.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""CI-only Stage gate; never calls Prisma deploy or touches another database.""" +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import time +import uuid + +DB = 'canino_db_stage' +NAME = '20261028120000_corporate_orchestration' +BACKUPS = '/var/lib/postgresql/data/canina-stage-backups' +ROOT = Path(__file__).resolve().parent.parent + + +def docker(*args, data=None, timeout=600): + # Do not relay stderr: tools may expose connection details or row data. + result = subprocess.run(['docker', *args], input=data, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=timeout) + if result.returncode: + raise RuntimeError('Docker operation failed: ' + args[0]) + return result.stdout + + +def stage(command, data=None): + # Expansion occurs only inside the existing DB container. No env inspection. + return docker('exec', '-i', DB, 'sh', '-ec', command, data=data) + + +def transaction(sql): + checksum = hashlib.sha256(sql).hexdigest() + # One connection, one transaction; client disconnect rolls back on all errors. + before = f'''\\set ON_ERROR_STOP on +BEGIN; +SET LOCAL lock_timeout = '30s'; +SET LOCAL statement_timeout = '300s'; +SET LOCAL search_path = public, pg_catalog; +SELECT pg_advisory_xact_lock(72707369); +LOCK TABLE public._prisma_migrations IN SHARE ROW EXCLUSIVE MODE; +DO $guard$ +BEGIN + IF EXISTS (SELECT 1 FROM public._prisma_migrations + WHERE finished_at IS NULL AND rolled_back_at IS NULL) THEN + RAISE EXCEPTION 'Unresolved Prisma migration'; + END IF; + IF (SELECT count(*) FROM public._prisma_migrations WHERE migration_name = '{NAME}') > 1 + OR EXISTS (SELECT 1 FROM public._prisma_migrations WHERE migration_name = '{NAME}' + AND (checksum <> '{checksum}' OR finished_at IS NULL + OR rolled_back_at IS NOT NULL OR applied_steps_count <> 1)) THEN + RAISE EXCEPTION 'Corporate migration history mismatch'; + END IF; +END $guard$; +SELECT NOT EXISTS (SELECT 1 FROM public._prisma_migrations + WHERE migration_name = '{NAME}') AS apply_needed \\gset +\\if :apply_needed +''' + after = f''' +INSERT INTO public._prisma_migrations + (id, checksum, finished_at, migration_name, logs, rolled_back_at, started_at, applied_steps_count) +VALUES ('{uuid.uuid4()}', '{checksum}', now(), '{NAME}', NULL, NULL, now(), 1); +\\endif +COMMIT; +''' + return before.encode() + sql + after.encode() + + +def apply_stage(payload): + stage('export PGPASSWORD="${POSTGRES_PASSWORD:-}"; exec psql -X -q -v ON_ERROR_STOP=1 ' + '-U "$POSTGRES_USER" -d "$POSTGRES_DB"', payload) + + +def run(): + sha = os.environ.get('DEPLOY_SHA', '') + if (os.environ.get('GITHUB_ACTIONS', os.environ.get('GITEA_ACTIONS')) != 'true' + or not re.fullmatch('[0-9a-f]{40}', sha)): + raise RuntimeError('CI SHA identity required') + relative = f'backend/prisma/migrations/{NAME}/migration.sql' + sql = (ROOT / relative).read_bytes() + manifest = json.loads((ROOT / '.release-manifest.json').read_text()) + if manifest['sha'] != sha or manifest['files'][relative] != hashlib.sha256(sql).hexdigest(): + raise RuntimeError('Release migration provenance mismatch') + # Fixed target, running state and exact durable mount: no configurable DB target. + info = json.loads(docker('inspect', '--format', + '{"Name":{{json .Name}},"State":{"Running":{{json .State.Running}}},' + '"Image":{{json .Image}},"Mounts":{{json .Mounts}}}', DB)) + if info['Name'] != '/' + DB or not info['State']['Running']: + raise RuntimeError('Stage database is not running') + if not any(m['Destination'] == '/var/lib/postgresql/data' + and m.get('RW') and m['Type'] in ('volume', 'bind') for m in info['Mounts']): + raise RuntimeError('Stage durable data mount missing') + image = info['Image'] + if not re.fullmatch('sha256:[0-9a-f]{64}', image): + raise RuntimeError('Stage image identity missing') + attempt = uuid.uuid4().hex + backup = f'{BACKUPS}/{sha}.{attempt}.dump' + # Never use runner filesystem as the only durable copy. Retain failed attempts. + stage(f'umask 077; mkdir -p {BACKUPS}; chmod 700 {BACKUPS}; ' + f'test ! -e {backup}; export PGPASSWORD="${{POSTGRES_PASSWORD:-}}"; ' + f'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f {backup}; ' + f'test -s {backup}; pg_restore --list {backup} >/dev/null; ' + f'chmod 600 {backup}; sync') + digest = stage(f'sha256sum {backup}').decode().split()[0] + if not re.fullmatch('[0-9a-f]{64}', digest): + raise RuntimeError('Backup digest missing') + clone = None + try: + clone = docker('create', '--name', 'canina-stage-migration-verify-' + attempt, + '--pull=never', '--network', 'none', '--tmpfs', + '/var/lib/postgresql/data:rw', '-e', 'POSTGRES_HOST_AUTH_METHOD=trust', + '-e', 'POSTGRES_DB=verify', image).decode().strip() + if not re.fullmatch('[0-9a-f]{12,64}', clone): + raise RuntimeError('Restore container identity missing') + docker('start', clone) + for _ in range(60): + try: + # TCP ensures the official image's temporary init server has ended. + docker('exec', clone, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres', + '-d', 'verify', timeout=10) + break + except RuntimeError: + time.sleep(1) + else: + raise RuntimeError('Restore server readiness timeout') + # Stream full backup, never store its sensitive contents in runner logs/memory. + source = subprocess.Popen(['docker', 'exec', DB, 'cat', backup], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + try: + assert source.stdout is not None + restored = subprocess.run(['docker', 'exec', '-i', clone, 'pg_restore', + '--exit-on-error', '--single-transaction', '--no-owner', + '--no-privileges', '-U', 'postgres', '-d', 'verify'], + stdin=source.stdout, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=600) + source.stdout.close() + if source.wait(timeout=30) or restored.returncode: + raise RuntimeError('Full backup restore failed') + finally: + if source.poll() is None: + source.kill() + source.wait() + payload = transaction(sql) + for _ in range(2): + docker('exec', '-i', clone, 'psql', '-X', '-q', '-v', 'ON_ERROR_STOP=1', + '-U', 'postgres', '-d', 'verify', data=payload) + # Verify persisted bytes again immediately before allowing live DDL. + if stage(f'sha256sum {backup}').decode().split()[0] != digest: + raise RuntimeError('Durable backup changed') + apply_stage(payload) + print(f'Stage corporate migration verified: sha={sha} backup={backup} sha256={digest}') + finally: + if clone: + docker('rm', '-f', clone, timeout=30) + + +if __name__ == '__main__': + def interrupted(signum, frame): + raise RuntimeError('Migration gate interrupted') + + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + try: + run() + except Exception as error: + # Exception text from subprocess/JSON could include sensitive output. + print('Stage corporate migration gate failed (' + type(error).__name__ + + '); retain backup and inspect privately; apps not recreated', file=sys.stderr) + sys.exit(1) diff --git a/scripts/release-cache-stage.sh b/scripts/release-cache-stage.sh index e37a46b9..c22d4684 100644 --- a/scripts/release-cache-stage.sh +++ b/scripts/release-cache-stage.sh @@ -162,6 +162,10 @@ python3 scripts/verify-stage-runtime.py --base-compose "$STAGE_DIR/docker-compos check_stage_lock timeout --foreground --kill-after=2s 1800 "${compose[@]}" --progress plain build backend_stage frontend_stage check_stage_lock +# CI-owned additive Stage migration: durable full backup + disposable restore +# verification must succeed before either application container is recreated. +timeout --foreground --kill-after=10s 1800 python3 scripts/corporate-stage-migrate.py +check_stage_lock timeout --foreground --kill-after=2s 120 "${compose[@]}" up -d --no-deps --force-recreate backend_stage frontend_stage for name in canino_backend_stage canino_frontend_stage; do # Every daemon interaction is bounded, including a hung exec/inspect.