fix(ci): wait for final TCP server before schema restore
This commit is contained in:
1 parent
b3b9e617b5
commit
5897abcbad
2 files changed
+120
-7
No files matched your search
@@ -31,7 +31,7 @@ fi
|
||||
# Both approvals are supplied only by the reviewed protected workflow.
|
||||
backup="$backup_root/$environment-$sha-$(date -u +%Y%m%dT%H%M%SZ).dump"
|
||||
# Full backup remains restricted on host; never emitted to CI logs or artifacts.
|
||||
docker exec "$source_container" sh -c 'PGOPTIONS="-c default_transaction_read_only=on" pg_dump -Fc --no-owner --no-privileges -U "$POSTGRES_USER" -d "$POSTGRES_DB"' > "$backup"
|
||||
docker exec "$source_container" sh -c 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; PGOPTIONS="-c default_transaction_read_only=on" pg_dump -h 127.0.0.1 -Fc --no-owner --no-privileges -U "$POSTGRES_USER" -d "$POSTGRES_DB"' > "$backup"
|
||||
[[ -s $backup ]]
|
||||
docker exec -i "$container" pg_restore --list < "$backup" > "$backup.toc"
|
||||
[[ -s $backup.toc ]]
|
||||
@@ -42,16 +42,18 @@ cleanup() { docker rm -f "$name" >/dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT
|
||||
docker run -d --name "$name" --network none --tmpfs /var/lib/postgresql/data:rw,size=1g -e POSTGRES_HOST_AUTH_METHOD=trust postgres:16-alpine >/dev/null
|
||||
ready=false
|
||||
for _ in {1..60}; do if docker exec "$name" pg_isready -U postgres >/dev/null 2>&1; then ready=true; break; fi; sleep 1; done
|
||||
# The entrypoint's temporary init server listens only on a Unix socket. TCP
|
||||
# becomes ready only after initialization hands off to the final server.
|
||||
for _ in {1..60}; do if docker exec "$name" pg_isready -h 127.0.0.1 -U postgres -d postgres >/dev/null 2>&1; then ready=true; break; fi; sleep 1; done
|
||||
[[ $ready == true ]]
|
||||
docker exec -i "$name" pg_restore --exit-on-error --no-owner --no-privileges -U postgres -d postgres < "$backup"
|
||||
docker exec -i "$name" psql -X -v ON_ERROR_STOP=1 -U postgres < "$sql"
|
||||
docker exec -i "$name" psql -X -qAt -v ON_ERROR_STOP=1 -U postgres < "$root/scripts/auth-schema-verify.sql" > "$backup.expected.json"
|
||||
docker exec -i "$name" pg_restore --exit-on-error -h 127.0.0.1 --no-owner --no-privileges -U postgres -d postgres < "$backup"
|
||||
docker exec -i "$name" psql -h 127.0.0.1 -X -v ON_ERROR_STOP=1 -U postgres -d postgres < "$sql"
|
||||
docker exec -i "$name" psql -h 127.0.0.1 -X -qAt -v ON_ERROR_STOP=1 -U postgres -d postgres < "$root/scripts/auth-schema-verify.sql" > "$backup.expected.json"
|
||||
if [[ $environment == production ]]; then
|
||||
# Only the reviewed SQL is executed. No Prisma CLI, seeds or ledger edits.
|
||||
docker exec -i "$container" sh -c 'psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$sql"
|
||||
docker exec -i "$container" sh -c 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; psql -h 127.0.0.1 -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$sql"
|
||||
fi
|
||||
docker exec -i "$container" sh -c 'PGOPTIONS="-c default_transaction_read_only=on" psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$root/scripts/auth-schema-verify.sql" > "$backup.actual.json"
|
||||
docker exec -i "$container" sh -c 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; PGOPTIONS="-c default_transaction_read_only=on" psql -h 127.0.0.1 -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$root/scripts/auth-schema-verify.sql" > "$backup.actual.json"
|
||||
cmp "$backup.expected.json" "$backup.actual.json"
|
||||
cleanup; trap - EXIT
|
||||
if [[ $environment == stage ]]; then printf '%s\n' "$sha" > "$receipt"; fi
|
||||
|
||||
@@ -1,7 +1,118 @@
|
||||
"""Offline safety-contract tests. DB execution evidence is in docs/auth-schema-rehearsal.json."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
# Exercise the actual shell workflow without Docker, host backups, or a database.
|
||||
FAKE_DOCKER = r'''#!/usr/bin/env python3
|
||||
import json, os, pathlib, sys
|
||||
args = sys.argv[1:]
|
||||
state_path = pathlib.Path(os.environ['FAKE_STATE'])
|
||||
state = json.loads(state_path.read_text()) if state_path.exists() else {'polls': 0, 'tcp_ready': False, 'calls': []}
|
||||
state['calls'].append(args)
|
||||
rc = 0
|
||||
if args[0] == 'exec':
|
||||
command = ' '.join(args)
|
||||
tcp = '-h 127.0.0.1' in command
|
||||
if 'pg_isready' in args:
|
||||
state['polls'] += 1
|
||||
# Entrypoint's temporary server accepts sockets immediately, not TCP.
|
||||
if tcp:
|
||||
state['tcp_ready'] = state['polls'] >= 3 and os.environ.get('FAKE_MODE') != 'never_ready'
|
||||
rc = 0 if state['tcp_ready'] else 1
|
||||
elif 'pg_dump' in command:
|
||||
print('fake full archive')
|
||||
elif 'pg_restore' in args and '--list' in args:
|
||||
sys.stdin.read()
|
||||
print('fake archive TOC')
|
||||
elif 'pg_restore' in args:
|
||||
sys.stdin.read()
|
||||
if not tcp or not state['tcp_ready']:
|
||||
print('pg_restore: transient Unix socket missing', file=sys.stderr)
|
||||
rc = 1
|
||||
elif os.environ.get('FAKE_MODE') == 'restore_fails':
|
||||
rc = 1
|
||||
elif 'psql' in command:
|
||||
sys.stdin.read()
|
||||
if not tcp:
|
||||
print('SQL command did not select stable TCP', file=sys.stderr)
|
||||
rc = 1
|
||||
else:
|
||||
print('{}')
|
||||
state_path.write_text(json.dumps(state))
|
||||
sys.exit(rc)
|
||||
'''
|
||||
|
||||
class RestoreReadiness(unittest.TestCase):
|
||||
def run_workflow(self, mode='transient'):
|
||||
with tempfile.TemporaryDirectory(prefix='auth-readiness-') as directory:
|
||||
work = pathlib.Path(directory)
|
||||
scripts = work/'scripts'
|
||||
scripts.mkdir()
|
||||
for name in ['auth-schema-reconcile.sql', 'auth-schema-verify.sql']:
|
||||
(scripts/name).write_bytes((ROOT/'scripts'/name).read_bytes())
|
||||
script = (ROOT/'scripts/ci-auth-schema-reconcile.sh').read_text()
|
||||
script = script.replace('backup_root=/opt/backups/canina-auth-reconcile',
|
||||
'backup_root="$FAKE_BACKUP_ROOT"')
|
||||
(scripts/'ci-auth-schema-reconcile.sh').write_text(script)
|
||||
binaries = work/'bin'
|
||||
binaries.mkdir()
|
||||
fakes = {
|
||||
'docker': FAKE_DOCKER,
|
||||
'git': '#!/bin/sh\ncase "$*" in\n "rev-parse --show-toplevel") printf "%s\\n" "$FAKE_ROOT";;\n "rev-parse HEAD") printf "%s\\n" "$FAKE_SHA";;\n "status --porcelain") exit 0;;\n *) exit 2;;\nesac\n',
|
||||
'sleep': '#!/bin/sh\nexit 0\n',
|
||||
}
|
||||
for name, content in fakes.items():
|
||||
(binaries/name).write_text(content)
|
||||
(binaries/name).chmod(0o755)
|
||||
sha = 'a'*40
|
||||
digest = hashlib.sha256((scripts/'auth-schema-reconcile.sql').read_bytes()).hexdigest()
|
||||
backups = work/'backups'
|
||||
backups.mkdir()
|
||||
(backups/f'stage-{digest}.receipt').write_text(sha+'\n')
|
||||
env = dict(os.environ, PATH=str(binaries)+os.pathsep+os.environ['PATH'],
|
||||
GITHUB_ACTIONS='true', FAKE_ROOT=str(work), FAKE_SHA=sha,
|
||||
FAKE_BACKUP_ROOT=str(backups), FAKE_STATE=str(work/'state.json'),
|
||||
FAKE_MODE=mode, STAGE_AUTH_ACCEPTANCE_APPROVED=f'{sha}:{digest}',
|
||||
PRODUCTION_SCHEMA_APPROVED=f'{sha}:{digest}')
|
||||
result = subprocess.run(['bash', str(scripts/'ci-auth-schema-reconcile.sh'),
|
||||
'production', sha, digest], env=env,
|
||||
text=True, capture_output=True, timeout=60)
|
||||
return result, json.loads((work/'state.json').read_text())
|
||||
|
||||
def test_waits_for_final_tcp_server_despite_ready_transient_socket(self):
|
||||
result, state = self.run_workflow()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(state['polls'], 3)
|
||||
self.assertTrue(state['tcp_ready'])
|
||||
calls = state['calls']
|
||||
restore = next(i for i, call in enumerate(calls) if '--exit-on-error' in call)
|
||||
self.assertEqual(sum('pg_isready' in c for c in calls[:restore]), 3)
|
||||
for call in calls:
|
||||
command = ' '.join(call)
|
||||
if any(tool in command for tool in ['pg_isready', 'pg_dump', 'psql']) or '--exit-on-error' in call:
|
||||
self.assertIn('-h 127.0.0.1', command)
|
||||
self.assertEqual(calls[-1][0:2], ['rm', '-f'])
|
||||
|
||||
def test_readiness_timeout_never_restores_or_executes_sql(self):
|
||||
result, state = self.run_workflow('never_ready')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(state['polls'], 60)
|
||||
self.assertFalse(any('--exit-on-error' in c or 'psql' in ' '.join(c)
|
||||
for c in state['calls']))
|
||||
self.assertEqual(state['calls'][-1][0:2], ['rm', '-f'])
|
||||
|
||||
def test_restore_failure_never_executes_sql_or_retries_restore(self):
|
||||
result, state = self.run_workflow('restore_fails')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(sum('--exit-on-error' in c for c in state['calls']), 1)
|
||||
self.assertFalse(any('psql' in ' '.join(c) for c in state['calls']))
|
||||
self.assertEqual(state['calls'][-1][0:2], ['rm', '-f'])
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
class Safety(unittest.TestCase):
|
||||
def test_shell_syntax(self):
|
||||
|
||||
Reference in new issue
Block a user