fix(ci): wait for final TCP server before schema restore
Approved narrow auth schema reconciliation / reconcile (push) Successful in 15s
Deploy Canina / deploy (push) Failing after 21s

This commit is contained in:
devops committed 2026-10-25 13:27:39 +00:00
1 parent b3b9e617b5
commit 5897abcbad
2 files changed
+120 -7

No files matched your search

+9 -7
View File
@@ -31,7 +31,7 @@ fi
# Both approvals are supplied only by the reviewed protected workflow.
backup="$backup_root/$environment-$sha-$(date -u +%Y%m%dT%H%M%SZ).dump"
# Full backup remains restricted on host; never emitted to CI logs or artifacts.
docker exec "$source_container" sh -c 'PGOPTIONS="-c default_transaction_read_only=on" pg_dump -Fc --no-owner --no-privileges -U "$POSTGRES_USER" -d "$POSTGRES_DB"' > "$backup"
docker exec "$source_container" sh -c 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; PGOPTIONS="-c default_transaction_read_only=on" pg_dump -h 127.0.0.1 -Fc --no-owner --no-privileges -U "$POSTGRES_USER" -d "$POSTGRES_DB"' > "$backup"
[[ -s $backup ]]
docker exec -i "$container" pg_restore --list < "$backup" > "$backup.toc"
[[ -s $backup.toc ]]
@@ -42,16 +42,18 @@ cleanup() { docker rm -f "$name" >/dev/null 2>&1 || true; }
trap cleanup EXIT
docker run -d --name "$name" --network none --tmpfs /var/lib/postgresql/data:rw,size=1g -e POSTGRES_HOST_AUTH_METHOD=trust postgres:16-alpine >/dev/null
ready=false
for _ in {1..60}; do if docker exec "$name" pg_isready -U postgres >/dev/null 2>&1; then ready=true; break; fi; sleep 1; done
# The entrypoint's temporary init server listens only on a Unix socket. TCP
# becomes ready only after initialization hands off to the final server.
for _ in {1..60}; do if docker exec "$name" pg_isready -h 127.0.0.1 -U postgres -d postgres >/dev/null 2>&1; then ready=true; break; fi; sleep 1; done
[[ $ready == true ]]
docker exec -i "$name" pg_restore --exit-on-error --no-owner --no-privileges -U postgres -d postgres < "$backup"
docker exec -i "$name" psql -X -v ON_ERROR_STOP=1 -U postgres < "$sql"
docker exec -i "$name" psql -X -qAt -v ON_ERROR_STOP=1 -U postgres < "$root/scripts/auth-schema-verify.sql" > "$backup.expected.json"
docker exec -i "$name" pg_restore --exit-on-error -h 127.0.0.1 --no-owner --no-privileges -U postgres -d postgres < "$backup"
docker exec -i "$name" psql -h 127.0.0.1 -X -v ON_ERROR_STOP=1 -U postgres -d postgres < "$sql"
docker exec -i "$name" psql -h 127.0.0.1 -X -qAt -v ON_ERROR_STOP=1 -U postgres -d postgres < "$root/scripts/auth-schema-verify.sql" > "$backup.expected.json"
if [[ $environment == production ]]; then
# Only the reviewed SQL is executed. No Prisma CLI, seeds or ledger edits.
docker exec -i "$container" sh -c 'psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$sql"
docker exec -i "$container" sh -c 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; psql -h 127.0.0.1 -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$sql"
fi
docker exec -i "$container" sh -c 'PGOPTIONS="-c default_transaction_read_only=on" psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$root/scripts/auth-schema-verify.sql" > "$backup.actual.json"
docker exec -i "$container" sh -c 'export PGPASSWORD="${POSTGRES_PASSWORD:-}"; PGOPTIONS="-c default_transaction_read_only=on" psql -h 127.0.0.1 -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < "$root/scripts/auth-schema-verify.sql" > "$backup.actual.json"
cmp "$backup.expected.json" "$backup.actual.json"
cleanup; trap - EXIT
if [[ $environment == stage ]]; then printf '%s\n' "$sha" > "$receipt"; fi
+111
View File
@@ -1,7 +1,118 @@
"""Offline safety-contract tests. DB execution evidence is in docs/auth-schema-rehearsal.json."""
import hashlib
import json
import os
import pathlib
import subprocess
import tempfile
import unittest
# Exercise the actual shell workflow without Docker, host backups, or a database.
FAKE_DOCKER = r'''#!/usr/bin/env python3
import json, os, pathlib, sys
args = sys.argv[1:]
state_path = pathlib.Path(os.environ['FAKE_STATE'])
state = json.loads(state_path.read_text()) if state_path.exists() else {'polls': 0, 'tcp_ready': False, 'calls': []}
state['calls'].append(args)
rc = 0
if args[0] == 'exec':
command = ' '.join(args)
tcp = '-h 127.0.0.1' in command
if 'pg_isready' in args:
state['polls'] += 1
# Entrypoint's temporary server accepts sockets immediately, not TCP.
if tcp:
state['tcp_ready'] = state['polls'] >= 3 and os.environ.get('FAKE_MODE') != 'never_ready'
rc = 0 if state['tcp_ready'] else 1
elif 'pg_dump' in command:
print('fake full archive')
elif 'pg_restore' in args and '--list' in args:
sys.stdin.read()
print('fake archive TOC')
elif 'pg_restore' in args:
sys.stdin.read()
if not tcp or not state['tcp_ready']:
print('pg_restore: transient Unix socket missing', file=sys.stderr)
rc = 1
elif os.environ.get('FAKE_MODE') == 'restore_fails':
rc = 1
elif 'psql' in command:
sys.stdin.read()
if not tcp:
print('SQL command did not select stable TCP', file=sys.stderr)
rc = 1
else:
print('{}')
state_path.write_text(json.dumps(state))
sys.exit(rc)
'''
class RestoreReadiness(unittest.TestCase):
def run_workflow(self, mode='transient'):
with tempfile.TemporaryDirectory(prefix='auth-readiness-') as directory:
work = pathlib.Path(directory)
scripts = work/'scripts'
scripts.mkdir()
for name in ['auth-schema-reconcile.sql', 'auth-schema-verify.sql']:
(scripts/name).write_bytes((ROOT/'scripts'/name).read_bytes())
script = (ROOT/'scripts/ci-auth-schema-reconcile.sh').read_text()
script = script.replace('backup_root=/opt/backups/canina-auth-reconcile',
'backup_root="$FAKE_BACKUP_ROOT"')
(scripts/'ci-auth-schema-reconcile.sh').write_text(script)
binaries = work/'bin'
binaries.mkdir()
fakes = {
'docker': FAKE_DOCKER,
'git': '#!/bin/sh\ncase "$*" in\n "rev-parse --show-toplevel") printf "%s\\n" "$FAKE_ROOT";;\n "rev-parse HEAD") printf "%s\\n" "$FAKE_SHA";;\n "status --porcelain") exit 0;;\n *) exit 2;;\nesac\n',
'sleep': '#!/bin/sh\nexit 0\n',
}
for name, content in fakes.items():
(binaries/name).write_text(content)
(binaries/name).chmod(0o755)
sha = 'a'*40
digest = hashlib.sha256((scripts/'auth-schema-reconcile.sql').read_bytes()).hexdigest()
backups = work/'backups'
backups.mkdir()
(backups/f'stage-{digest}.receipt').write_text(sha+'\n')
env = dict(os.environ, PATH=str(binaries)+os.pathsep+os.environ['PATH'],
GITHUB_ACTIONS='true', FAKE_ROOT=str(work), FAKE_SHA=sha,
FAKE_BACKUP_ROOT=str(backups), FAKE_STATE=str(work/'state.json'),
FAKE_MODE=mode, STAGE_AUTH_ACCEPTANCE_APPROVED=f'{sha}:{digest}',
PRODUCTION_SCHEMA_APPROVED=f'{sha}:{digest}')
result = subprocess.run(['bash', str(scripts/'ci-auth-schema-reconcile.sh'),
'production', sha, digest], env=env,
text=True, capture_output=True, timeout=60)
return result, json.loads((work/'state.json').read_text())
def test_waits_for_final_tcp_server_despite_ready_transient_socket(self):
result, state = self.run_workflow()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(state['polls'], 3)
self.assertTrue(state['tcp_ready'])
calls = state['calls']
restore = next(i for i, call in enumerate(calls) if '--exit-on-error' in call)
self.assertEqual(sum('pg_isready' in c for c in calls[:restore]), 3)
for call in calls:
command = ' '.join(call)
if any(tool in command for tool in ['pg_isready', 'pg_dump', 'psql']) or '--exit-on-error' in call:
self.assertIn('-h 127.0.0.1', command)
self.assertEqual(calls[-1][0:2], ['rm', '-f'])
def test_readiness_timeout_never_restores_or_executes_sql(self):
result, state = self.run_workflow('never_ready')
self.assertNotEqual(result.returncode, 0)
self.assertEqual(state['polls'], 60)
self.assertFalse(any('--exit-on-error' in c or 'psql' in ' '.join(c)
for c in state['calls']))
self.assertEqual(state['calls'][-1][0:2], ['rm', '-f'])
def test_restore_failure_never_executes_sql_or_retries_restore(self):
result, state = self.run_workflow('restore_fails')
self.assertNotEqual(result.returncode, 0)
self.assertEqual(sum('--exit-on-error' in c for c in state['calls']), 1)
self.assertFalse(any('psql' in ' '.join(c) for c in state['calls']))
self.assertEqual(state['calls'][-1][0:2], ['rm', '-f'])
ROOT = pathlib.Path(__file__).resolve().parents[1]
class Safety(unittest.TestCase):
def test_shell_syntax(self):