test(stage-schema): fault-inject fail-closed apply prerequisites
This commit is contained in:
1 parent
db3979d212
commit
0a156fd7b4
6 files changed
+198
-1
No files matched your search
@@ -124,6 +124,8 @@ def main():
|
||||
args = ['exec', STAGE, 'sh','-c', 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-owner --no-acl -Fc' + (' --schema-only' if schema else '')]
|
||||
return d.run(args,timeout=180)
|
||||
backup = dump(a.mode == 'rehearse')
|
||||
if not backup:
|
||||
raise ValueError('Empty backup refused')
|
||||
backup_path = evidence / ('stage-schema.dump' if a.mode == 'rehearse' else 'stage-full.dump')
|
||||
fd = os.open(backup_path,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
|
||||
with os.fdopen(fd,'wb') as f: f.write(backup)
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Transport doubles exercise the real main(), never a live daemon/database."""
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import operation as m
|
||||
|
||||
SHA = 'a' * 40
|
||||
CATALOG = {'tables': {}, 'enums': {}}
|
||||
|
||||
|
||||
class Transport:
|
||||
def __init__(self, fault=None):
|
||||
self.fault = fault
|
||||
self.calls = []
|
||||
self.events = []
|
||||
self.sql_calls = []
|
||||
self.created = []
|
||||
self.ready_attempts = 0
|
||||
|
||||
def run(self, args, data=None, timeout=180):
|
||||
self.calls.append(args)
|
||||
self.events.append(('run', args))
|
||||
if args[0] == 'inspect':
|
||||
return b'runner-image'
|
||||
if args[0] == 'create':
|
||||
if self.fault == 'lock-contended':
|
||||
raise subprocess.CalledProcessError(1, args)
|
||||
self.created.append('owned-lock')
|
||||
return b'owned-lock'
|
||||
if args[0] == 'logs':
|
||||
return b'' if self.fault == 'lock-marker' else b'LOCKED'
|
||||
if args[0] == 'run':
|
||||
cid = 'owned-clone-' + str(len(self.created))
|
||||
self.created.append(cid)
|
||||
return cid.encode()
|
||||
if args[0] == 'rm':
|
||||
if self.fault == 'cleanup':
|
||||
raise subprocess.CalledProcessError(1, args)
|
||||
return b''
|
||||
if args[0] == 'exec' and 'pg_isready' in args:
|
||||
self.ready_attempts += 1
|
||||
if self.fault in ('readiness', 'readiness-timeout'):
|
||||
if self.fault == 'readiness-timeout':
|
||||
raise subprocess.TimeoutExpired(args, timeout)
|
||||
raise subprocess.CalledProcessError(1, args)
|
||||
return b''
|
||||
if args[0] == 'exec' and 'pg_restore' in args:
|
||||
if self.fault == 'restore':
|
||||
raise subprocess.CalledProcessError(1, args)
|
||||
return b''
|
||||
if args[0] == 'exec' and 'df -Pk' in args[-1]:
|
||||
return b'filesystem 9999999 1 9999998 1% /data'
|
||||
if args[0] == 'exec' and 'pg_dump' in args[-1]:
|
||||
if self.fault == 'backup':
|
||||
raise subprocess.CalledProcessError(1, args)
|
||||
return b'' if self.fault == 'empty-backup' else b'test-only dump bytes'
|
||||
return b''
|
||||
|
||||
def sql(self, container, query):
|
||||
self.sql_calls.append((container, query))
|
||||
self.events.append(('sql', (container, query)))
|
||||
if 'current_database()' in query and 'READ ONLY' in query:
|
||||
return 'caninodb_stage'
|
||||
if 'pg_database_size' in query:
|
||||
return '1024'
|
||||
if '_prisma_migrations' in query:
|
||||
return '[]'
|
||||
if m.CATALOG_SQL in query:
|
||||
if self.fault == 'clone-catalog' and container == 'owned-clone-2':
|
||||
return json.dumps({'tables': {'unexpected': {}}, 'enums': {}})
|
||||
return json.dumps(CATALOG)
|
||||
return ''
|
||||
|
||||
@property
|
||||
def live_writes(self):
|
||||
return [q for c, q in self.sql_calls if c == m.STAGE and q.startswith('BEGIN;')]
|
||||
|
||||
@property
|
||||
def removed(self):
|
||||
return [a[2] for a in self.calls if a[0] == 'rm']
|
||||
|
||||
|
||||
class MainFaults(unittest.TestCase):
|
||||
def invoke(self, transport, env=None, source=SHA):
|
||||
def git(args, **kwargs):
|
||||
if args[-2:] == ['rev-parse', 'HEAD']:
|
||||
return source
|
||||
return (m.ROOT / args[-1].split(':', 1)[1]).read_bytes()
|
||||
environment = dict(GITHUB_ACTIONS='true', GITHUB_SHA=SHA,
|
||||
GITHUB_REF='refs/heads/ops/seo-schema-stage-apply',
|
||||
STAGE_SCHEMA_REVIEWED_SHA=SHA)
|
||||
environment.update(env or {})
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
evidence = Path(tmp) / 'evidence'
|
||||
with patch.dict(os.environ, environment, clear=True), \
|
||||
patch('sys.argv', ['operation.py', 'apply', '--sha', SHA, '--evidence', str(evidence)]), \
|
||||
patch.object(m.subprocess, 'check_output', side_effect=git), \
|
||||
patch.object(m, 'Docker', return_value=transport), \
|
||||
patch.object(m.time, 'sleep'), \
|
||||
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
||||
m.main()
|
||||
self.assertEqual(json.loads((evidence / 'receipt.json').read_text())['sha'], SHA)
|
||||
self.assertEqual((evidence / 'stage-full.dump').stat().st_mode & 0o777, 0o600)
|
||||
self.assertEqual(evidence.stat().st_mode & 0o777, 0o700)
|
||||
|
||||
def test_backup_restore_readiness_failures_prevent_live_apply_and_clean_owned(self):
|
||||
for fault in ('backup', 'empty-backup', 'restore', 'readiness', 'readiness-timeout', 'lock-marker', 'clone-catalog'):
|
||||
with self.subTest(fault=fault):
|
||||
d = Transport(fault)
|
||||
with self.assertRaises((ValueError, subprocess.SubprocessError)):
|
||||
self.invoke(d)
|
||||
self.assertEqual(d.live_writes, [])
|
||||
self.assertEqual(d.removed, list(reversed(d.created)))
|
||||
if fault == 'readiness':
|
||||
self.assertEqual(d.ready_attempts, 60)
|
||||
|
||||
def test_contended_lock_is_never_removed(self):
|
||||
d = Transport('lock-contended')
|
||||
with self.assertRaises(subprocess.CalledProcessError):
|
||||
self.invoke(d)
|
||||
self.assertEqual(d.removed, [])
|
||||
self.assertEqual(d.sql_calls, [])
|
||||
|
||||
def test_cleanup_failure_attempts_every_owned_id(self):
|
||||
d = Transport('cleanup')
|
||||
self.invoke(d)
|
||||
self.assertEqual(d.removed, list(reversed(d.created)))
|
||||
self.assertEqual(len(d.live_writes), 1)
|
||||
|
||||
def test_full_restore_and_two_clone_applies_precede_live_apply(self):
|
||||
d = Transport()
|
||||
self.invoke(d)
|
||||
self.assertEqual(len(d.live_writes), 1)
|
||||
self.assertEqual(sum(q.startswith('BEGIN;') and c == 'owned-clone-2'
|
||||
for c, q in d.sql_calls), 2)
|
||||
dump = next(a for a in d.calls if a[0] == 'exec' and 'pg_dump' in a[-1])
|
||||
self.assertNotIn('--schema-only', dump[-1])
|
||||
restore = next(a for a in d.calls if 'pg_restore' in a)
|
||||
self.assertIn('--exit-on-error', restore)
|
||||
self.assertIn('127.0.0.1', restore)
|
||||
restore_index = d.events.index(('run', restore))
|
||||
live_index = next(i for i, (kind, value) in enumerate(d.events)
|
||||
if kind == 'sql' and value[0] == m.STAGE and value[1].startswith('BEGIN;'))
|
||||
clone_indices = [i for i, (kind, value) in enumerate(d.events)
|
||||
if kind == 'sql' and value[0] == 'owned-clone-2'
|
||||
and value[1].startswith('BEGIN;')]
|
||||
self.assertTrue(restore_index < clone_indices[0] < clone_indices[1] < live_index)
|
||||
self.assertEqual(d.removed, list(reversed(d.created)))
|
||||
|
||||
def test_secret_event_source_ref_bindings_fail_before_docker(self):
|
||||
cases = [({'STAGE_SCHEMA_REVIEWED_SHA': v}, SHA) for v in ('', 'b'*40, SHA+'\n')]
|
||||
cases += [({'GITHUB_SHA': 'b'*40}, SHA), ({'GITHUB_ACTIONS': 'false'}, SHA),
|
||||
({'GITHUB_REF': 'refs/heads/main'}, SHA),
|
||||
({'GITHUB_REF': 'refs/heads/ops/seo-schema-stage-rehearse'}, SHA),
|
||||
({}, 'b'*40)]
|
||||
for env, source in cases:
|
||||
with self.subTest(env=env, source=source):
|
||||
d = Transport()
|
||||
with self.assertRaises(ValueError):
|
||||
self.invoke(d, env, source)
|
||||
self.assertEqual(d.calls, [])
|
||||
self.assertEqual(d.sql_calls, [])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user