test(stage-schema): fault-inject fail-closed apply prerequisites

This commit is contained in:
Canina Automation committed 2026-10-28 08:01:38 +00:00
1 parent db3979d212
commit 0a156fd7b4
6 files changed
+198 -1

No files matched your search

+2
View File
@@ -124,6 +124,8 @@ def main():
args = ['exec', STAGE, 'sh','-c', 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-owner --no-acl -Fc' + (' --schema-only' if schema else '')]
return d.run(args,timeout=180)
backup = dump(a.mode == 'rehearse')
if not backup:
raise ValueError('Empty backup refused')
backup_path = evidence / ('stage-schema.dump' if a.mode == 'rehearse' else 'stage-full.dump')
fd = os.open(backup_path,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
with os.fdopen(fd,'wb') as f: f.write(backup)
@@ -0,0 +1,173 @@
"""Transport doubles exercise the real main(), never a live daemon/database."""
import contextlib
import io
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
import operation as m
SHA = 'a' * 40
CATALOG = {'tables': {}, 'enums': {}}
class Transport:
def __init__(self, fault=None):
self.fault = fault
self.calls = []
self.events = []
self.sql_calls = []
self.created = []
self.ready_attempts = 0
def run(self, args, data=None, timeout=180):
self.calls.append(args)
self.events.append(('run', args))
if args[0] == 'inspect':
return b'runner-image'
if args[0] == 'create':
if self.fault == 'lock-contended':
raise subprocess.CalledProcessError(1, args)
self.created.append('owned-lock')
return b'owned-lock'
if args[0] == 'logs':
return b'' if self.fault == 'lock-marker' else b'LOCKED'
if args[0] == 'run':
cid = 'owned-clone-' + str(len(self.created))
self.created.append(cid)
return cid.encode()
if args[0] == 'rm':
if self.fault == 'cleanup':
raise subprocess.CalledProcessError(1, args)
return b''
if args[0] == 'exec' and 'pg_isready' in args:
self.ready_attempts += 1
if self.fault in ('readiness', 'readiness-timeout'):
if self.fault == 'readiness-timeout':
raise subprocess.TimeoutExpired(args, timeout)
raise subprocess.CalledProcessError(1, args)
return b''
if args[0] == 'exec' and 'pg_restore' in args:
if self.fault == 'restore':
raise subprocess.CalledProcessError(1, args)
return b''
if args[0] == 'exec' and 'df -Pk' in args[-1]:
return b'filesystem 9999999 1 9999998 1% /data'
if args[0] == 'exec' and 'pg_dump' in args[-1]:
if self.fault == 'backup':
raise subprocess.CalledProcessError(1, args)
return b'' if self.fault == 'empty-backup' else b'test-only dump bytes'
return b''
def sql(self, container, query):
self.sql_calls.append((container, query))
self.events.append(('sql', (container, query)))
if 'current_database()' in query and 'READ ONLY' in query:
return 'caninodb_stage'
if 'pg_database_size' in query:
return '1024'
if '_prisma_migrations' in query:
return '[]'
if m.CATALOG_SQL in query:
if self.fault == 'clone-catalog' and container == 'owned-clone-2':
return json.dumps({'tables': {'unexpected': {}}, 'enums': {}})
return json.dumps(CATALOG)
return ''
@property
def live_writes(self):
return [q for c, q in self.sql_calls if c == m.STAGE and q.startswith('BEGIN;')]
@property
def removed(self):
return [a[2] for a in self.calls if a[0] == 'rm']
class MainFaults(unittest.TestCase):
def invoke(self, transport, env=None, source=SHA):
def git(args, **kwargs):
if args[-2:] == ['rev-parse', 'HEAD']:
return source
return (m.ROOT / args[-1].split(':', 1)[1]).read_bytes()
environment = dict(GITHUB_ACTIONS='true', GITHUB_SHA=SHA,
GITHUB_REF='refs/heads/ops/seo-schema-stage-apply',
STAGE_SCHEMA_REVIEWED_SHA=SHA)
environment.update(env or {})
with tempfile.TemporaryDirectory() as tmp:
evidence = Path(tmp) / 'evidence'
with patch.dict(os.environ, environment, clear=True), \
patch('sys.argv', ['operation.py', 'apply', '--sha', SHA, '--evidence', str(evidence)]), \
patch.object(m.subprocess, 'check_output', side_effect=git), \
patch.object(m, 'Docker', return_value=transport), \
patch.object(m.time, 'sleep'), \
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
m.main()
self.assertEqual(json.loads((evidence / 'receipt.json').read_text())['sha'], SHA)
self.assertEqual((evidence / 'stage-full.dump').stat().st_mode & 0o777, 0o600)
self.assertEqual(evidence.stat().st_mode & 0o777, 0o700)
def test_backup_restore_readiness_failures_prevent_live_apply_and_clean_owned(self):
for fault in ('backup', 'empty-backup', 'restore', 'readiness', 'readiness-timeout', 'lock-marker', 'clone-catalog'):
with self.subTest(fault=fault):
d = Transport(fault)
with self.assertRaises((ValueError, subprocess.SubprocessError)):
self.invoke(d)
self.assertEqual(d.live_writes, [])
self.assertEqual(d.removed, list(reversed(d.created)))
if fault == 'readiness':
self.assertEqual(d.ready_attempts, 60)
def test_contended_lock_is_never_removed(self):
d = Transport('lock-contended')
with self.assertRaises(subprocess.CalledProcessError):
self.invoke(d)
self.assertEqual(d.removed, [])
self.assertEqual(d.sql_calls, [])
def test_cleanup_failure_attempts_every_owned_id(self):
d = Transport('cleanup')
self.invoke(d)
self.assertEqual(d.removed, list(reversed(d.created)))
self.assertEqual(len(d.live_writes), 1)
def test_full_restore_and_two_clone_applies_precede_live_apply(self):
d = Transport()
self.invoke(d)
self.assertEqual(len(d.live_writes), 1)
self.assertEqual(sum(q.startswith('BEGIN;') and c == 'owned-clone-2'
for c, q in d.sql_calls), 2)
dump = next(a for a in d.calls if a[0] == 'exec' and 'pg_dump' in a[-1])
self.assertNotIn('--schema-only', dump[-1])
restore = next(a for a in d.calls if 'pg_restore' in a)
self.assertIn('--exit-on-error', restore)
self.assertIn('127.0.0.1', restore)
restore_index = d.events.index(('run', restore))
live_index = next(i for i, (kind, value) in enumerate(d.events)
if kind == 'sql' and value[0] == m.STAGE and value[1].startswith('BEGIN;'))
clone_indices = [i for i, (kind, value) in enumerate(d.events)
if kind == 'sql' and value[0] == 'owned-clone-2'
and value[1].startswith('BEGIN;')]
self.assertTrue(restore_index < clone_indices[0] < clone_indices[1] < live_index)
self.assertEqual(d.removed, list(reversed(d.created)))
def test_secret_event_source_ref_bindings_fail_before_docker(self):
cases = [({'STAGE_SCHEMA_REVIEWED_SHA': v}, SHA) for v in ('', 'b'*40, SHA+'\n')]
cases += [({'GITHUB_SHA': 'b'*40}, SHA), ({'GITHUB_ACTIONS': 'false'}, SHA),
({'GITHUB_REF': 'refs/heads/main'}, SHA),
({'GITHUB_REF': 'refs/heads/ops/seo-schema-stage-rehearse'}, SHA),
({}, 'b'*40)]
for env, source in cases:
with self.subTest(env=env, source=source):
d = Transport()
with self.assertRaises(ValueError):
self.invoke(d, env, source)
self.assertEqual(d.calls, [])
self.assertEqual(d.sql_calls, [])
if __name__ == '__main__':
unittest.main()