test(stage-schema): fault-inject fail-closed apply prerequisites

This commit is contained in:
Canina Automation committed 2026-10-28 08:01:38 +00:00
1 parent db3979d212
commit 0a156fd7b4
6 files changed
+198 -1

No files matched your search

+2
View File
@@ -124,6 +124,8 @@ def main():
args = ['exec', STAGE, 'sh','-c', 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-owner --no-acl -Fc' + (' --schema-only' if schema else '')]
return d.run(args,timeout=180)
backup = dump(a.mode == 'rehearse')
if not backup:
raise ValueError('Empty backup refused')
backup_path = evidence / ('stage-schema.dump' if a.mode == 'rehearse' else 'stage-full.dump')
fd = os.open(backup_path,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
with os.fdopen(fd,'wb') as f: f.write(backup)
@@ -0,0 +1,173 @@
"""Transport doubles exercise the real main(), never a live daemon/database."""
import contextlib
import io
import json
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
import operation as m
SHA = 'a' * 40
CATALOG = {'tables': {}, 'enums': {}}
class Transport:
def __init__(self, fault=None):
self.fault = fault
self.calls = []
self.events = []
self.sql_calls = []
self.created = []
self.ready_attempts = 0
def run(self, args, data=None, timeout=180):
self.calls.append(args)
self.events.append(('run', args))
if args[0] == 'inspect':
return b'runner-image'
if args[0] == 'create':
if self.fault == 'lock-contended':
raise subprocess.CalledProcessError(1, args)
self.created.append('owned-lock')
return b'owned-lock'
if args[0] == 'logs':
return b'' if self.fault == 'lock-marker' else b'LOCKED'
if args[0] == 'run':
cid = 'owned-clone-' + str(len(self.created))
self.created.append(cid)
return cid.encode()
if args[0] == 'rm':
if self.fault == 'cleanup':
raise subprocess.CalledProcessError(1, args)
return b''
if args[0] == 'exec' and 'pg_isready' in args:
self.ready_attempts += 1
if self.fault in ('readiness', 'readiness-timeout'):
if self.fault == 'readiness-timeout':
raise subprocess.TimeoutExpired(args, timeout)
raise subprocess.CalledProcessError(1, args)
return b''
if args[0] == 'exec' and 'pg_restore' in args:
if self.fault == 'restore':
raise subprocess.CalledProcessError(1, args)
return b''
if args[0] == 'exec' and 'df -Pk' in args[-1]:
return b'filesystem 9999999 1 9999998 1% /data'
if args[0] == 'exec' and 'pg_dump' in args[-1]:
if self.fault == 'backup':
raise subprocess.CalledProcessError(1, args)
return b'' if self.fault == 'empty-backup' else b'test-only dump bytes'
return b''
def sql(self, container, query):
self.sql_calls.append((container, query))
self.events.append(('sql', (container, query)))
if 'current_database()' in query and 'READ ONLY' in query:
return 'caninodb_stage'
if 'pg_database_size' in query:
return '1024'
if '_prisma_migrations' in query:
return '[]'
if m.CATALOG_SQL in query:
if self.fault == 'clone-catalog' and container == 'owned-clone-2':
return json.dumps({'tables': {'unexpected': {}}, 'enums': {}})
return json.dumps(CATALOG)
return ''
@property
def live_writes(self):
return [q for c, q in self.sql_calls if c == m.STAGE and q.startswith('BEGIN;')]
@property
def removed(self):
return [a[2] for a in self.calls if a[0] == 'rm']
class MainFaults(unittest.TestCase):
def invoke(self, transport, env=None, source=SHA):
def git(args, **kwargs):
if args[-2:] == ['rev-parse', 'HEAD']:
return source
return (m.ROOT / args[-1].split(':', 1)[1]).read_bytes()
environment = dict(GITHUB_ACTIONS='true', GITHUB_SHA=SHA,
GITHUB_REF='refs/heads/ops/seo-schema-stage-apply',
STAGE_SCHEMA_REVIEWED_SHA=SHA)
environment.update(env or {})
with tempfile.TemporaryDirectory() as tmp:
evidence = Path(tmp) / 'evidence'
with patch.dict(os.environ, environment, clear=True), \
patch('sys.argv', ['operation.py', 'apply', '--sha', SHA, '--evidence', str(evidence)]), \
patch.object(m.subprocess, 'check_output', side_effect=git), \
patch.object(m, 'Docker', return_value=transport), \
patch.object(m.time, 'sleep'), \
contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
m.main()
self.assertEqual(json.loads((evidence / 'receipt.json').read_text())['sha'], SHA)
self.assertEqual((evidence / 'stage-full.dump').stat().st_mode & 0o777, 0o600)
self.assertEqual(evidence.stat().st_mode & 0o777, 0o700)
def test_backup_restore_readiness_failures_prevent_live_apply_and_clean_owned(self):
for fault in ('backup', 'empty-backup', 'restore', 'readiness', 'readiness-timeout', 'lock-marker', 'clone-catalog'):
with self.subTest(fault=fault):
d = Transport(fault)
with self.assertRaises((ValueError, subprocess.SubprocessError)):
self.invoke(d)
self.assertEqual(d.live_writes, [])
self.assertEqual(d.removed, list(reversed(d.created)))
if fault == 'readiness':
self.assertEqual(d.ready_attempts, 60)
def test_contended_lock_is_never_removed(self):
d = Transport('lock-contended')
with self.assertRaises(subprocess.CalledProcessError):
self.invoke(d)
self.assertEqual(d.removed, [])
self.assertEqual(d.sql_calls, [])
def test_cleanup_failure_attempts_every_owned_id(self):
d = Transport('cleanup')
self.invoke(d)
self.assertEqual(d.removed, list(reversed(d.created)))
self.assertEqual(len(d.live_writes), 1)
def test_full_restore_and_two_clone_applies_precede_live_apply(self):
d = Transport()
self.invoke(d)
self.assertEqual(len(d.live_writes), 1)
self.assertEqual(sum(q.startswith('BEGIN;') and c == 'owned-clone-2'
for c, q in d.sql_calls), 2)
dump = next(a for a in d.calls if a[0] == 'exec' and 'pg_dump' in a[-1])
self.assertNotIn('--schema-only', dump[-1])
restore = next(a for a in d.calls if 'pg_restore' in a)
self.assertIn('--exit-on-error', restore)
self.assertIn('127.0.0.1', restore)
restore_index = d.events.index(('run', restore))
live_index = next(i for i, (kind, value) in enumerate(d.events)
if kind == 'sql' and value[0] == m.STAGE and value[1].startswith('BEGIN;'))
clone_indices = [i for i, (kind, value) in enumerate(d.events)
if kind == 'sql' and value[0] == 'owned-clone-2'
and value[1].startswith('BEGIN;')]
self.assertTrue(restore_index < clone_indices[0] < clone_indices[1] < live_index)
self.assertEqual(d.removed, list(reversed(d.created)))
def test_secret_event_source_ref_bindings_fail_before_docker(self):
cases = [({'STAGE_SCHEMA_REVIEWED_SHA': v}, SHA) for v in ('', 'b'*40, SHA+'\n')]
cases += [({'GITHUB_SHA': 'b'*40}, SHA), ({'GITHUB_ACTIONS': 'false'}, SHA),
({'GITHUB_REF': 'refs/heads/main'}, SHA),
({'GITHUB_REF': 'refs/heads/ops/seo-schema-stage-rehearse'}, SHA),
({}, 'b'*40)]
for env, source in cases:
with self.subTest(env=env, source=source):
d = Transport()
with self.assertRaises(ValueError):
self.invoke(d, env, source)
self.assertEqual(d.calls, [])
self.assertEqual(d.sql_calls, [])
if __name__ == '__main__':
unittest.main()
+2
View File
@@ -1,2 +1,4 @@
# Plan
Python Docker runner fixes source container to canino_db_stage and verifies database caninodb_stage. Dedicated Gitea 1.22 push refs ops/seo-schema-stage-rehearse and ops/seo-schema-stage-apply are disjoint from develop/main. Explicit CI repository secret reviewed SHA controls apply; rehearsal needs event/source binding only. CI downloads archive by immutable SHA. PostgreSQL 16 image pinned to actual Stage image. Full backup bytes remain mode600 in runner private evidence directory. Transactions use advisory locks and bounded timeouts.
Fault-injection review: exercise real main() with an explicit in-process Docker transport double and isolated private evidence directory. Preserve source-file checks, exact SQL digest and binding gates; intercept Git reads only to bind the fixture SHA. Assert restore/clone-apply/live-apply ordering and reverse captured-ID cleanup. Reject empty dumps immediately. No frontend/backend edits, new approval layer or live mutation.
+3
View File
@@ -1,2 +1,5 @@
# Stage SEO schema operation
Stage-only dedicated push refs rehearse then apply reviewed SQL, never generic migration chain. Bind exact 40-character event SHA and SQL digest. Preserve all operational configuration and concurrent application edits. Rehearse sanitized actual Stage schema clone; before live application require private full Stage backup and full restore. Validate complete scoped native catalog; leave Prisma historical ledger untouched and record separate operation receipt. No Production access or automatic application deployment.
## Fault-injection acceptance
Backup errors (including an empty dump), restore errors and final-TCP readiness failure must abort before any live DDL. Every captured owned container ID must receive reverse-order cleanup on success/failure, without deleting an existing contended lock. Exact reviewed-SHA secret, event SHA, source SHA and dedicated apply ref must bind before any Docker call. Tests use explicit transport doubles; they do not establish full-data recovery or CI acceptance.
+4
View File
@@ -3,3 +3,7 @@
- [ ] Exercise actual Stage schema-only sanitized clone restore and SQL catalog parity.
- [ ] Inspect supported Gitea push path; commit owned files only; no push pending parent safety review.
- [ ] Future authorized CI: full Stage backup/restore then apply and typed readback, retaining receipt.
- [x] Independently verify c62d983 receipt against existing dump, catalog, SQL and commit; verify private modes.
- [x] Add real-main transport-double failure injection: backup/empty dump, restore, final TCP readiness and timeout, clone drift, lock ownership/cleanup, secret/event/source/ref binding and success ordering.
- [x] Record behavioral RED (empty dump accepted) and minimal guard, then complete Stage tooling suite GREEN.
- [x] Inspect existing Gitea admin and repository-secret API metadata only; no token creation, secret mutation or push.
+14 -1
View File
@@ -12,4 +12,17 @@ Readiness: local tooling and sanitized restore rehearsed. Parent must review com
Graphify queried before edit and before commit in isolated /opt/data/cache/scratch/seo-qa-graph (4304 nodes, stale graph binding e7ded849f6b1e3bdab2a20d89b3ec7dd0df79cab, original hash 6a73473cee9e207543fdeeb668bea544f766200a1b16b8a3a9ce18d4e42e51af). Navigation only; migration/workflow contracts checked directly. New tooling absent from old graph; no generated graph committed.
Initial RED: operation test import failed because operation.py did not exist; catalog child also recorded initial RED. Final GREEN recorded in command output. Unit coverage is binding/typed catalogs/transaction guards, not simulated full CI/restore-failure injection. Actual successful sanitized restore supplies independent integration evidence.
Initial RED: operation test import failed because operation.py did not exist; catalog child also recorded initial RED. Final GREEN recorded in command output. Initial unit coverage was binding/typed catalogs/transaction guards. Actual successful sanitized restore supplies independent integration evidence.
## Independent fault-injection review — baseline c62d983
Independently read `/opt/data/cache/scratch/stage-seo-rehearsal-c62d983/receipt.json` and recomputed dump size/hash, canonical sorted catalog hash and SQL hash against the actual existing files: all match. The receipt binds c62d9836e95c3fd1a5bc196bb759b6416d0c4908; dump 106701 bytes, SHA256 d9d310a4f6a5255600a3b2bd459aefa50fbf8a4d12fe66cc4af12d759fc59f57; catalog b4289a3ab06ede914bb41996925361c8ad42974b4810c02b050fee477442b0e4. Evidence directory mode700 and dump mode600. This verifies artifact consistency, not independent observation of the original run, full-data recovery or current live schema.
New tests exercise actual main() with an explicitly synthetic in-process transport double: backup process failure, empty dump, restore failure, TCP-readiness exhaustion (60 probes), transport timeout, clone catalog mismatch and missing lock-acquisition marker all prevent live apply and clean only captured IDs in reverse order. Existing contended lock is never removed. Cleanup error attempts every captured ID; stderr remains observable but is not treated as proof cleanup succeeded. Successful path asserts full dump (not schema-only), exit-on-error TCP restore, two clone applies before the single Stage apply and private backup permissions. Missing/stale/newline-reviewed SHA, wrong event/source SHA, non-CI and wrong destination refs all fail before Docker.
Behavioral RED: 17 tests ran, one failure because a successful pg_dump process returning zero bytes was accepted by main(). Minimal fix refuses an empty backup immediately. Post-fix suite: `python -m unittest discover -s scripts/stage-schema -v`, 17 passed. Transport doubles prove control flow only, not real full-data restore or CI execution. No application builds were needed: owned changes are Stage tooling/specs only; application/catalog mode, secrets, runtime and deployments untouched.
### Existing credential/admin path (metadata only)
Pinned SSH inspection confirms `gitea_main` Gitea 1.22.6 and existing active administrator `parsa`. Gitea admin CLI supports user/token administration but has no repository-secret subcommand; no token was generated or credential read. Public instance Swagger confirms GET `/repos/{owner}/{repo}/actions/secrets` and PUT/DELETE `/repos/{owner}/{repo}/actions/secrets/{secretname}`. Easiest least-change route is the existing administrator's authenticated repository Settings → Actions → Secrets UI at `https://git.parsaaghayi.ir/parsa/canina/settings/actions/secrets`, setting only `STAGE_SCHEMA_REVIEWED_SHA` to the final reviewed full commit SHA. This is an existing account/admin path, not evidence that a saved reusable API token is available; do not mint an extra token or change any password merely for this operation. No secret values inspected or modified.
Release still requires parent review and a frozen tested SHA: non-force push that exact commit to the dedicated rehearsal ref, inspect real terminal CI success, then bind the same reviewed SHA through existing repository-secret administration and push the identical commit to the dedicated apply ref. That CI must complete private full-data backup/restore before Stage DDL, retain receipt/backup and verify Stage scoped catalog/ledger/cleanup. Current sanitized receipt alone is not sufficient. No develop/main application push, runtime restart, generic migrate deploy or production operation is authorized by this review.