ops(stage): version isolated SEO schema rehearsal and CI operation

This commit is contained in:
Canina Automation committed 2026-10-28 07:53:18 +00:00
1 parent 312fcc2acc
commit c62d9836e9
10 files changed
+520

No files matched your search

+22
View File
@@ -0,0 +1,22 @@
# Stage-only SEO schema operation
Normal develop/main deployment remains code-only. Gitea 1.22.6 supports the `push` event; this workflow uses only `ops/seo-schema-stage-rehearse` and `ops/seo-schema-stage-apply`, disjoint from application refs. No workflow_dispatch assumption, Production access, generic Prisma migration, seed, reset, or operational checkout edit.
## Run after safety review
1. Freeze/review exact operation commit, including workflow, `operation.py`, `catalog.py`, SQL and tests. Apply requires repository secret `STAGE_SCHEMA_REVIEWED_SHA` set through authorized repository-secret administration to that **full commit SHA**. This is a technical immutable-source binding, not a new business authorization requirement. Never log credentials. Rehearsal does not require this secret.
2. Transfer the committed source through the already pinned SSH host transport into an isolated server repository, never the dirty operational checkout. Inspect authoritative refs and push the exact commit non-force to `refs/heads/ops/seo-schema-stage-rehearse`. Do not push application develop/main. CI fetches the immutable event SHA through HTTPS (runner read-only transport verified).
3. Inspect actual terminal Gitea Actions log at exact SHA. The rehearsal restores a schema-only sanitized dump of actual Stage into isolated PostgreSQL16, applies exact digest twice and checks complete scoped typed catalog. It is **not** full-data recovery proof.
4. After parent review, provision exact reviewed-SHA secret and non-force push the identical commit to `refs/heads/ops/seo-schema-stage-apply`. This operation first takes a private full Stage dump, restores all data into an isolated clone, rehearses SQL, checks complete scoped types/defaults/nullability/enum/constraints/indexes, and only then applies bounded transactional SQL to `canino_db_stage` / `caninodb_stage`. Never set CI flags in SSH to bypass the supported path.
5. Inspect actual terminal log, restricted receipt and live typed catalog. Do not retry ambiguous DDL outcomes blindly. SQL is additive and reentrant; the runner never restores over Stage or drops live tables. Backup/restore uses no-owner/no-acl: it proves data recovery, not role/ACL or cluster disaster recovery.
## Ledger and backup retention
Historical `_prisma_migrations` remains byte-for-byte JSON equal before/after; no historical entries are synthesized, and no migration is marked applied. The incomplete baseline makes generic migrate deploy unsafe. A separate private receipt binds SHA, exact SQL digest, backup checksum/size and typed catalog hash. Reconcile this ledger policy before any future generic migration tooling use.
CI prints only its restricted private evidence directory. Dumps are mode600 in a mode700 directory, never artifacts or Git. Retain backup and receipt through Stage acceptance; authorized retention cleanup must remove only that exact directory after recovery requirements are met. Disposable containers alone are cleaned by captured IDs. Existing configs, secrets, cron, database containers, application containers and catalog-mode commerce flags are untouched. Shared host deployment flock prevents overlap with app CI; existing lock helpers fail closed.
Local diagnosis (read-only Stage plus disposable clone, never live DDL):
`python scripts/stage-schema/operation.py rehearse --sha <HEAD> --ssh-rehearsal --evidence <new-private-directory>`
Tests: `python -m unittest discover -s scripts/stage-schema -v`. Unit safety/typed tests do not substitute for actual CI execution or full-data restore. The operation pins the observed Stage PostgreSQL image ID and fails if unavailable; it never pulls an unreviewed replacement.
+122
View File
@@ -0,0 +1,122 @@
"""Read and compare the SEO migration's scoped PostgreSQL catalog.
Execute CATALOG_SQL on both databases with the same PostgreSQL major version
and search_path (normally public). The expected value must come from a clean
reference database: create public.blogs(id UUID PRIMARY KEY), then apply the
reviewed migration. Do not derive the reference from the database under test.
This module owns no connection, process, migration execution, or credentials.
"""
import json
from typing import Any, Mapping, TypedDict
class ColumnCatalog(TypedDict):
type: str
not_null: bool
default: str | None
class ConstraintCatalog(TypedDict):
type: str
definition: str
validated: bool
deferrable: bool
deferred: bool
class IndexCatalog(TypedDict):
definition: str
valid: bool
ready: bool
unique: bool
class TableCatalog(TypedDict):
columns: dict[str, ColumnCatalog]
constraints: dict[str, ConstraintCatalog]
indexes: dict[str, IndexCatalog]
class Catalog(TypedDict):
tables: dict[str, TableCatalog]
enums: dict[str, list[str]]
# Names are the entire scope: do not include blogs or _prisma_migrations.
# Enumerating existing objects, rather than projecting known columns/indexes,
# also detects unexpected additions within the two owned tables.
CATALOG_SQL = r'''
WITH scoped_tables AS (
SELECT c.oid, c.relname
FROM pg_catalog.pg_class AS c
JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace
WHERE n.nspname = 'public'
AND c.relkind IN ('r', 'p')
AND c.relname IN ('content_research_findings',
'content_performance_snapshots')
), table_catalog AS (
SELECT t.relname, jsonb_build_object(
'columns', COALESCE((
SELECT jsonb_object_agg(a.attname, jsonb_build_object(
'type', pg_catalog.format_type(a.atttypid, a.atttypmod),
'not_null', a.attnotnull,
'default', pg_catalog.pg_get_expr(d.adbin, d.adrelid, false)
) ORDER BY a.attname)
FROM pg_catalog.pg_attribute AS a
LEFT JOIN pg_catalog.pg_attrdef AS d
ON d.adrelid = a.attrelid AND d.adnum = a.attnum
WHERE a.attrelid = t.oid AND a.attnum > 0 AND NOT a.attisdropped
), '{}'::jsonb),
'constraints', COALESCE((
SELECT jsonb_object_agg(k.conname, jsonb_build_object(
'type', k.contype,
'definition', pg_catalog.pg_get_constraintdef(k.oid, false),
'validated', k.convalidated,
'deferrable', k.condeferrable,
'deferred', k.condeferred
) ORDER BY k.conname)
FROM pg_catalog.pg_constraint AS k WHERE k.conrelid = t.oid
), '{}'::jsonb),
'indexes', COALESCE((
SELECT jsonb_object_agg(ic.relname, jsonb_build_object(
'definition', pg_catalog.pg_get_indexdef(i.indexrelid, 0, false),
'valid', i.indisvalid,
'ready', i.indisready,
'unique', i.indisunique
) ORDER BY ic.relname)
FROM pg_catalog.pg_index AS i
JOIN pg_catalog.pg_class AS ic ON ic.oid = i.indexrelid
WHERE i.indrelid = t.oid
), '{}'::jsonb)
) AS value FROM scoped_tables AS t
), enum_catalog AS (
SELECT e_type.typname,
jsonb_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels
FROM pg_catalog.pg_type AS e_type
JOIN pg_catalog.pg_namespace AS n ON n.oid = e_type.typnamespace
JOIN pg_catalog.pg_enum AS e ON e.enumtypid = e_type.oid
WHERE n.nspname = 'public' AND e_type.typname = 'ResearchFindingStatus'
GROUP BY e_type.typname
)
SELECT jsonb_build_object(
'tables', COALESCE((SELECT jsonb_object_agg(relname, value ORDER BY relname)
FROM table_catalog), '{}'::jsonb),
'enums', COALESCE((SELECT jsonb_object_agg(typname, labels ORDER BY typname)
FROM enum_catalog), '{}'::jsonb)
) AS catalog;
'''
def validate(actual: Mapping[str, Any], expected: Mapping[str, Any]) -> None:
"""Accept exact JSON equality, ignoring object-key order only.
Array order is significant (especially enum labels); no fields, defaults,
definitions, or unknown scoped objects are stripped or normalized away.
Raise ValueError on drift, including both canonical catalogs for diagnosis.
"""
actual_json = json.dumps(actual, sort_keys=True, separators=(',', ':'), allow_nan=False)
expected_json = json.dumps(expected, sort_keys=True, separators=(',', ':'), allow_nan=False)
if actual_json != expected_json:
raise ValueError(
f"Scoped SEO catalog mismatch\nexpected: {expected_json}\nactual: {actual_json}"
)
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Bounded Stage-only SEO operation; no generic Prisma migration execution."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import shlex
import shutil
import subprocess
import time
import uuid
from catalog import CATALOG_SQL, validate
ROOT = Path(__file__).resolve().parents[2]
SQL_PATH = 'backend/prisma/migrations/20261028120000_add_seo_tracking_contract/migration.sql'
SQL_DIGEST = 'de1386398026c6373ce8692fb165b904e189f830d7b20914d8a4b512a47f9d16'
IMAGE = 'sha256:de3a4eab8fdfa507ea92aac488b916b08089e515db49b055fe71dfa271ba3a28'
STAGE = 'canino_db_stage'
def check_binding(sha, source, mode, reviewed):
if not re.fullmatch('[0-9a-f]{40}', sha) or sha != source:
raise ValueError('Exact source/event SHA binding required')
if mode == 'apply' and reviewed != sha:
raise ValueError('Independent reviewed operation SHA required')
def check_database(name):
if name.strip() != 'caninodb_stage':
raise ValueError('Not Stage database')
class Docker:
def __init__(self, remote=False):
self.remote = remote
def run(self, args, data=None, timeout=180):
cmd = ['docker'] + args
if self.remote:
cmd = ['ssh', '-F', '/opt/data/.ssh/config', '-o', 'StrictHostKeyChecking=yes',
'-o', 'UserKnownHostsFile=/opt/data/.ssh/known_hosts', 'parsa-host', shlex.join(cmd)]
return subprocess.run(cmd, input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
timeout=timeout, check=True).stdout
def sql(self, container, query):
# Live identity and credentials are never read into Python or printed.
if container == STAGE:
args = ['exec', '-i', STAGE, 'sh', '-c',
'exec psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"']
else:
args = ['exec', '-i', container, 'psql', '-h', '127.0.0.1', '-U', 'postgres',
'-d', 'rehearsal', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1']
return self.run(args, query.encode()).decode().strip()
def catalog(docker, name):
return json.loads(docker.sql(name, 'SET search_path=public;\n' + CATALOG_SQL))
def transaction(sql):
return ("BEGIN; SET LOCAL lock_timeout='5s'; SET LOCAL statement_timeout='60s'; "
"SELECT pg_advisory_xact_lock(731920260); SET LOCAL search_path=public;\n" + sql + '\nCOMMIT;')
def main():
ap = argparse.ArgumentParser()
ap.add_argument('mode', choices=['rehearse', 'apply'])
ap.add_argument('--sha', required=True)
ap.add_argument('--ssh-rehearsal', action='store_true')
ap.add_argument('--evidence', required=True)
a = ap.parse_args()
source = subprocess.check_output(['git', '-C', str(ROOT), 'rev-parse', 'HEAD'], text=True).strip()
check_binding(a.sha, source, a.mode, os.environ.get('STAGE_SCHEMA_REVIEWED_SHA', ''))
if a.mode == 'apply' or not a.ssh_rehearsal:
if os.environ.get('GITHUB_ACTIONS') != 'true' or os.environ.get('GITHUB_SHA') != a.sha:
raise ValueError('CI exact SHA required')
if os.environ.get('GITHUB_REF') != 'refs/heads/ops/seo-schema-stage-' + a.mode:
raise ValueError('Dedicated Stage operation ref required')
if a.ssh_rehearsal and a.mode != 'rehearse':
raise ValueError('SSH transport permits rehearsal only')
sql_bytes = (ROOT / SQL_PATH).read_bytes()
if hashlib.sha256(sql_bytes).hexdigest() != SQL_DIGEST:
raise ValueError('Reviewed SQL digest mismatch')
if subprocess.check_output(['git','-C',str(ROOT),'show',a.sha+':'+SQL_PATH]) != sql_bytes:
raise ValueError('Dirty SQL refused')
# A local read-only rehearsal may exercise pending tooling. CI must execute
# only committed operation and catalog sources from the event SHA.
if not a.ssh_rehearsal:
for rel in ('scripts/stage-schema/operation.py', 'scripts/stage-schema/catalog.py'):
if subprocess.check_output(['git','-C',str(ROOT),'show',a.sha+':'+rel]) != (ROOT/rel).read_bytes():
raise ValueError('Dirty operation source refused')
if a.mode == 'apply' and a.ssh_rehearsal:
raise ValueError('Live apply cannot use SSH rehearsal transport')
evidence = Path(a.evidence)
evidence.mkdir(mode=0o700, parents=True, exist_ok=False)
os.chmod(evidence, 0o700)
d = Docker(a.ssh_rehearsal)
owned = []
try:
# Share application deployment lock; never chmod operational lock/config.
runner = d.run(['inspect','-f','{{.Config.Image}}','gitea_runner']).decode().strip()
lock = d.run(['create','--name','canina-stage-deploy-lock','--pull=never',
'--network','none','--cap-drop=ALL','--security-opt=no-new-privileges',
'--mount','type=bind,src=/opt/services/canina,dst=/host-canina,readonly',
'--entrypoint','/bin/sh',runner,'-c',
'exec 9</host-canina/stage-backend-deploy.lock; flock -n 9 || exit 1; '
'printf "LOCKED\\n"; while :; do sleep 60; done']).decode().strip()
owned.append(lock)
d.run(['start',lock])
for _ in range(10):
if d.run(['logs',lock]).strip() == b'LOCKED': break
time.sleep(1)
else: raise ValueError('Host lock not acquired')
check_database(d.sql(STAGE, 'BEGIN READ ONLY; SELECT current_database(); COMMIT;'))
size = int(d.sql(STAGE, 'SELECT pg_database_size(current_database());'))
free = int(d.run(['exec',STAGE,'sh','-c','df -Pk /var/lib/postgresql/data | tail -1']).split()[3])*1024
if free < max(size*5, 256*1024*1024): raise ValueError('Insufficient backup/restore capacity')
if shutil.disk_usage(evidence).free < max(size*2, 128*1024*1024):
raise ValueError('Insufficient private backup capacity')
ledger_before = d.sql(STAGE, 'SELECT coalesce(jsonb_agg(to_jsonb(m) ORDER BY id),\'[]\'::jsonb) FROM _prisma_migrations m;')
def dump(schema):
args = ['exec', STAGE, 'sh','-c', 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --no-owner --no-acl -Fc' + (' --schema-only' if schema else '')]
return d.run(args,timeout=180)
backup = dump(a.mode == 'rehearse')
backup_path = evidence / ('stage-schema.dump' if a.mode == 'rehearse' else 'stage-full.dump')
fd = os.open(backup_path,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
with os.fdopen(fd,'wb') as f: f.write(backup)
def disposable():
name = 'seo-schema-rehearsal-' + uuid.uuid4().hex[:16]
cid = d.run(['run','-d','--name',name,'--pull=never','--network','none',
'-e','POSTGRES_HOST_AUTH_METHOD=trust','-e','POSTGRES_DB=rehearsal',IMAGE]).decode().strip()
owned.append(cid)
for _ in range(60):
try:
d.run(['exec',cid,'pg_isready','-h','127.0.0.1','-U','postgres','-d','rehearsal'],timeout=10)
return cid
except subprocess.CalledProcessError: time.sleep(1)
raise ValueError('Final TCP server readiness timed out')
reference = disposable()
d.sql(reference, 'CREATE TABLE public.blogs(id UUID PRIMARY KEY);')
d.sql(reference, transaction(sql_bytes.decode()))
expected = catalog(d, reference)
clone = disposable()
d.run(['exec','-i',clone,'pg_restore','-h','127.0.0.1','-U','postgres','-d','rehearsal',
'--no-owner','--no-acl','--exit-on-error'],data=backup)
d.sql(clone, transaction(sql_bytes.decode()))
validate(catalog(d,clone),expected)
# Prove repeat is safe and no catalog drift from idempotent migration.
d.sql(clone, transaction(sql_bytes.decode()))
validate(catalog(d,clone),expected)
if a.mode == 'apply':
d.sql(STAGE, transaction(sql_bytes.decode()))
validate(catalog(d,STAGE),expected)
ledger_after = d.sql(STAGE, 'SELECT coalesce(jsonb_agg(to_jsonb(m) ORDER BY id),\'[]\'::jsonb) FROM _prisma_migrations m;')
if ledger_before != ledger_after: raise ValueError('Historical Prisma ledger changed')
receipt = dict(sha=a.sha,sql_sha256=SQL_DIGEST,mode=a.mode,database='caninodb_stage',
backup_sha256=hashlib.sha256(backup).hexdigest(), backup_bytes=len(backup),
restore_scope='schema-only sanitized clone' if a.mode=='rehearse' else 'full data without ACL/owner',
catalog_sha256=hashlib.sha256(json.dumps(expected,sort_keys=True).encode()).hexdigest(),
ledger_policy='Prisma ledger untouched; separate operation receipt', ledger_unchanged=True)
(evidence/'receipt.json').write_text(json.dumps(receipt,indent=2)+'\n')
(evidence/'catalog.json').write_text(json.dumps(expected,indent=2)+'\n')
print(json.dumps(receipt))
finally:
for cid in reversed(owned):
try: d.run(['rm','-f',cid],timeout=30)
except Exception: print('Owned container cleanup failed: '+cid, file=__import__('sys').stderr)
if __name__ == '__main__': main()
+114
View File
@@ -0,0 +1,114 @@
"""Focused contract tests; reference catalogs are collected by the caller."""
import copy
import importlib.util
from pathlib import Path
import unittest
def load_catalog():
path = Path(__file__).with_name("catalog.py")
if not path.exists():
raise AssertionError("catalog.py helper has not been implemented")
spec = importlib.util.spec_from_file_location("stage_schema_catalog", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
# A small comparison fixture, NOT a replacement for the migration-derived
# reference DB. The caller collects the complete expected catalog with SQL.
BASELINE = {
"tables": {
"content_research_findings": {
"columns": {
"title": {"type": "character varying(255)", "not_null": True, "default": None},
"version": {"type": "integer", "not_null": True, "default": "0"},
},
"constraints": {
"content_research_findings_blog_id_fkey": {
"type": "f", "validated": True, "deferrable": False, "deferred": False,
"definition": "FOREIGN KEY (blog_id) REFERENCES blogs(id) ON UPDATE CASCADE ON DELETE SET NULL",
},
},
"indexes": {
"content_research_findings_blog_id_key": {
"definition": "CREATE UNIQUE INDEX content_research_findings_blog_id_key ON public.content_research_findings USING btree (blog_id)",
"valid": True, "ready": True, "unique": True,
},
},
},
"content_performance_snapshots": {"columns": {}, "constraints": {}, "indexes": {}},
},
"enums": {"ResearchFindingStatus": ["RESEARCHED", "ACTIONED", "EVALUATING", "SCALED", "PRUNED"]},
}
class CatalogTests(unittest.TestCase):
def test_exact_baseline_accepted(self):
self.assertIsNone(load_catalog().validate(copy.deepcopy(BASELINE), BASELINE))
def test_object_key_order_ignored(self):
reordered = dict(reversed(list(BASELINE.items())))
self.assertIsNone(load_catalog().validate(reordered, BASELINE))
def test_schema_drift_rejected(self):
table = ("tables", "content_research_findings")
cases = [
(table + ("columns", "title", "type"), "text"),
(table + ("columns", "version", "default"), "1"),
(table + ("columns", "title", "not_null"), False),
(("enums", "ResearchFindingStatus"), ["ACTIONED", "RESEARCHED", "EVALUATING", "SCALED", "PRUNED"]),
(("enums", "ResearchFindingStatus"), ["RESEARCHED"]),
(table + ("indexes", "content_research_findings_blog_id_key", "definition"), "CREATE INDEX wrong ON public.content_research_findings USING btree (title)"),
(table + ("indexes", "content_research_findings_blog_id_key", "valid"), False),
(table + ("indexes", "content_research_findings_blog_id_key", "ready"), False),
(table + ("indexes", "content_research_findings_blog_id_key", "unique"), False),
(table + ("constraints", "content_research_findings_blog_id_fkey", "definition"), "FOREIGN KEY (blog_id) REFERENCES blogs(id) ON DELETE CASCADE"),
(table + ("constraints", "content_research_findings_blog_id_fkey", "validated"), False),
(table + ("constraints", "content_research_findings_blog_id_fkey", "deferrable"), True),
]
for path, replacement in cases:
with self.subTest(path=path):
actual = copy.deepcopy(BASELINE)
target = actual
for key in path[:-1]:
target = target[key]
target[path[-1]] = replacement
with self.assertRaisesRegex(ValueError, "Scoped SEO catalog mismatch"):
load_catalog().validate(actual, BASELINE)
def test_missing_and_extra_scoped_objects_rejected(self):
for kind in ("columns", "constraints", "indexes"):
for extra in (False, True):
with self.subTest(kind=kind, extra=extra):
actual = copy.deepcopy(BASELINE)
objects = actual["tables"]["content_research_findings"][kind]
if extra:
objects["unexpected"] = {}
else:
objects.pop(next(iter(objects)))
with self.assertRaises(ValueError):
load_catalog().validate(actual, BASELINE)
def test_json_scalar_types_not_python_equality(self):
actual = copy.deepcopy(BASELINE)
actual["tables"]["content_research_findings"]["columns"]["title"]["not_null"] = 1
with self.assertRaises(ValueError):
load_catalog().validate(actual, BASELINE)
def test_query_scope_and_required_catalog_fields(self):
sql = load_catalog().CATALOG_SQL
self.assertIn("n.nspname = 'public'", sql)
self.assertIn("c.relname IN ('content_research_findings',", sql)
self.assertIn("'content_performance_snapshots')", sql)
self.assertIn("e_type.typname = 'ResearchFindingStatus'", sql)
self.assertNotIn("_prisma_migrations", sql)
for expression in ("format_type(a.atttypid, a.atttypmod)", "a.attnotnull", "pg_get_expr", "pg_get_constraintdef", "pg_get_indexdef", "i.indisvalid", "ORDER BY e.enumsortorder", "NOT a.attisdropped"):
with self.subTest(expression=expression):
self.assertIn(expression, sql)
self.assertEqual(sql.count(" AS catalog;"), 1)
if __name__ == "__main__":
unittest.main()
+32
View File
@@ -0,0 +1,32 @@
import importlib.util
from pathlib import Path
import unittest
p=Path(__file__).with_name('operation.py')
s=importlib.util.spec_from_file_location('operation',p)
assert s is not None and s.loader is not None
m=importlib.util.module_from_spec(s); s.loader.exec_module(m)
class Safety(unittest.TestCase):
def test_sha(self):
for bad in ('', 'main','a'*39,'A'*40,'a'*40+';'):
with self.assertRaises(ValueError): m.check_binding(bad,bad,'rehearse','')
def test_apply_needs_independent_binding(self):
with self.assertRaises(ValueError): m.check_binding('a'*40,'a'*40,'apply','b'*40)
def test_sha_mismatch(self):
with self.assertRaises(ValueError): m.check_binding('a'*40,'b'*40,'rehearse','')
def test_bindings(self):
m.check_binding('a'*40,'a'*40,'apply','a'*40)
m.check_binding('a'*40,'a'*40,'rehearse','')
def test_transaction_has_bounded_guards(self):
sql=m.transaction('SELECT 1;')
self.assertTrue(sql.startswith('BEGIN;'))
self.assertTrue(sql.endswith('COMMIT;'))
self.assertIn("lock_timeout='5s'",sql)
self.assertIn("statement_timeout='60s'",sql)
self.assertIn('pg_advisory_xact_lock',sql)
def test_live_db_identity(self):
with self.assertRaises(ValueError): m.check_database('caninodb')
m.check_database('caninodb_stage')
if __name__=='__main__': unittest.main()