From c62d9836e95c3fd1a5bc196bb759b6416d0c4908 Mon Sep 17 00:00:00 2001 From: Canina Automation Date: Wed, 28 Oct 2026 07:53:18 +0000 Subject: [PATCH] ops(stage): version isolated SEO schema rehearsal and CI operation --- .gitea/workflows/stage-seo-schema.yml | 35 +++++ scripts/stage-schema/README.md | 22 +++ scripts/stage-schema/catalog.py | 122 +++++++++++++++ scripts/stage-schema/operation.py | 171 +++++++++++++++++++++ scripts/stage-schema/test_catalog.py | 114 ++++++++++++++ scripts/stage-schema/test_operation.py | 32 ++++ specs/003-stage-seo-schema/plan.md | 2 + specs/003-stage-seo-schema/spec.md | 2 + specs/003-stage-seo-schema/tasks.md | 5 + specs/003-stage-seo-schema/verification.md | 15 ++ 10 files changed, 520 insertions(+) create mode 100644 .gitea/workflows/stage-seo-schema.yml create mode 100644 scripts/stage-schema/README.md create mode 100644 scripts/stage-schema/catalog.py create mode 100644 scripts/stage-schema/operation.py create mode 100644 scripts/stage-schema/test_catalog.py create mode 100644 scripts/stage-schema/test_operation.py create mode 100644 specs/003-stage-seo-schema/plan.md create mode 100644 specs/003-stage-seo-schema/spec.md create mode 100644 specs/003-stage-seo-schema/tasks.md create mode 100644 specs/003-stage-seo-schema/verification.md diff --git a/.gitea/workflows/stage-seo-schema.yml b/.gitea/workflows/stage-seo-schema.yml new file mode 100644 index 00000000..e572a8fd --- /dev/null +++ b/.gitea/workflows/stage-seo-schema.yml @@ -0,0 +1,35 @@ +name: Stage SEO schema operation +on: + push: + branches: + - ops/seo-schema-stage-rehearse + - ops/seo-schema-stage-apply +jobs: + schema: + runs-on: canina + steps: + - name: Exact immutable Stage operation + env: + GITHUB_SHA: ${{ github.sha }} + GITHUB_REF: ${{ github.ref }} + GITHUB_ACTIONS: 'true' + STAGE_SCHEMA_REVIEWED_SHA: ${{ secrets.STAGE_SCHEMA_REVIEWED_SHA }} + run: | + set -eu + case "$GITHUB_REF" in + refs/heads/ops/seo-schema-stage-rehearse) mode=rehearse ;; + refs/heads/ops/seo-schema-stage-apply) mode=apply ;; + *) exit 2 ;; + esac + export no_proxy="git.parsaaghayi.ir,localhost,127.0.0.1,87.248.133.138" + echo "87.248.133.138 git.parsaaghayi.ir" >> /etc/hosts + unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY + root=$(mktemp -d) + chmod 700 "$root" + git init "$root/source" + git -C "$root/source" fetch --depth=1 https://git.parsaaghayi.ir/parsa/canina.git "$GITHUB_SHA" + git -C "$root/source" checkout --detach FETCH_HEAD + test "$(git -C "$root/source" rev-parse HEAD)" = "$GITHUB_SHA" + python3 "$root/source/scripts/stage-schema/operation.py" "$mode" --sha "$GITHUB_SHA" --evidence "$root/evidence" + # Private backups remain in this restricted runner directory; never upload dumps. + printf 'Private Stage evidence directory: %s\n' "$root/evidence" diff --git a/scripts/stage-schema/README.md b/scripts/stage-schema/README.md new file mode 100644 index 00000000..a488a192 --- /dev/null +++ b/scripts/stage-schema/README.md @@ -0,0 +1,22 @@ +# Stage-only SEO schema operation + +Normal develop/main deployment remains code-only. Gitea 1.22.6 supports the `push` event; this workflow uses only `ops/seo-schema-stage-rehearse` and `ops/seo-schema-stage-apply`, disjoint from application refs. No workflow_dispatch assumption, Production access, generic Prisma migration, seed, reset, or operational checkout edit. + +## Run after safety review + +1. Freeze/review exact operation commit, including workflow, `operation.py`, `catalog.py`, SQL and tests. Apply requires repository secret `STAGE_SCHEMA_REVIEWED_SHA` set through authorized repository-secret administration to that **full commit SHA**. This is a technical immutable-source binding, not a new business authorization requirement. Never log credentials. Rehearsal does not require this secret. +2. Transfer the committed source through the already pinned SSH host transport into an isolated server repository, never the dirty operational checkout. Inspect authoritative refs and push the exact commit non-force to `refs/heads/ops/seo-schema-stage-rehearse`. Do not push application develop/main. CI fetches the immutable event SHA through HTTPS (runner read-only transport verified). +3. Inspect actual terminal Gitea Actions log at exact SHA. The rehearsal restores a schema-only sanitized dump of actual Stage into isolated PostgreSQL16, applies exact digest twice and checks complete scoped typed catalog. It is **not** full-data recovery proof. +4. After parent review, provision exact reviewed-SHA secret and non-force push the identical commit to `refs/heads/ops/seo-schema-stage-apply`. This operation first takes a private full Stage dump, restores all data into an isolated clone, rehearses SQL, checks complete scoped types/defaults/nullability/enum/constraints/indexes, and only then applies bounded transactional SQL to `canino_db_stage` / `caninodb_stage`. Never set CI flags in SSH to bypass the supported path. +5. Inspect actual terminal log, restricted receipt and live typed catalog. Do not retry ambiguous DDL outcomes blindly. SQL is additive and reentrant; the runner never restores over Stage or drops live tables. Backup/restore uses no-owner/no-acl: it proves data recovery, not role/ACL or cluster disaster recovery. + +## Ledger and backup retention + +Historical `_prisma_migrations` remains byte-for-byte JSON equal before/after; no historical entries are synthesized, and no migration is marked applied. The incomplete baseline makes generic migrate deploy unsafe. A separate private receipt binds SHA, exact SQL digest, backup checksum/size and typed catalog hash. Reconcile this ledger policy before any future generic migration tooling use. + +CI prints only its restricted private evidence directory. Dumps are mode600 in a mode700 directory, never artifacts or Git. Retain backup and receipt through Stage acceptance; authorized retention cleanup must remove only that exact directory after recovery requirements are met. Disposable containers alone are cleaned by captured IDs. Existing configs, secrets, cron, database containers, application containers and catalog-mode commerce flags are untouched. Shared host deployment flock prevents overlap with app CI; existing lock helpers fail closed. + +Local diagnosis (read-only Stage plus disposable clone, never live DDL): +`python scripts/stage-schema/operation.py rehearse --sha --ssh-rehearsal --evidence ` + +Tests: `python -m unittest discover -s scripts/stage-schema -v`. Unit safety/typed tests do not substitute for actual CI execution or full-data restore. The operation pins the observed Stage PostgreSQL image ID and fails if unavailable; it never pulls an unreviewed replacement. diff --git a/scripts/stage-schema/catalog.py b/scripts/stage-schema/catalog.py new file mode 100644 index 00000000..47f701fc --- /dev/null +++ b/scripts/stage-schema/catalog.py @@ -0,0 +1,122 @@ +"""Read and compare the SEO migration's scoped PostgreSQL catalog. + +Execute CATALOG_SQL on both databases with the same PostgreSQL major version +and search_path (normally public). The expected value must come from a clean +reference database: create public.blogs(id UUID PRIMARY KEY), then apply the +reviewed migration. Do not derive the reference from the database under test. +This module owns no connection, process, migration execution, or credentials. +""" +import json +from typing import Any, Mapping, TypedDict + + +class ColumnCatalog(TypedDict): + type: str + not_null: bool + default: str | None + + +class ConstraintCatalog(TypedDict): + type: str + definition: str + validated: bool + deferrable: bool + deferred: bool + + +class IndexCatalog(TypedDict): + definition: str + valid: bool + ready: bool + unique: bool + + +class TableCatalog(TypedDict): + columns: dict[str, ColumnCatalog] + constraints: dict[str, ConstraintCatalog] + indexes: dict[str, IndexCatalog] + + +class Catalog(TypedDict): + tables: dict[str, TableCatalog] + enums: dict[str, list[str]] + + +# Names are the entire scope: do not include blogs or _prisma_migrations. +# Enumerating existing objects, rather than projecting known columns/indexes, +# also detects unexpected additions within the two owned tables. +CATALOG_SQL = r''' +WITH scoped_tables AS ( + SELECT c.oid, c.relname + FROM pg_catalog.pg_class AS c + JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND c.relname IN ('content_research_findings', + 'content_performance_snapshots') +), table_catalog AS ( + SELECT t.relname, jsonb_build_object( + 'columns', COALESCE(( + SELECT jsonb_object_agg(a.attname, jsonb_build_object( + 'type', pg_catalog.format_type(a.atttypid, a.atttypmod), + 'not_null', a.attnotnull, + 'default', pg_catalog.pg_get_expr(d.adbin, d.adrelid, false) + ) ORDER BY a.attname) + FROM pg_catalog.pg_attribute AS a + LEFT JOIN pg_catalog.pg_attrdef AS d + ON d.adrelid = a.attrelid AND d.adnum = a.attnum + WHERE a.attrelid = t.oid AND a.attnum > 0 AND NOT a.attisdropped + ), '{}'::jsonb), + 'constraints', COALESCE(( + SELECT jsonb_object_agg(k.conname, jsonb_build_object( + 'type', k.contype, + 'definition', pg_catalog.pg_get_constraintdef(k.oid, false), + 'validated', k.convalidated, + 'deferrable', k.condeferrable, + 'deferred', k.condeferred + ) ORDER BY k.conname) + FROM pg_catalog.pg_constraint AS k WHERE k.conrelid = t.oid + ), '{}'::jsonb), + 'indexes', COALESCE(( + SELECT jsonb_object_agg(ic.relname, jsonb_build_object( + 'definition', pg_catalog.pg_get_indexdef(i.indexrelid, 0, false), + 'valid', i.indisvalid, + 'ready', i.indisready, + 'unique', i.indisunique + ) ORDER BY ic.relname) + FROM pg_catalog.pg_index AS i + JOIN pg_catalog.pg_class AS ic ON ic.oid = i.indexrelid + WHERE i.indrelid = t.oid + ), '{}'::jsonb) + ) AS value FROM scoped_tables AS t +), enum_catalog AS ( + SELECT e_type.typname, + jsonb_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels + FROM pg_catalog.pg_type AS e_type + JOIN pg_catalog.pg_namespace AS n ON n.oid = e_type.typnamespace + JOIN pg_catalog.pg_enum AS e ON e.enumtypid = e_type.oid + WHERE n.nspname = 'public' AND e_type.typname = 'ResearchFindingStatus' + GROUP BY e_type.typname +) +SELECT jsonb_build_object( + 'tables', COALESCE((SELECT jsonb_object_agg(relname, value ORDER BY relname) + FROM table_catalog), '{}'::jsonb), + 'enums', COALESCE((SELECT jsonb_object_agg(typname, labels ORDER BY typname) + FROM enum_catalog), '{}'::jsonb) +) AS catalog; +''' + + +def validate(actual: Mapping[str, Any], expected: Mapping[str, Any]) -> None: + """Accept exact JSON equality, ignoring object-key order only. + + Array order is significant (especially enum labels); no fields, defaults, + definitions, or unknown scoped objects are stripped or normalized away. + Raise ValueError on drift, including both canonical catalogs for diagnosis. + """ + actual_json = json.dumps(actual, sort_keys=True, separators=(',', ':'), allow_nan=False) + expected_json = json.dumps(expected, sort_keys=True, separators=(',', ':'), allow_nan=False) + if actual_json != expected_json: + raise ValueError( + f"Scoped SEO catalog mismatch\nexpected: {expected_json}\nactual: {actual_json}" + ) diff --git a/scripts/stage-schema/operation.py b/scripts/stage-schema/operation.py new file mode 100644 index 00000000..5d4f1416 --- /dev/null +++ b/scripts/stage-schema/operation.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Bounded Stage-only SEO operation; no generic Prisma migration execution.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import time +import uuid +from catalog import CATALOG_SQL, validate + +ROOT = Path(__file__).resolve().parents[2] +SQL_PATH = 'backend/prisma/migrations/20261028120000_add_seo_tracking_contract/migration.sql' +SQL_DIGEST = 'de1386398026c6373ce8692fb165b904e189f830d7b20914d8a4b512a47f9d16' +IMAGE = 'sha256:de3a4eab8fdfa507ea92aac488b916b08089e515db49b055fe71dfa271ba3a28' +STAGE = 'canino_db_stage' + + +def check_binding(sha, source, mode, reviewed): + if not re.fullmatch('[0-9a-f]{40}', sha) or sha != source: + raise ValueError('Exact source/event SHA binding required') + if mode == 'apply' and reviewed != sha: + raise ValueError('Independent reviewed operation SHA required') + + +def check_database(name): + if name.strip() != 'caninodb_stage': + raise ValueError('Not Stage database') + + +class Docker: + def __init__(self, remote=False): + self.remote = remote + + def run(self, args, data=None, timeout=180): + cmd = ['docker'] + args + if self.remote: + cmd = ['ssh', '-F', '/opt/data/.ssh/config', '-o', 'StrictHostKeyChecking=yes', + '-o', 'UserKnownHostsFile=/opt/data/.ssh/known_hosts', 'parsa-host', shlex.join(cmd)] + return subprocess.run(cmd, input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=timeout, check=True).stdout + + def sql(self, container, query): + # Live identity and credentials are never read into Python or printed. + if container == STAGE: + args = ['exec', '-i', STAGE, 'sh', '-c', + 'exec psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"'] + else: + args = ['exec', '-i', container, 'psql', '-h', '127.0.0.1', '-U', 'postgres', + '-d', 'rehearsal', '-X', '-qAt', '-v', 'ON_ERROR_STOP=1'] + return self.run(args, query.encode()).decode().strip() + + +def catalog(docker, name): + return json.loads(docker.sql(name, 'SET search_path=public;\n' + CATALOG_SQL)) + + +def transaction(sql): + return ("BEGIN; SET LOCAL lock_timeout='5s'; SET LOCAL statement_timeout='60s'; " + "SELECT pg_advisory_xact_lock(731920260); SET LOCAL search_path=public;\n" + sql + '\nCOMMIT;') + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('mode', choices=['rehearse', 'apply']) + ap.add_argument('--sha', required=True) + ap.add_argument('--ssh-rehearsal', action='store_true') + ap.add_argument('--evidence', required=True) + a = ap.parse_args() + source = subprocess.check_output(['git', '-C', str(ROOT), 'rev-parse', 'HEAD'], text=True).strip() + check_binding(a.sha, source, a.mode, os.environ.get('STAGE_SCHEMA_REVIEWED_SHA', '')) + if a.mode == 'apply' or not a.ssh_rehearsal: + if os.environ.get('GITHUB_ACTIONS') != 'true' or os.environ.get('GITHUB_SHA') != a.sha: + raise ValueError('CI exact SHA required') + if os.environ.get('GITHUB_REF') != 'refs/heads/ops/seo-schema-stage-' + a.mode: + raise ValueError('Dedicated Stage operation ref required') + if a.ssh_rehearsal and a.mode != 'rehearse': + raise ValueError('SSH transport permits rehearsal only') + sql_bytes = (ROOT / SQL_PATH).read_bytes() + if hashlib.sha256(sql_bytes).hexdigest() != SQL_DIGEST: + raise ValueError('Reviewed SQL digest mismatch') + if subprocess.check_output(['git','-C',str(ROOT),'show',a.sha+':'+SQL_PATH]) != sql_bytes: + raise ValueError('Dirty SQL refused') + # A local read-only rehearsal may exercise pending tooling. CI must execute + # only committed operation and catalog sources from the event SHA. + if not a.ssh_rehearsal: + for rel in ('scripts/stage-schema/operation.py', 'scripts/stage-schema/catalog.py'): + if subprocess.check_output(['git','-C',str(ROOT),'show',a.sha+':'+rel]) != (ROOT/rel).read_bytes(): + raise ValueError('Dirty operation source refused') + if a.mode == 'apply' and a.ssh_rehearsal: + raise ValueError('Live apply cannot use SSH rehearsal transport') + evidence = Path(a.evidence) + evidence.mkdir(mode=0o700, parents=True, exist_ok=False) + os.chmod(evidence, 0o700) + d = Docker(a.ssh_rehearsal) + owned = [] + try: + # Share application deployment lock; never chmod operational lock/config. + runner = d.run(['inspect','-f','{{.Config.Image}}','gitea_runner']).decode().strip() + lock = d.run(['create','--name','canina-stage-deploy-lock','--pull=never', + '--network','none','--cap-drop=ALL','--security-opt=no-new-privileges', + '--mount','type=bind,src=/opt/services/canina,dst=/host-canina,readonly', + '--entrypoint','/bin/sh',runner,'-c', + 'exec 9