From 448c2509448dc288531d2ba02bae58c621523b60 Mon Sep 17 00:00:00 2001 From: Canina Engineering Date: Sun, 25 Oct 2026 06:52:32 +0000 Subject: [PATCH] feat(deploy): gate immutable production CI promotion on audited Stage acceptance --- .gitea/workflows/deploy.yml | 4 +- scripts/ci-host-deploy.sh | 15 +++-- scripts/deploy.sh | 14 ++++- scripts/production-release.py | 97 ++++++++++++++++++++++++++++++ scripts/rollout-production.sh | 87 +++++++++++++++++++++++++++ scripts/test_production_release.py | 62 +++++++++++++++++++ 6 files changed, 268 insertions(+), 11 deletions(-) create mode 100644 scripts/production-release.py create mode 100644 scripts/rollout-production.sh create mode 100644 scripts/test_production_release.py diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index a626cf50..2d6713a2 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -10,7 +10,7 @@ jobs: deploy: runs-on: canina steps: - - name: Authenticated host Stage deployment (Production blocked) + - name: Authenticated immutable host deployment (Production acceptance gated) shell: bash env: BRANCH: ${{ github.ref_name }} @@ -20,7 +20,7 @@ jobs: run: | set -euo pipefail set +x - [[ "${BRANCH}" == develop ]] || { echo 'Production blocked' >&2; exit 2; } + [[ "${BRANCH}" == develop || "${BRANCH}" == main ]] || exit 2 [[ "${DEPLOY_SHA}" =~ ^[0-9a-f]{40}$ ]] || exit 2 # Provision DEPLOY_KNOWN_HOSTS from an independently verified host key. # Never use baked-in /opt/services, runner SSH keys or docker.sock. diff --git a/scripts/ci-host-deploy.sh b/scripts/ci-host-deploy.sh index 8487ba1a..35bb5fb6 100644 --- a/scripts/ci-host-deploy.sh +++ b/scripts/ci-host-deploy.sh @@ -3,7 +3,7 @@ set -euo pipefail set +x fail() { printf 'ERROR: %s\n' "$1" >&2; exit 2; } -[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${BRANCH:-} == develop && $# == 0 ]] || fail 'CI-only Stage deployment; Production is blocked' +[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ( ${BRANCH:-} == develop || ${BRANCH:-} == main ) && $# == 0 ]] || fail 'CI-only immutable deployment' [[ ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || fail 'immutable DEPLOY_SHA required' # Repository secret DEPLOY_KEY is the historical approved devops SSH credential. # Never fall back to baked-in runner keys, agent forwarding or docker.sock. @@ -22,12 +22,12 @@ ssh -F /dev/null -T -p 2234 -i "$scratch/key" \ -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none \ -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" \ -o GlobalKnownHostsFile=/dev/null -o ConnectTimeout=15 \ - devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA" <<'HOST' + devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA $BRANCH" <<'HOST' set -euo pipefail set +x umask 077 fail() { printf 'ERROR: host context: %s\n' "$1" >&2; exit 2; } -[[ $# == 1 && $1 =~ ^[0-9a-f]{40}$ ]] || fail 'immutable SHA required' +[[ $# == 2 && $1 =~ ^[0-9a-f]{40}$ && ( $2 == develop || $2 == main ) ]] || fail 'immutable SHA and approved branch required' [[ ! -e /.dockerenv && ! -e /run/.containerenv ]] || fail 'refusing container filesystem' [[ $(id -u) == 0 ]] || fail 'noninteractive approved sudo required' for tool in bash python3 docker git flock realpath install wget; do command -v "$tool" >/dev/null || fail "missing $tool"; done @@ -36,10 +36,13 @@ for tool in bash python3 docker git flock realpath install wget; do command -v " docker compose version >/dev/null docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null docker network inspect traefik_public >/dev/null -printf 'Host context validated; Stage commit=%s; runtime env present; Production blocked\n' "$1" +if [[ $2 == main ]]; then + [[ -s /opt/services/canina/production-acceptance/$1.json ]] || fail 'audited Stage acceptance receipt missing' +fi +printf 'Host context validated; branch=%s commit=%s\n' "$2" "$1" script=$(mktemp) trap 'rm -f "$script"' EXIT wget --no-proxy -qO "$script" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/${1}/scripts/deploy.sh" -# CI marker exists only across this authenticated, fixed Stage boundary. -GITHUB_ACTIONS=true STAGE_PROVISION_ENABLED=true DEPLOY_SHA="$1" bash "$script" develop +# CI marker exists only across this authenticated, fixed host boundary. +GITHUB_ACTIONS=true STAGE_PROVISION_ENABLED=true DEPLOY_SHA="$1" bash "$script" "$2" HOST diff --git a/scripts/deploy.sh b/scripts/deploy.sh index f9424a72..25526d32 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,9 +1,9 @@ #!/bin/bash set -euo pipefail set +x -# CI/CD-only Stage rollout; Production awaits separate reviewed authorization. -if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [ "${1:-}" != develop ] || [ "$#" != 1 ]; then - echo 'ERROR: refusing deployment outside CI/CD Stage rollout' >&2 +# CI-only boundary: main is a code-only immutable promotion with private receipt. +if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [[ "${1:-}" != develop && "${1:-}" != main ]] || [ "$#" != 1 ]; then + echo 'ERROR: refusing deployment outside authenticated CI/CD rollout' >&2 exit 2 fi if [[ ! "${DEPLOY_SHA:-}" =~ ^[0-9a-f]{40}$ ]]; then @@ -11,6 +11,14 @@ if [[ ! "${DEPLOY_SHA:-}" =~ ^[0-9a-f]{40}$ ]]; then exit 2 fi # END ROLLOUT GUARD +if [[ $1 == main ]]; then + umask 077 + helper=$(mktemp) + trap 'rm -f "$helper"' EXIT + wget --no-proxy -qO "$helper" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/${DEPLOY_SHA}/scripts/rollout-production.sh" + bash "$helper" + exit +fi exec 9>/opt/services/canina/stage-backend-deploy.lock flock -n 9 || { echo 'Stage deployment already in progress' >&2; exit 1; } umask 077 diff --git a/scripts/production-release.py b/scripts/production-release.py new file mode 100644 index 00000000..32d5a7f4 --- /dev/null +++ b/scripts/production-release.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Private production acceptance and runtime snapshot helpers. Never log config.""" +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import sys + +ALLOWED = {'nginx.conf', 'Dockerfile', 'backend/Dockerfile', '.dockerignore', + '.gitea/workflows/deploy.yml', 'scripts/deploy.sh', 'scripts/ci-host-deploy.sh', + 'scripts/rollout-production.sh', 'scripts/production-release.py', 'scripts/test_production_release.py', + 'scripts/compose.stage.yml', 'scripts/rollout-stage.sh', 'scripts/revalidation-runtime.py', + 'scripts/provision-revalidation-env.py', 'scripts/tests/test_revalidation_runtime.py', + 'backend/src/common/revalidation/revalidation.service.ts', + 'backend/src/common/revalidation/revalidation.service.spec.ts', + 'backend/src/orchestration-worker/worker-registration.spec.ts', + 'backend/src/support-chat/support-chat.http.spec.ts', + 'frontend/application/components/BlogPage.tsx', + 'frontend/application/components/__tests__/BlogPage.sorting.test.tsx'} + +def validate_receipt(r, sha, baseline, paths): + if not (r.get('accepted') is True and r.get('scope') == 'reliability-only' + and r.get('sha') == sha and r.get('baseline') == baseline + and re.fullmatch(r'https://git\.parsaaghayi\.ir/parsa/canina/actions/runs/[0-9]+', r.get('stage_run_url','')) + and isinstance(r.get('reviewer'), str) and r['reviewer'].strip() + and sorted(r.get('paths',[])) == sorted(paths) + and paths and set(paths) <= ALLOWED): + raise ValueError('Exact SHA, audited Stage acceptance and narrow reviewed diff required') + +def private_json(path): + p=Path(path); s=p.lstat() + if not stat.S_ISREG(s.st_mode) or s.st_uid != 0 or s.st_mode & 0o077 or s.st_nlink != 1: + raise ValueError('Root-owned private regular acceptance receipt required') + return json.loads(p.read_text()) + +def snapshot(items): + c={'services':{},'volumes':{},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}} + for x in items: + cfg=x['Config']; labels=cfg.get('Labels') or {}; name=labels.get('com.docker.compose.service') + if name not in {'backend_prod','frontend_prod'} or labels.get('com.docker.compose.project')!='prod' or x['Name']!='/canino_'+name: + raise ValueError('Unexpected production identity') + if set(x['NetworkSettings']['Networks']) != {'traefik_public'} or x['HostConfig'].get('Privileged') or x['HostConfig'].get('PortBindings'): + raise ValueError('Unexpected production isolation') + s={'image':x['Image'],'container_name':x['Name'][1:],'restart':x['HostConfig']['RestartPolicy']['Name'], + 'environment':cfg['Env'],'labels':{k:v for k,v in labels.items() if not k.startswith('com.docker.compose.')}, + 'networks':['traefik_public'],'volumes':[]} + for key, source in [('user','User'),('working_dir','WorkingDir'),('entrypoint','Entrypoint'),('command','Cmd')]: + if cfg.get(source): s[key]=cfg[source] + for m in x['Mounts']: + if m['Type']=='volume' and m.get('Name')=='prod_canina_prod_uploads' and m['Destination']=='/app/uploads': + c['volumes'][m['Name']]={'external':True,'name':m['Name']} + source=m['Name'] + elif m['Type']=='bind' and m['Source']=='/opt/services/proxy/certs/canina' and m['Destination']=='/app/ssl_certs': source=m['Source'] + else: raise ValueError('Unreviewed production mount') + s['volumes'].append({'type':m['Type'],'source':source,'target':m['Destination'],'read_only':not m['RW']}) + if name=='backend_prod' and not any(v['target']=='/app/uploads' for v in s['volumes']): raise ValueError('Missing uploads') + c['services'][name]=s + return c + +def main(): + action=sys.argv[1] + if action=='receipt': + sha=os.environ['DEPLOY_SHA']; base=subprocess.check_output(['git','-C','/opt/services/canina/prod','rev-parse','HEAD'],text=True).strip() + paths=subprocess.check_output(['git','diff','--name-only',base,sha],text=True).splitlines() + r=private_json('/opt/services/canina/production-acceptance/'+sha+'.json') + validate_receipt(r,sha,base,paths) + # Bind audited evidence to the two currently running Stage artifacts. + stage=json.loads(subprocess.check_output(['docker','inspect','canino_backend_stage','canino_frontend_stage'])) + actual={x['Name'][1:]:x['Image'] for x in stage} + if r.get('stage_images') != actual or not all(x['State']['Running'] for x in stage): + raise ValueError('Audited Stage artifacts no longer running') + for x in stage: + wd=x['Config']['Labels'].get('com.docker.compose.project.working_dir') + if wd != '/opt/services/canina/releases/'+sha: + raise ValueError('Stage source SHA mismatch') + marker=subprocess.check_output(['docker','exec','canino_backend_stage','cat','/app/STAGE_TEST_ARTIFACT_SHA'],text=True) + if marker != sha: raise ValueError('Stage backend artifact SHA mismatch') + elif action=='snapshot': + items=json.loads(subprocess.check_output(['docker','inspect','canino_backend_prod','canino_frontend_prod'])) + c=snapshot(items) + Path('.prod-rollback.json').write_text(json.dumps(c)) + sha=os.environ['DEPLOY_SHA'] + for name,s in c['services'].items(): + # Keep operational environment, labels, mounts and networks, not dirty checkout compose. + s['image']='canina-production-'+name+':'+sha + s.pop('entrypoint',None); s.pop('command',None) + s.setdefault('labels',{})['ir.canina.release.sha']=sha + s['env_file']=[{'path':str(Path.cwd()/'.revalidation.env'),'required':True}] + s['environment']=[v for v in s['environment'] if not v.startswith('REVALIDATION_SECRET=')] + s['build']={'context':str(Path.cwd() / ('backend' if name=='backend_prod' else '.')),'dockerfile':'Dockerfile'} + if name=='frontend_prod': s['build']['args']={'NEXT_PUBLIC_API_URL':'https://api.canina.ir/api','VITE_API_URL':'https://api.canina.ir'} + Path('.prod-compose.json').write_text(json.dumps(c)) + else: raise ValueError('Unknown action') + +if __name__=='__main__': main() diff --git a/scripts/rollout-production.sh b/scripts/rollout-production.sh new file mode 100644 index 00000000..0576f6ec --- /dev/null +++ b/scripts/rollout-production.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Invoked only by the authenticated CI host boundary; never run manually. +set -euo pipefail +set +x +fail() { printf 'ERROR: %s\n' "$1" >&2; exit 2; } +[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ && $# == 0 ]] || fail 'CI-only immutable production release' +[[ $(id -u) == 0 && ! -e /.dockerenv && ! -e /run/.containerenv ]] || fail 'approved host root context required' +exec 8>/opt/services/canina/production-deploy.lock +flock -n 8 || fail 'Production rollout already in progress' +# Acceptance depends on a Stage artifact: serialize against Stage replacing it. +exec 9>/opt/services/canina/stage-backend-deploy.lock +flock -n 9 || fail 'Stage rollout in progress' +umask 077 +export DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 +ROOT=/opt/services/canina/production-releases +mkdir -p "$ROOT" +[[ ! -L "$ROOT" && $(realpath "$ROOT") == "$ROOT" ]] || fail 'unsafe release root' +TARGET_DIR="$ROOT/$DEPLOY_SHA" +# Existing directories (including failed attempts) are never overwritten. +mkdir "$TARGET_DIR" +git clone --no-checkout --single-branch --branch main https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR" +cd "$TARGET_DIR" +[[ $(git rev-parse origin/main) == "$DEPLOY_SHA" ]] || fail 'main must equal accepted Stage SHA (fast-forward promotion only)' +git checkout --detach "$DEPLOY_SHA" +python3 scripts/production-release.py receipt +# No schema/seed operation is allowed by this code-only release path. +# Operational checkout is intentionally dirty and is NEVER reset/pulled/rendered. +python3 scripts/provision-revalidation-env.py --environment prod --source /opt/services/canina/prod/.revalidation.env --output "$TARGET_DIR/.revalidation.env" +python3 - <<'PY' +from pathlib import Path +p=Path('nginx.conf');s=p.read_text() +for k,v in {'__BACKEND_HOST__':'backend_prod','__APP_HOST__':'canina.ir','__ADMIN_HOST__':'admin.canina.ir'}.items(): s=s.replace(k,v) +assert '__BACKEND_HOST__' not in s +p.write_text(s) +PY +# Build contexts must not contain private snapshots or env files. +printf '\n.prod-*.json\n.*.env\n' >> .dockerignore +printf '\n.prod-*.json\n' >> .git/info/exclude +python3 scripts/production-release.py snapshot +docker volume inspect prod_canina_prod_db prod_canina_prod_uploads >/dev/null +docker network inspect traefik_public >/dev/null +compose() { command docker compose -p prod --env-file /dev/null --project-directory "$TARGET_DIR" -f "$TARGET_DIR/$1" "${@:2}"; } +compose .prod-compose.json config --quiet +compose .prod-rollback.json config --quiet +compose .prod-compose.json --progress plain build backend_prod frontend_prod +# Snapshot uses immutable old image IDs, runtime environment, mounts and labels. +# DB/Redis/support bot are not in this app-only model and must not be orphan-pruned. +rollback() { + rc=$? + trap - EXIT + if [[ ${started:-false} == true ]]; then + if compose .prod-rollback.json up -d --no-deps --force-recreate backend_prod frontend_prod; then + printf 'Production failed; prior immutable app images restored; live acceptance still required\n' >&2 + else + printf 'CRITICAL: automatic rollback failed; retain private release evidence and escalate\n' >&2 + fi + fi + exit "$rc" +} +trap rollback EXIT +started=true +compose .prod-compose.json up -d --no-deps --force-recreate backend_prod frontend_prod +# Private acceptance output has no response bodies, SQL or credentials. +python3 - <<'PY' +import json, subprocess, time, urllib.request +from pathlib import Path +expected=json.loads(Path('.prod-compose.json').read_text())['services'] +for name,s in expected.items(): + x=json.loads(subprocess.check_output(['docker','inspect',s['container_name']]))[0] + image=json.loads(subprocess.check_output(['docker','image','inspect',s['image']]))[0]['Id'] + assert x['State']['Running'] and x['Image']==image +for url in ['https://api.canina.ir/api/products?limit=1','https://canina.ir/blog','https://admin.canina.ir']: + for n in range(30): + try: + with urllib.request.urlopen(url,timeout=10) as r: assert r.status==200 + break + except Exception: + if n==29: raise RuntimeError('Production public acceptance failed') from None + time.sleep(2) +PY +# Verify real backend-to-Next authentication, never expose the key or body. +docker exec canino_backend_prod node -e ' +(async()=>{const key=process.env.REVALIDATION_SECRET;if(!key)throw Error(); +const r=await fetch("http://frontend_prod:3000/api/revalidate",{method:"POST",headers:{"Content-Type":"application/json","x-revalidate-secret":key},body:JSON.stringify({paths:["/blog"]}),signal:AbortSignal.timeout(10000)});if(!r.ok)throw Error();})().catch(()=>{console.error("Production authenticated revalidation failed");process.exitCode=1});' +started=false +trap - EXIT +printf 'Production immutable code-only rollout complete: %s; protected/live feature acceptance remains required\n' "$DEPLOY_SHA" diff --git a/scripts/test_production_release.py b/scripts/test_production_release.py new file mode 100644 index 00000000..cc79b615 --- /dev/null +++ b/scripts/test_production_release.py @@ -0,0 +1,62 @@ +import importlib.util +import tempfile +import unittest +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location('production', Path(__file__).with_name('production-release.py')) +m = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(m) + +class ProductionGuards(unittest.TestCase): + def receipt(self): + return dict(sha='a'*40, baseline='b'*40, stage_run_url='https://git.parsaaghayi.ir/parsa/canina/actions/runs/123', reviewer='CEO delegated acceptance', accepted=True, scope='reliability-only', paths=['nginx.conf']) + + def test_exact_stage_sha(self): + r=self.receipt() + m.validate_receipt(r, 'a'*40, 'b'*40, ['nginx.conf']) + with self.assertRaises(ValueError): m.validate_receipt(r, 'c'*40, 'b'*40, ['nginx.conf']) + + def test_scope_and_baseline_fail_closed(self): + r=self.receipt() + for baseline, paths in [('c'*40, ['nginx.conf']), ('b'*40, ['backend/prisma/schema.prisma']), ('b'*40, ['nginx.conf','backend/src/other.ts'])]: + with self.assertRaises(ValueError): m.validate_receipt(r, 'a'*40, baseline, paths) + r['accepted']=False + with self.assertRaises(ValueError): m.validate_receipt(r,'a'*40,'b'*40,['nginx.conf']) + + def test_receipt_rejects_non_gitea_evidence(self): + r=self.receipt();r['stage_run_url']='https://evil.test/run/1' + with self.assertRaises(ValueError): m.validate_receipt(r,'a'*40,'b'*40,['nginx.conf']) + + def test_snapshot_preserves_runtime_and_immutable_image(self): + x={'Name':'/canino_backend_prod','Image':'sha256:'+'a'*64,'Config':{'Env':['DATABASE_URL=private'], 'Labels':{'com.docker.compose.project':'prod','com.docker.compose.service':'backend_prod','traefik.enable':'true'},'User':'node','WorkingDir':'/app','Entrypoint':['node'],'Cmd':['dist/main']},'Mounts':[{'Type':'volume','Name':'prod_canina_prod_uploads','Destination':'/app/uploads','RW':True},{'Type':'bind','Source':'/opt/services/proxy/certs/canina','Destination':'/app/ssl_certs','RW':True}],'NetworkSettings':{'Networks':{'traefik_public':{}}},'HostConfig':{'RestartPolicy':{'Name':'always'},'PortBindings':{},'Privileged':False}} + c=m.snapshot([x]); s=c['services']['backend_prod'] + self.assertEqual(s['image'],x['Image']) + self.assertEqual(s['environment'],['DATABASE_URL=private']) + self.assertTrue(c['volumes']['prod_canina_prod_uploads']['external']) + self.assertNotIn('com.docker.compose.project',s['labels']) + x['Mounts'][0]['Name']='stage_canina_stage_uploads' + with self.assertRaises(ValueError): m.snapshot([x]) + + def test_private_receipt_permissions(self): + import os + with tempfile.TemporaryDirectory() as d: + p=Path(d)/'receipt.json';p.write_text('{}');p.chmod(0o644) + with self.assertRaises(ValueError): m.private_json(p) + p.unlink();p.symlink_to('/dev/null') + with self.assertRaises(ValueError): m.private_json(p) + + def test_no_schema_or_orphan_mutations_in_production(self): + text=Path(__file__).with_name('rollout-production.sh').read_text() + for dangerous in ['migrate deploy','db push','--remove-orphans','docker compose down','git reset']: + self.assertNotIn(dangerous,text) + self.assertIn('trap rollback EXIT',text) + self.assertIn('x-revalidate-secret',text) + self.assertIn('prod_canina_prod_db prod_canina_prod_uploads',text) + + def test_ci_and_nonsha_refused_before_effects(self): + import subprocess, os + for script,args in [('deploy.sh',['main']),('ci-host-deploy.sh',[]) ,('rollout-production.sh',[])]: + p=subprocess.run(['bash',str(Path(__file__).with_name(script)),*args],env={'PATH':os.environ['PATH']},capture_output=True) + self.assertEqual(p.returncode,2, p.stderr) + +if __name__=='__main__': unittest.main()