feat(deploy): gate immutable production CI promotion on audited Stage acceptance

This commit is contained in:
Canina Engineering committed 2026-10-25 06:52:32 +00:00
1 parent 5f5c3eb3b9
commit 448c250944
6 files changed
+268 -11

No files matched your search

+9 -6
View File
@@ -3,7 +3,7 @@
set -euo pipefail
set +x
fail() { printf 'ERROR: %s\n' "$1" >&2; exit 2; }
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${BRANCH:-} == develop && $# == 0 ]] || fail 'CI-only Stage deployment; Production is blocked'
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ( ${BRANCH:-} == develop || ${BRANCH:-} == main ) && $# == 0 ]] || fail 'CI-only immutable deployment'
[[ ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ ]] || fail 'immutable DEPLOY_SHA required'
# Repository secret DEPLOY_KEY is the historical approved devops SSH credential.
# Never fall back to baked-in runner keys, agent forwarding or docker.sock.
@@ -22,12 +22,12 @@ ssh -F /dev/null -T -p 2234 -i "$scratch/key" \
-o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none \
-o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" \
-o GlobalKnownHostsFile=/dev/null -o ConnectTimeout=15 \
devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA" <<'HOST'
devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA $BRANCH" <<'HOST'
set -euo pipefail
set +x
umask 077
fail() { printf 'ERROR: host context: %s\n' "$1" >&2; exit 2; }
[[ $# == 1 && $1 =~ ^[0-9a-f]{40}$ ]] || fail 'immutable SHA required'
[[ $# == 2 && $1 =~ ^[0-9a-f]{40}$ && ( $2 == develop || $2 == main ) ]] || fail 'immutable SHA and approved branch required'
[[ ! -e /.dockerenv && ! -e /run/.containerenv ]] || fail 'refusing container filesystem'
[[ $(id -u) == 0 ]] || fail 'noninteractive approved sudo required'
for tool in bash python3 docker git flock realpath install wget; do command -v "$tool" >/dev/null || fail "missing $tool"; done
@@ -36,10 +36,13 @@ for tool in bash python3 docker git flock realpath install wget; do command -v "
docker compose version >/dev/null
docker volume inspect stage_canina_stage_db stage_canina_stage_uploads >/dev/null
docker network inspect traefik_public >/dev/null
printf 'Host context validated; Stage commit=%s; runtime env present; Production blocked\n' "$1"
if [[ $2 == main ]]; then
[[ -s /opt/services/canina/production-acceptance/$1.json ]] || fail 'audited Stage acceptance receipt missing'
fi
printf 'Host context validated; branch=%s commit=%s\n' "$2" "$1"
script=$(mktemp)
trap 'rm -f "$script"' EXIT
wget --no-proxy -qO "$script" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/${1}/scripts/deploy.sh"
# CI marker exists only across this authenticated, fixed Stage boundary.
GITHUB_ACTIONS=true STAGE_PROVISION_ENABLED=true DEPLOY_SHA="$1" bash "$script" develop
# CI marker exists only across this authenticated, fixed host boundary.
GITHUB_ACTIONS=true STAGE_PROVISION_ENABLED=true DEPLOY_SHA="$1" bash "$script" "$2"
HOST
+11 -3
View File
@@ -1,9 +1,9 @@
#!/bin/bash
set -euo pipefail
set +x
# CI/CD-only Stage rollout; Production awaits separate reviewed authorization.
if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [ "${1:-}" != develop ] || [ "$#" != 1 ]; then
echo 'ERROR: refusing deployment outside CI/CD Stage rollout' >&2
# CI-only boundary: main is a code-only immutable promotion with private receipt.
if [ "${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}}" != true ] || [[ "${1:-}" != develop && "${1:-}" != main ]] || [ "$#" != 1 ]; then
echo 'ERROR: refusing deployment outside authenticated CI/CD rollout' >&2
exit 2
fi
if [[ ! "${DEPLOY_SHA:-}" =~ ^[0-9a-f]{40}$ ]]; then
@@ -11,6 +11,14 @@ if [[ ! "${DEPLOY_SHA:-}" =~ ^[0-9a-f]{40}$ ]]; then
exit 2
fi
# END ROLLOUT GUARD
if [[ $1 == main ]]; then
umask 077
helper=$(mktemp)
trap 'rm -f "$helper"' EXIT
wget --no-proxy -qO "$helper" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/${DEPLOY_SHA}/scripts/rollout-production.sh"
bash "$helper"
exit
fi
exec 9>/opt/services/canina/stage-backend-deploy.lock
flock -n 9 || { echo 'Stage deployment already in progress' >&2; exit 1; }
umask 077
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Private production acceptance and runtime snapshot helpers. Never log config."""
import json
import os
from pathlib import Path
import re
import stat
import subprocess
import sys
ALLOWED = {'nginx.conf', 'Dockerfile', 'backend/Dockerfile', '.dockerignore',
'.gitea/workflows/deploy.yml', 'scripts/deploy.sh', 'scripts/ci-host-deploy.sh',
'scripts/rollout-production.sh', 'scripts/production-release.py', 'scripts/test_production_release.py',
'scripts/compose.stage.yml', 'scripts/rollout-stage.sh', 'scripts/revalidation-runtime.py',
'scripts/provision-revalidation-env.py', 'scripts/tests/test_revalidation_runtime.py',
'backend/src/common/revalidation/revalidation.service.ts',
'backend/src/common/revalidation/revalidation.service.spec.ts',
'backend/src/orchestration-worker/worker-registration.spec.ts',
'backend/src/support-chat/support-chat.http.spec.ts',
'frontend/application/components/BlogPage.tsx',
'frontend/application/components/__tests__/BlogPage.sorting.test.tsx'}
def validate_receipt(r, sha, baseline, paths):
if not (r.get('accepted') is True and r.get('scope') == 'reliability-only'
and r.get('sha') == sha and r.get('baseline') == baseline
and re.fullmatch(r'https://git\.parsaaghayi\.ir/parsa/canina/actions/runs/[0-9]+', r.get('stage_run_url',''))
and isinstance(r.get('reviewer'), str) and r['reviewer'].strip()
and sorted(r.get('paths',[])) == sorted(paths)
and paths and set(paths) <= ALLOWED):
raise ValueError('Exact SHA, audited Stage acceptance and narrow reviewed diff required')
def private_json(path):
p=Path(path); s=p.lstat()
if not stat.S_ISREG(s.st_mode) or s.st_uid != 0 or s.st_mode & 0o077 or s.st_nlink != 1:
raise ValueError('Root-owned private regular acceptance receipt required')
return json.loads(p.read_text())
def snapshot(items):
c={'services':{},'volumes':{},'networks':{'traefik_public':{'external':True,'name':'traefik_public'}}}
for x in items:
cfg=x['Config']; labels=cfg.get('Labels') or {}; name=labels.get('com.docker.compose.service')
if name not in {'backend_prod','frontend_prod'} or labels.get('com.docker.compose.project')!='prod' or x['Name']!='/canino_'+name:
raise ValueError('Unexpected production identity')
if set(x['NetworkSettings']['Networks']) != {'traefik_public'} or x['HostConfig'].get('Privileged') or x['HostConfig'].get('PortBindings'):
raise ValueError('Unexpected production isolation')
s={'image':x['Image'],'container_name':x['Name'][1:],'restart':x['HostConfig']['RestartPolicy']['Name'],
'environment':cfg['Env'],'labels':{k:v for k,v in labels.items() if not k.startswith('com.docker.compose.')},
'networks':['traefik_public'],'volumes':[]}
for key, source in [('user','User'),('working_dir','WorkingDir'),('entrypoint','Entrypoint'),('command','Cmd')]:
if cfg.get(source): s[key]=cfg[source]
for m in x['Mounts']:
if m['Type']=='volume' and m.get('Name')=='prod_canina_prod_uploads' and m['Destination']=='/app/uploads':
c['volumes'][m['Name']]={'external':True,'name':m['Name']}
source=m['Name']
elif m['Type']=='bind' and m['Source']=='/opt/services/proxy/certs/canina' and m['Destination']=='/app/ssl_certs': source=m['Source']
else: raise ValueError('Unreviewed production mount')
s['volumes'].append({'type':m['Type'],'source':source,'target':m['Destination'],'read_only':not m['RW']})
if name=='backend_prod' and not any(v['target']=='/app/uploads' for v in s['volumes']): raise ValueError('Missing uploads')
c['services'][name]=s
return c
def main():
action=sys.argv[1]
if action=='receipt':
sha=os.environ['DEPLOY_SHA']; base=subprocess.check_output(['git','-C','/opt/services/canina/prod','rev-parse','HEAD'],text=True).strip()
paths=subprocess.check_output(['git','diff','--name-only',base,sha],text=True).splitlines()
r=private_json('/opt/services/canina/production-acceptance/'+sha+'.json')
validate_receipt(r,sha,base,paths)
# Bind audited evidence to the two currently running Stage artifacts.
stage=json.loads(subprocess.check_output(['docker','inspect','canino_backend_stage','canino_frontend_stage']))
actual={x['Name'][1:]:x['Image'] for x in stage}
if r.get('stage_images') != actual or not all(x['State']['Running'] for x in stage):
raise ValueError('Audited Stage artifacts no longer running')
for x in stage:
wd=x['Config']['Labels'].get('com.docker.compose.project.working_dir')
if wd != '/opt/services/canina/releases/'+sha:
raise ValueError('Stage source SHA mismatch')
marker=subprocess.check_output(['docker','exec','canino_backend_stage','cat','/app/STAGE_TEST_ARTIFACT_SHA'],text=True)
if marker != sha: raise ValueError('Stage backend artifact SHA mismatch')
elif action=='snapshot':
items=json.loads(subprocess.check_output(['docker','inspect','canino_backend_prod','canino_frontend_prod']))
c=snapshot(items)
Path('.prod-rollback.json').write_text(json.dumps(c))
sha=os.environ['DEPLOY_SHA']
for name,s in c['services'].items():
# Keep operational environment, labels, mounts and networks, not dirty checkout compose.
s['image']='canina-production-'+name+':'+sha
s.pop('entrypoint',None); s.pop('command',None)
s.setdefault('labels',{})['ir.canina.release.sha']=sha
s['env_file']=[{'path':str(Path.cwd()/'.revalidation.env'),'required':True}]
s['environment']=[v for v in s['environment'] if not v.startswith('REVALIDATION_SECRET=')]
s['build']={'context':str(Path.cwd() / ('backend' if name=='backend_prod' else '.')),'dockerfile':'Dockerfile'}
if name=='frontend_prod': s['build']['args']={'NEXT_PUBLIC_API_URL':'https://api.canina.ir/api','VITE_API_URL':'https://api.canina.ir'}
Path('.prod-compose.json').write_text(json.dumps(c))
else: raise ValueError('Unknown action')
if __name__=='__main__': main()
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env bash
# Invoked only by the authenticated CI host boundary; never run manually.
set -euo pipefail
set +x
fail() { printf 'ERROR: %s\n' "$1" >&2; exit 2; }
[[ ${GITHUB_ACTIONS:-${GITEA_ACTIONS:-}} == true && ${DEPLOY_SHA:-} =~ ^[0-9a-f]{40}$ && $# == 0 ]] || fail 'CI-only immutable production release'
[[ $(id -u) == 0 && ! -e /.dockerenv && ! -e /run/.containerenv ]] || fail 'approved host root context required'
exec 8>/opt/services/canina/production-deploy.lock
flock -n 8 || fail 'Production rollout already in progress'
# Acceptance depends on a Stage artifact: serialize against Stage replacing it.
exec 9>/opt/services/canina/stage-backend-deploy.lock
flock -n 9 || fail 'Stage rollout in progress'
umask 077
export DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1
ROOT=/opt/services/canina/production-releases
mkdir -p "$ROOT"
[[ ! -L "$ROOT" && $(realpath "$ROOT") == "$ROOT" ]] || fail 'unsafe release root'
TARGET_DIR="$ROOT/$DEPLOY_SHA"
# Existing directories (including failed attempts) are never overwritten.
mkdir "$TARGET_DIR"
git clone --no-checkout --single-branch --branch main https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
cd "$TARGET_DIR"
[[ $(git rev-parse origin/main) == "$DEPLOY_SHA" ]] || fail 'main must equal accepted Stage SHA (fast-forward promotion only)'
git checkout --detach "$DEPLOY_SHA"
python3 scripts/production-release.py receipt
# No schema/seed operation is allowed by this code-only release path.
# Operational checkout is intentionally dirty and is NEVER reset/pulled/rendered.
python3 scripts/provision-revalidation-env.py --environment prod --source /opt/services/canina/prod/.revalidation.env --output "$TARGET_DIR/.revalidation.env"
python3 - <<'PY'
from pathlib import Path
p=Path('nginx.conf');s=p.read_text()
for k,v in {'__BACKEND_HOST__':'backend_prod','__APP_HOST__':'canina.ir','__ADMIN_HOST__':'admin.canina.ir'}.items(): s=s.replace(k,v)
assert '__BACKEND_HOST__' not in s
p.write_text(s)
PY
# Build contexts must not contain private snapshots or env files.
printf '\n.prod-*.json\n.*.env\n' >> .dockerignore
printf '\n.prod-*.json\n' >> .git/info/exclude
python3 scripts/production-release.py snapshot
docker volume inspect prod_canina_prod_db prod_canina_prod_uploads >/dev/null
docker network inspect traefik_public >/dev/null
compose() { command docker compose -p prod --env-file /dev/null --project-directory "$TARGET_DIR" -f "$TARGET_DIR/$1" "${@:2}"; }
compose .prod-compose.json config --quiet
compose .prod-rollback.json config --quiet
compose .prod-compose.json --progress plain build backend_prod frontend_prod
# Snapshot uses immutable old image IDs, runtime environment, mounts and labels.
# DB/Redis/support bot are not in this app-only model and must not be orphan-pruned.
rollback() {
rc=$?
trap - EXIT
if [[ ${started:-false} == true ]]; then
if compose .prod-rollback.json up -d --no-deps --force-recreate backend_prod frontend_prod; then
printf 'Production failed; prior immutable app images restored; live acceptance still required\n' >&2
else
printf 'CRITICAL: automatic rollback failed; retain private release evidence and escalate\n' >&2
fi
fi
exit "$rc"
}
trap rollback EXIT
started=true
compose .prod-compose.json up -d --no-deps --force-recreate backend_prod frontend_prod
# Private acceptance output has no response bodies, SQL or credentials.
python3 - <<'PY'
import json, subprocess, time, urllib.request
from pathlib import Path
expected=json.loads(Path('.prod-compose.json').read_text())['services']
for name,s in expected.items():
x=json.loads(subprocess.check_output(['docker','inspect',s['container_name']]))[0]
image=json.loads(subprocess.check_output(['docker','image','inspect',s['image']]))[0]['Id']
assert x['State']['Running'] and x['Image']==image
for url in ['https://api.canina.ir/api/products?limit=1','https://canina.ir/blog','https://admin.canina.ir']:
for n in range(30):
try:
with urllib.request.urlopen(url,timeout=10) as r: assert r.status==200
break
except Exception:
if n==29: raise RuntimeError('Production public acceptance failed') from None
time.sleep(2)
PY
# Verify real backend-to-Next authentication, never expose the key or body.
docker exec canino_backend_prod node -e '
(async()=>{const key=process.env.REVALIDATION_SECRET;if(!key)throw Error();
const r=await fetch("http://frontend_prod:3000/api/revalidate",{method:"POST",headers:{"Content-Type":"application/json","x-revalidate-secret":key},body:JSON.stringify({paths:["/blog"]}),signal:AbortSignal.timeout(10000)});if(!r.ok)throw Error();})().catch(()=>{console.error("Production authenticated revalidation failed");process.exitCode=1});'
started=false
trap - EXIT
printf 'Production immutable code-only rollout complete: %s; protected/live feature acceptance remains required\n' "$DEPLOY_SHA"
+62
View File
@@ -0,0 +1,62 @@
import importlib.util
import tempfile
import unittest
from pathlib import Path
SPEC = importlib.util.spec_from_file_location('production', Path(__file__).with_name('production-release.py'))
m = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(m)
class ProductionGuards(unittest.TestCase):
def receipt(self):
return dict(sha='a'*40, baseline='b'*40, stage_run_url='https://git.parsaaghayi.ir/parsa/canina/actions/runs/123', reviewer='CEO delegated acceptance', accepted=True, scope='reliability-only', paths=['nginx.conf'])
def test_exact_stage_sha(self):
r=self.receipt()
m.validate_receipt(r, 'a'*40, 'b'*40, ['nginx.conf'])
with self.assertRaises(ValueError): m.validate_receipt(r, 'c'*40, 'b'*40, ['nginx.conf'])
def test_scope_and_baseline_fail_closed(self):
r=self.receipt()
for baseline, paths in [('c'*40, ['nginx.conf']), ('b'*40, ['backend/prisma/schema.prisma']), ('b'*40, ['nginx.conf','backend/src/other.ts'])]:
with self.assertRaises(ValueError): m.validate_receipt(r, 'a'*40, baseline, paths)
r['accepted']=False
with self.assertRaises(ValueError): m.validate_receipt(r,'a'*40,'b'*40,['nginx.conf'])
def test_receipt_rejects_non_gitea_evidence(self):
r=self.receipt();r['stage_run_url']='https://evil.test/run/1'
with self.assertRaises(ValueError): m.validate_receipt(r,'a'*40,'b'*40,['nginx.conf'])
def test_snapshot_preserves_runtime_and_immutable_image(self):
x={'Name':'/canino_backend_prod','Image':'sha256:'+'a'*64,'Config':{'Env':['DATABASE_URL=private'], 'Labels':{'com.docker.compose.project':'prod','com.docker.compose.service':'backend_prod','traefik.enable':'true'},'User':'node','WorkingDir':'/app','Entrypoint':['node'],'Cmd':['dist/main']},'Mounts':[{'Type':'volume','Name':'prod_canina_prod_uploads','Destination':'/app/uploads','RW':True},{'Type':'bind','Source':'/opt/services/proxy/certs/canina','Destination':'/app/ssl_certs','RW':True}],'NetworkSettings':{'Networks':{'traefik_public':{}}},'HostConfig':{'RestartPolicy':{'Name':'always'},'PortBindings':{},'Privileged':False}}
c=m.snapshot([x]); s=c['services']['backend_prod']
self.assertEqual(s['image'],x['Image'])
self.assertEqual(s['environment'],['DATABASE_URL=private'])
self.assertTrue(c['volumes']['prod_canina_prod_uploads']['external'])
self.assertNotIn('com.docker.compose.project',s['labels'])
x['Mounts'][0]['Name']='stage_canina_stage_uploads'
with self.assertRaises(ValueError): m.snapshot([x])
def test_private_receipt_permissions(self):
import os
with tempfile.TemporaryDirectory() as d:
p=Path(d)/'receipt.json';p.write_text('{}');p.chmod(0o644)
with self.assertRaises(ValueError): m.private_json(p)
p.unlink();p.symlink_to('/dev/null')
with self.assertRaises(ValueError): m.private_json(p)
def test_no_schema_or_orphan_mutations_in_production(self):
text=Path(__file__).with_name('rollout-production.sh').read_text()
for dangerous in ['migrate deploy','db push','--remove-orphans','docker compose down','git reset']:
self.assertNotIn(dangerous,text)
self.assertIn('trap rollback EXIT',text)
self.assertIn('x-revalidate-secret',text)
self.assertIn('prod_canina_prod_db prod_canina_prod_uploads',text)
def test_ci_and_nonsha_refused_before_effects(self):
import subprocess, os
for script,args in [('deploy.sh',['main']),('ci-host-deploy.sh',[]) ,('rollout-production.sh',[])]:
p=subprocess.run(['bash',str(Path(__file__).with_name(script)),*args],env={'PATH':os.environ['PATH']},capture_output=True)
self.assertEqual(p.returncode,2, p.stderr)
if __name__=='__main__': unittest.main()