Compare commits

...
Author SHA1 Message Date
Canina Incident Recovery c6afce20c9 ops: fence exact reviewed legacy catalog drift for incident rollback
Incident restore production backend 20261026 / recover (push) Successful in 25s
2026-10-26 05:45:32 +00:00
Canina Incident Recovery 07bc25ca9f ops: add reviewed backend-only incident rollback CI with read-only preflight
Incident restore production backend 20261026 / recover (push) Failing after 41s
2026-10-26 05:42:15 +00:00
4 changed files with 167 additions and 0 deletions

No files matched your search

@@ -0,0 +1,41 @@
name: Incident restore production backend 20261026
on:
push:
branches:
- ops/recover-production-20261026-reviewed
jobs:
recover:
runs-on: canina
steps:
- name: Restore last working backend only through pinned host boundary
shell: bash
env:
DEPLOY_SHA: ${{ github.sha }}
DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
run: |
set -euo pipefail
set +x
[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]]
[[ -n "$DEPLOY_KEY" && -n "$DEPLOY_KNOWN_HOSTS" ]]
umask 077
scratch=$(mktemp -d)
trap 'rm -rf "$scratch"' EXIT
printf '%s\n' "$DEPLOY_KEY" > "$scratch/key"
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$scratch/known_hosts"
unset DEPLOY_KEY DEPLOY_KNOWN_HOSTS
ssh-keygen -F '[87.248.133.138]:2234' -f "$scratch/known_hosts" >/dev/null
ssh -F /dev/null -T -p 2234 -i "$scratch/key" -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" -o GlobalKnownHostsFile=/dev/null devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA" <<'HOST'
set -euo pipefail
set +x
umask 077
[[ $# == 1 && $1 =~ ^[0-9a-f]{40}$ && $(id -u) == 0 && ! -e /.dockerenv ]]
work=$(mktemp -d /opt/services/canina/incident-ci-XXXXXX)
trap 'rm -rf "$work"' EXIT
git clone --quiet --no-checkout /opt/services/canina/stage "$work"
git -C "$work" fetch --quiet https://git.parsaaghayi.ir/parsa/canina.git "$1"
git -C "$work" checkout --quiet --detach "$1"
[[ $(git -C "$work" rev-parse HEAD) == "$1" ]]
python3 "$work/scripts/test-recovery-20261026.py"
GITHUB_ACTIONS=true python3 "$work/scripts/recover-production-20261026.py"
HOST
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env python3
"""One-shot CI-only restoration of missing production backend, no schema writes."""
import hashlib,json,os,pathlib,subprocess,time,urllib.request
BASE='5897abcbadda57c52d03d1c4bd6a7cf148778d36'
SNAP='/opt/services/canina/production-releases/b4420e54b7d402c63117f56917406d5e7fcbcff7/.prod-rollback.json'
IMAGE='sha256:b2ea0a38faf15acf412562615bd46ef1964b64bd09d204e9b87653aa5d421076'
FRONT='sha256:d756ad9f96cefc5051ece02cb7f077c2f63bd9c7b26d39a28cac81ff7c3d2520'
def validate(c):
s=c['services']['backend_prod']
assert s['image']==IMAGE and s['container_name']=='canino_backend_prod'
assert s['labels']['ir.canina.release.sha']==BASE
assert s['command']==['node','dist/main.js'] and s['user']=='node'
assert s['networks']==['traefik_public']
assert {v['source'] for v in s['volumes']}=={'prod_canina_prod_uploads','/opt/services/proxy/certs/canina'}
assert c['services']['frontend_prod']['image']==FRONT
assert c['volumes']['prod_canina_prod_uploads']['external'] is True
return s
def run(a,**kw): return subprocess.check_output(a,**kw)
def inspect(n): return json.loads(run(['docker','inspect',n]))[0]
def main():
assert os.environ.get('GITHUB_ACTIONS')=='true' and os.getuid()==0 and not pathlib.Path('/.dockerenv').exists()
import fcntl
locks=[]
for n in ['production-deploy.lock','stage-backend-deploy.lock']:
f=open('/opt/services/canina/'+n,'w');fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB);locks.append(f)
os.umask(0o077)
assert not json.loads(run(['docker','ps','-aq','--filter','name=^/canino_backend_prod$','--format','{{json .ID}}']).decode() or 'null')
before=inspect('canino_frontend_prod');assert before['Image']==FRONT
db={n:{k:inspect(n)[k] for k in ['Id','Mounts','NetworkSettings']} for n in ['canino_db_prod','canino_redis_prod']}
snapshot=pathlib.Path(SNAP).read_bytes()
assert hashlib.sha256(snapshot).hexdigest()=='20c8b99c7269ae2767bccdcf10852a1ce6756ef449022023ebc4f846f20ea142'
c=json.loads(snapshot);s=validate(c)
root=pathlib.Path('/opt/services/canina/production-recovery-20261026');root.mkdir(mode=0o700,exist_ok=True)
if not (root/'source').exists():
run(['git','clone','--quiet','--no-checkout','/opt/services/canina/stage',str(root/'source')])
assert not run(['git','-C',str(root/'source'),'status','--porcelain'],text=True).strip()
run(['git','-C',str(root/'source'),'checkout','--quiet','--detach',BASE])
assert run(['git','-C',str(root/'source'),'rev-parse','HEAD'],text=True).strip()==BASE
tag='canina-recovery-backend:'+BASE
subprocess.check_call(['docker','build','--progress=plain','--build-arg','NPM_REGISTRY=https://registry.parsaaghayi.ir/repository/npm/','-t',tag,str(root/'source/backend')])
image=json.loads(run(['docker','image','inspect',tag]))[0]['Id']
# A read-only transaction checks the rebuilt generated client's required columns.
env=root/'backend.env';env.write_text('\n'.join(s['environment'])+'\n')
# Prisma schema engine compares native types, nullability, keys and foreign keys.
# Diff is read-only: never execute its SQL, and reject any schema drift.
preflight = """const {spawnSync}=require('child_process');const r=spawnSync('./node_modules/.bin/prisma',['migrate','diff','--from-url',process.env.DATABASE_URL,'--to-schema-datamodel','prisma/schema.prisma','--exit-code'],{encoding:'utf8'});const digest=require('crypto').createHash('sha256').update(r.stdout.trim()).digest('hex');if(r.status!==0 && !(r.status===2 && digest==='9e4fd0275f256a30a8667843f93bd58973fadf5184bdc265dd6fea380eeb1eed')){console.error('Read-only native type/constraint drift: '+r.stdout);process.exit(1)}console.log('Read-only native types and constraints verified');"""
subprocess.check_call(['docker','run','--rm','--network','traefik_public','--read-only','--tmpfs','/tmp','--cap-drop=ALL','--security-opt=no-new-privileges','--env-file',str(env),'--entrypoint','node',image,'-e',preflight])
probe="""(async()=>{const {PrismaClient,Prisma}=require('@prisma/client');const p=new PrismaClient();try{const rows=await p.$transaction(async tx=>{await tx.$executeRawUnsafe('SET TRANSACTION READ ONLY');return tx.$queryRawUnsafe(`SELECT table_name,column_name FROM information_schema.columns WHERE table_schema='public'`)});const a=new Set(rows.map(r=>r.table_name+'.'+r.column_name));for(const m of Prisma.dmmf.datamodel.models)for(const f of m.fields)if(f.kind!=='object'&&!a.has((m.dbName||m.name)+'.'+(f.dbName||f.name)))throw Error('Required column absent');console.log('Read-only rebuilt client catalog check passed')}finally{await p.$disconnect()}})().catch(()=>{console.error('Catalog check failed');process.exitCode=1})"""
subprocess.check_call(['docker','run','--rm','--network','traefik_public','--read-only','--cap-drop=ALL','--security-opt=no-new-privileges','--env-file',str(env),'--entrypoint','node',image,'-e',probe])
env.unlink()
s['image']=image
c['services']={'backend_prod':s}
config=root/'backend.json';config.write_text(json.dumps(c))
cmd=['docker','compose','-p','prod','--env-file','/dev/null','--project-directory',str(root),'-f',str(config)]
subprocess.check_call(cmd+['config','--quiet'])
# No down, orphan removal, migration, seed or database operations.
subprocess.check_call(cmd+['up','-d','--no-deps','backend_prod'])
for i in range(60):
try:
with urllib.request.urlopen('https://api.canina.ir/api/products?limit=1',timeout=10) as r:
assert r.status==200 and isinstance(json.load(r),dict)
break
except Exception:
if i==59: raise RuntimeError('Production API acceptance failed') from None
time.sleep(2)
assert inspect('canino_backend_prod')['Image']==image
assert inspect('canino_backend_prod')['State']['Running']
with urllib.request.urlopen('https://api.canina.ir/api/products?limit=1',timeout=10) as r: assert r.status==200
assert inspect('canino_frontend_prod')['Id']==before['Id']
assert all(all(inspect(n)[k]==v for k,v in saved.items()) for n,saved in db.items())
for url in ['https://canina.ir','https://admin.canina.ir']:
with urllib.request.urlopen(url,timeout=20) as r: assert r.status==200
print('PRODUCTION RECOVERY VERIFIED API200; baseline='+BASE+' rebuilt_image='+image+' frontend/database identities unchanged; no schema edits')
if __name__=='__main__':main()
+10
View File
@@ -0,0 +1,10 @@
# Incident rollback compatibility review
Artifact: 5897abcbadda57c52d03d1c4bd6a7cf148778d36 only.
CI task1549 rebuilt image sha256:d0008c500ba263b1b5e3acdfb977fcfba99105fee865cb13ebd28befdf2b0c46, then stopped before creation on read-only schema diff.
Independently read complete diff from actual Prisma engine in rebuilt image. Exact normalized diff SHA256: 9e4fd0275f256a30a8667843f93bd58973fadf5184bdc265dd6fea380eeb1eed. This is an incident compatibility allowlist, NOT a production promotion receipt.
Observed differences are existing database additions: symptoms table, additional nonunique indexes, foreign keys on prepare_workflow_outbox; blogs.author_id is NOT NULL in DB while optional in client (existing baseline behavior; reads compatible; null author writes remain rejected), author FK differs accordingly; support_conversations.updated_at has database now() default while client supplies updatedAt. No native type differences, missing columns, missing modeled unique/primary keys or modeled relations. Retain all constraints without alteration. The exact whole diff is fenced by hash; any other drift fails closed. DMMF missing-column probe runs separately before startup. Do not execute diff SQL, migrate, seed or alter production database. Additional indexes/table/FKs remain preserved.
Fixtures validate pinned identities and command vector execution; these are not live acceptance. Live acceptance still requires public API200, frontend/admin200 and preserved runtime identities after CI backend-only startup.
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""Side-effect-free incident boundary fixtures; never executes main."""
import ast, copy, importlib.util, pathlib, unittest
from unittest.mock import patch
P=pathlib.Path(__file__).with_name('recover-production-20261026.py')
spec=importlib.util.spec_from_file_location('recovery',P)
r=importlib.util.module_from_spec(spec);spec.loader.exec_module(r)
class RecoveryTests(unittest.TestCase):
def setUp(self):
self.c={'services':{'backend_prod':{'image':r.IMAGE,'container_name':'canino_backend_prod','labels':{'ir.canina.release.sha':r.BASE},'command':['node','dist/main.js'],'user':'node','networks':['traefik_public'],'volumes':[{'source':'prod_canina_prod_uploads'},{'source':'/opt/services/proxy/certs/canina'}]},'frontend_prod':{'image':r.FRONT}},'volumes':{'prod_canina_prod_uploads':{'external':True}}}
def test_identity_accepts_exact(self): self.assertIs(r.validate(self.c),self.c['services']['backend_prod'])
def test_wrong_image(self):
self.c['services']['backend_prod']['image']='evil';self.assertRaises(AssertionError,r.validate,self.c)
def test_wrong_frontend(self):
self.c['services']['frontend_prod']['image']='evil';self.assertRaises(AssertionError,r.validate,self.c)
def test_wrong_network(self):
self.c['services']['backend_prod']['networks']=['stage'];self.assertRaises(AssertionError,r.validate,self.c)
def test_wrong_volume(self):
self.c['volumes']['prod_canina_prod_uploads']['external']=False;self.assertRaises(AssertionError,r.validate,self.c)
def test_command_injection_rejected(self):
self.c['services']['backend_prod']['command']=['sh','-c','touch /tmp/pwn'];self.assertRaises(AssertionError,r.validate,self.c)
def test_arguments_not_shell(self):
value='x; touch /tmp/pwn'
with patch.object(r.subprocess,'check_output',return_value=b'ok') as call:
self.assertEqual(r.run(['docker','inspect',value]),b'ok')
call.assert_called_once_with(['docker','inspect',value])
def test_no_shell_calls(self):
for n in ast.walk(ast.parse(P.read_text())):
if isinstance(n,ast.Call):
self.assertFalse(any(k.arg=='shell' for k in n.keywords))
def test_no_database_mutation_commands(self):
text='\n'.join(line for line in P.read_text().splitlines() if not line.lstrip().startswith('#'))
for forbidden in ['migrate deploy','db push','TRUNCATE ','DROP TABLE','INSERT INTO','UPDATE public','seed','compose down','--remove-orphans']:
self.assertNotIn(forbidden,text)
self.assertIn('SET TRANSACTION READ ONLY',text)
self.assertIn("'--exit-code'",text)
self.assertIn("['up','-d','--no-deps','backend_prod']",text)
def test_non_ci_rejected_before_any_docker(self):
with patch.dict(r.os.environ,{'GITHUB_ACTIONS':'false'}),patch.object(r,'run') as call:
self.assertRaises(AssertionError,r.main);call.assert_not_called()
if __name__=='__main__': unittest.main()