Incident restore production backend 20261026 / recover (push) Failing after 41s
74 lines
6.0 KiB
Python
74 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
|
"""One-shot CI-only restoration of missing production backend, no schema writes."""
|
|
import hashlib,json,os,pathlib,subprocess,time,urllib.request
|
|
BASE='5897abcbadda57c52d03d1c4bd6a7cf148778d36'
|
|
SNAP='/opt/services/canina/production-releases/b4420e54b7d402c63117f56917406d5e7fcbcff7/.prod-rollback.json'
|
|
IMAGE='sha256:b2ea0a38faf15acf412562615bd46ef1964b64bd09d204e9b87653aa5d421076'
|
|
FRONT='sha256:d756ad9f96cefc5051ece02cb7f077c2f63bd9c7b26d39a28cac81ff7c3d2520'
|
|
def validate(c):
|
|
s=c['services']['backend_prod']
|
|
assert s['image']==IMAGE and s['container_name']=='canino_backend_prod'
|
|
assert s['labels']['ir.canina.release.sha']==BASE
|
|
assert s['command']==['node','dist/main.js'] and s['user']=='node'
|
|
assert s['networks']==['traefik_public']
|
|
assert {v['source'] for v in s['volumes']}=={'prod_canina_prod_uploads','/opt/services/proxy/certs/canina'}
|
|
assert c['services']['frontend_prod']['image']==FRONT
|
|
assert c['volumes']['prod_canina_prod_uploads']['external'] is True
|
|
return s
|
|
|
|
def run(a,**kw): return subprocess.check_output(a,**kw)
|
|
def inspect(n): return json.loads(run(['docker','inspect',n]))[0]
|
|
def main():
|
|
assert os.environ.get('GITHUB_ACTIONS')=='true' and os.getuid()==0 and not pathlib.Path('/.dockerenv').exists()
|
|
import fcntl
|
|
locks=[]
|
|
for n in ['production-deploy.lock','stage-backend-deploy.lock']:
|
|
f=open('/opt/services/canina/'+n,'w');fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB);locks.append(f)
|
|
os.umask(0o077)
|
|
assert not json.loads(run(['docker','ps','-aq','--filter','name=^/canino_backend_prod$','--format','{{json .ID}}']).decode() or 'null')
|
|
before=inspect('canino_frontend_prod');assert before['Image']==FRONT
|
|
db={n:{k:inspect(n)[k] for k in ['Id','Mounts','NetworkSettings']} for n in ['canino_db_prod','canino_redis_prod']}
|
|
snapshot=pathlib.Path(SNAP).read_bytes()
|
|
assert hashlib.sha256(snapshot).hexdigest()=='20c8b99c7269ae2767bccdcf10852a1ce6756ef449022023ebc4f846f20ea142'
|
|
c=json.loads(snapshot);s=validate(c)
|
|
root=pathlib.Path('/opt/services/canina/production-recovery-20261026');root.mkdir(mode=0o700)
|
|
run(['git','clone','--quiet','--no-checkout','/opt/services/canina/stage',str(root/'source')])
|
|
run(['git','-C',str(root/'source'),'checkout','--quiet','--detach',BASE])
|
|
assert run(['git','-C',str(root/'source'),'rev-parse','HEAD'],text=True).strip()==BASE
|
|
tag='canina-recovery-backend:'+BASE
|
|
subprocess.check_call(['docker','build','--progress=plain','--build-arg','NPM_REGISTRY=https://registry.parsaaghayi.ir/repository/npm/','-t',tag,str(root/'source/backend')])
|
|
image=json.loads(run(['docker','image','inspect',tag]))[0]['Id']
|
|
# A read-only transaction checks the rebuilt generated client's required columns.
|
|
env=root/'backend.env';env.write_text('\n'.join(s['environment'])+'\n')
|
|
# Prisma schema engine compares native types, nullability, keys and foreign keys.
|
|
# Diff is read-only: never execute its SQL, and reject any schema drift.
|
|
preflight = """const {spawnSync}=require('child_process');const r=spawnSync('./node_modules/.bin/prisma',['migrate','diff','--from-url',process.env.DATABASE_URL,'--to-schema-datamodel','prisma/schema.prisma','--exit-code'],{encoding:'utf8'});if(r.status!==0){console.error('Read-only native type/constraint drift: '+r.stdout);process.exit(1)}console.log('Read-only native types and constraints verified');"""
|
|
subprocess.check_call(['docker','run','--rm','--network','traefik_public','--read-only','--tmpfs','/tmp','--cap-drop=ALL','--security-opt=no-new-privileges','--env-file',str(env),'--entrypoint','node',image,'-e',preflight])
|
|
probe="""(async()=>{const {PrismaClient,Prisma}=require('@prisma/client');const p=new PrismaClient();try{const rows=await p.$transaction(async tx=>{await tx.$executeRawUnsafe('SET TRANSACTION READ ONLY');return tx.$queryRawUnsafe(`SELECT table_name,column_name FROM information_schema.columns WHERE table_schema='public'`)});const a=new Set(rows.map(r=>r.table_name+'.'+r.column_name));for(const m of Prisma.dmmf.datamodel.models)for(const f of m.fields)if(f.kind!=='object'&&!a.has((m.dbName||m.name)+'.'+(f.dbName||f.name)))throw Error('Required column absent');console.log('Read-only rebuilt client catalog check passed')}finally{await p.$disconnect()}})().catch(()=>{console.error('Catalog check failed');process.exitCode=1})"""
|
|
subprocess.check_call(['docker','run','--rm','--network','traefik_public','--read-only','--cap-drop=ALL','--security-opt=no-new-privileges','--env-file',str(env),'--entrypoint','node',image,'-e',probe])
|
|
env.unlink()
|
|
s['image']=image
|
|
c['services']={'backend_prod':s}
|
|
config=root/'backend.json';config.write_text(json.dumps(c))
|
|
cmd=['docker','compose','-p','prod','--env-file','/dev/null','--project-directory',str(root),'-f',str(config)]
|
|
subprocess.check_call(cmd+['config','--quiet'])
|
|
# No down, orphan removal, migration, seed or database operations.
|
|
subprocess.check_call(cmd+['up','-d','--no-deps','backend_prod'])
|
|
for i in range(60):
|
|
try:
|
|
with urllib.request.urlopen('https://api.canina.ir/api/products?limit=1',timeout=10) as r:
|
|
assert r.status==200 and isinstance(json.load(r),dict)
|
|
break
|
|
except Exception:
|
|
if i==59: raise RuntimeError('Production API acceptance failed') from None
|
|
time.sleep(2)
|
|
assert inspect('canino_backend_prod')['Image']==image
|
|
assert inspect('canino_backend_prod')['State']['Running']
|
|
with urllib.request.urlopen('https://api.canina.ir/api/products?limit=1',timeout=10) as r: assert r.status==200
|
|
assert inspect('canino_frontend_prod')['Id']==before['Id']
|
|
assert all(all(inspect(n)[k]==v for k,v in saved.items()) for n,saved in db.items())
|
|
for url in ['https://canina.ir','https://admin.canina.ir']:
|
|
with urllib.request.urlopen(url,timeout=20) as r: assert r.status==200
|
|
print('PRODUCTION RECOVERY VERIFIED API200; baseline='+BASE+' rebuilt_image='+image+' frontend/database identities unchanged; no schema edits')
|
|
if __name__=='__main__':main()
|