Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c6afce20c9 | ||
|
|
07bc25ca9f | ||
|
|
99f0c0d626 | ||
|
|
1e27387016 | ||
|
|
c733a806bd | ||
|
|
23cbd693a8 | ||
|
|
366bc9fc88 | ||
|
|
b4420e54b7 | ||
|
|
44d74d35e6 |
No files matched your search
@@ -0,0 +1,41 @@
|
||||
name: Incident restore production backend 20261026
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- ops/recover-production-20261026-reviewed
|
||||
jobs:
|
||||
recover:
|
||||
runs-on: canina
|
||||
steps:
|
||||
- name: Restore last working backend only through pinned host boundary
|
||||
shell: bash
|
||||
env:
|
||||
DEPLOY_SHA: ${{ github.sha }}
|
||||
DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }}
|
||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ -n "$DEPLOY_KEY" && -n "$DEPLOY_KNOWN_HOSTS" ]]
|
||||
umask 077
|
||||
scratch=$(mktemp -d)
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
printf '%s\n' "$DEPLOY_KEY" > "$scratch/key"
|
||||
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$scratch/known_hosts"
|
||||
unset DEPLOY_KEY DEPLOY_KNOWN_HOSTS
|
||||
ssh-keygen -F '[87.248.133.138]:2234' -f "$scratch/known_hosts" >/dev/null
|
||||
ssh -F /dev/null -T -p 2234 -i "$scratch/key" -o BatchMode=yes -o IdentitiesOnly=yes -o IdentityAgent=none -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$scratch/known_hosts" -o GlobalKnownHostsFile=/dev/null devops@87.248.133.138 "sudo -n bash -s -- $DEPLOY_SHA" <<'HOST'
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
[[ $# == 1 && $1 =~ ^[0-9a-f]{40}$ && $(id -u) == 0 && ! -e /.dockerenv ]]
|
||||
work=$(mktemp -d /opt/services/canina/incident-ci-XXXXXX)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
git clone --quiet --no-checkout /opt/services/canina/stage "$work"
|
||||
git -C "$work" fetch --quiet https://git.parsaaghayi.ir/parsa/canina.git "$1"
|
||||
git -C "$work" checkout --quiet --detach "$1"
|
||||
[[ $(git -C "$work" rev-parse HEAD) == "$1" ]]
|
||||
python3 "$work/scripts/test-recovery-20261026.py"
|
||||
GITHUB_ACTIONS=true python3 "$work/scripts/recover-production-20261026.py"
|
||||
HOST
|
||||
@@ -16,3 +16,7 @@ playwright-report/
|
||||
playwright/.auth/
|
||||
blob-report/
|
||||
|
||||
|
||||
# Python CI helpers must not dirty immutable release checkouts.
|
||||
__pycache__/
|
||||
*.pyc
|
||||
@@ -0,0 +1,9 @@
|
||||
# Host disk retention
|
||||
|
||||
Successful authenticated CI rollout runs release-retention.py. Default keep policy is only the current active release per environment; Docker mounts and Compose working directories always protect older still-used artifacts. Dry/unpublished source and unknown ignored files are preserved, so this is not a hard quota. CI restores only its own tracked nginx/dockerignore render changes after successful acceptance. New clones use sparse checkout to omit the 85 MiB tracked graphify-out history. No node_modules were bundled in the audited releases: their ~218 MiB is tracked source/assets plus an independent ~44 MiB Git store. Historical dirty releases require individual source review; never reset them for cleanup.
|
||||
|
||||
Retained small private runtime configuration is copied mode0600 under .retained-runtime before deleting a clean obsolete release. This avoids secret deletion; this archive is not a release rollback and requires explicit secret retention review separately. No volumes, databases, sessions, evidence, images or build cache are deleted.
|
||||
|
||||
The CI-installed canina-disk-watch.timer checks every five minutes with stdlib Python, no model calls. It logs daemon.err and returns nonzero at >=85% usage or <10GiB free. This is local detection, NOT off-host paging. systemctl status canina-disk-watch.service and journalctl -t canina-disk-watch expose alarms. Monitoring does not guarantee the disk cannot fill.
|
||||
|
||||
Audit 2026-10-26: Docker daemon already bounds new json-file containers to 3x10MiB and BuildKit GC to 6GB. Fifteen pre-existing containers (including Nexus, Gitea, databases and proxy) have empty LogConfig and remain unbounded until their owners roll them through their own Git/CI. Journald has no explicit SystemMaxUse and currently occupies 1.3GB. Nexus data is real persistent storage; retention tasks were not authenticated/audited and no blobs were deleted. Neither unrelated service config nor daemon/journal configuration is changed by Canina CI. Worktree branches with unpushed commits remain intentionally retained. These residual growth mechanisms prevent any honest claim of a globally bounded host or <50GB total.
|
||||
@@ -109,6 +109,10 @@ export async function generateMetadata({ params }: { params: Promise<{ slug: str
|
||||
keywords: keywordsList,
|
||||
alternates: {
|
||||
canonical: canonicalUrl,
|
||||
languages: {
|
||||
'fa-IR': canonicalUrl,
|
||||
'x-default': canonicalUrl,
|
||||
},
|
||||
},
|
||||
robots: isStaging
|
||||
? {
|
||||
@@ -138,8 +142,6 @@ export async function generateMetadata({ params }: { params: Promise<{ slug: str
|
||||
images: [
|
||||
{
|
||||
url: ogImageUrl,
|
||||
width: 1200,
|
||||
height: 630,
|
||||
alt: blog.imageAlt || blog.title || title,
|
||||
},
|
||||
],
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { afterEach, expect, it, vi } from 'vitest';
|
||||
import { generateMetadata } from './page';
|
||||
|
||||
vi.mock('../../../lib/seo', () => ({
|
||||
getSeoConfig: async () => ({ brandNameFa: 'کنینا', canonicalBaseUrl: 'https://canina.ir', ogImageUrl: '/fallback.png' }),
|
||||
formatPageTitle: (title: string) => title,
|
||||
}));
|
||||
|
||||
afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); });
|
||||
|
||||
it('uses the exact canonical for monolingual self-referencing language hints', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => ({ title: 'Ear care', canonicalUrl: 'https://canina.ir/blog/ear-care/' }) }));
|
||||
const metadata = await generateMetadata({ params: Promise.resolve({ slug: 'ear-care' }) });
|
||||
expect(metadata.alternates).toEqual({ canonical: 'https://canina.ir/blog/ear-care', languages: { 'fa-IR': 'https://canina.ir/blog/ear-care', 'x-default': 'https://canina.ir/blog/ear-care' } });
|
||||
});
|
||||
|
||||
it('does not invent dimensions for a publisher supplied OG asset', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({ ok: true, json: async () => ({ title: 'Ear care', imageUrl: '/ear-800x450.jpg' }) }));
|
||||
const metadata = await generateMetadata({ params: Promise.resolve({ slug: 'ear-care' }) });
|
||||
expect((metadata.openGraph as any).images[0]).toEqual({ url: '/ear-800x450.jpg', alt: 'Ear care' });
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
import { expect, it } from 'vitest';
|
||||
import { resolveBlogAuthorIdentity } from './blogAuthor';
|
||||
|
||||
it('models the specialty team account as Organization without inventing a person', () => {
|
||||
expect(resolveBlogAuthorIdentity({ author: { firstName: 'کارشناس رفتار و نگهداری', lastName: 'تیم کنینا' }, authorTeam: 'پوست، مو و آلرژی' })).toEqual({
|
||||
type: 'Organization', name: 'کارشناس رفتار و نگهداری تیم کنینا', teamName: 'پوست، مو و آلرژی',
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps a named individual a Person even when affiliated with an editorial team', () => {
|
||||
expect(resolveBlogAuthorIdentity({ author: { firstName: 'سارا', lastName: 'احمدی' }, authorTeam: 'تیم کنینا' }).type).toBe('Person');
|
||||
});
|
||||
@@ -32,7 +32,10 @@ export function resolveBlogAuthorIdentity(blog: BlogAuthorInput): ResolvedBlogAu
|
||||
const teamName = blog.authorTeam?.trim() || BLOG_TEAM_FALLBACK;
|
||||
|
||||
if (personName) {
|
||||
return { type: 'Person', name: personName, teamName };
|
||||
// Specialty accounts identify themselves as a team, not a named individual.
|
||||
// An individual's authorTeam affiliation alone does not change their type.
|
||||
const isTeamAccount = /(?:^|\s)تیم(?:\s|$)/u.test(personName);
|
||||
return { type: isTeamAccount ? 'Organization' : 'Person', name: personName, teamName };
|
||||
}
|
||||
|
||||
return { type: 'Organization', name: BLOG_AUTHOR_FALLBACK, teamName };
|
||||
|
||||
@@ -35,6 +35,7 @@ git clone --no-checkout --single-branch --branch develop https://git.parsaaghayi
|
||||
cd "$TARGET_DIR"
|
||||
git cat-file -e "${DEPLOY_SHA}^{commit}"
|
||||
git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop
|
||||
git sparse-checkout set --no-cone '/*' '!/graphify-out/'
|
||||
git checkout --detach "$DEPLOY_SHA"
|
||||
[ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ]
|
||||
# Provision against the pristine approved SHA, before any tracked render edits.
|
||||
@@ -125,4 +126,9 @@ source scripts/rollout-stage.sh
|
||||
# This preserves the root-private intent journal and fails closed on timer failure.
|
||||
python3 scripts/install-stage-principal-cleanup.py
|
||||
stage_rollout
|
||||
python3 scripts/test-release-retention.py
|
||||
# Restore only CI-owned render edits after images and acceptance succeed.
|
||||
git restore -- .dockerignore nginx.conf
|
||||
python3 scripts/install-disk-watch.py
|
||||
python3 scripts/release-retention.py --apply --environment stage --current-sha "$DEPLOY_SHA" > /opt/services/canina/tmp/retention-stage.json
|
||||
echo 'Deployment for develop completed successfully!'
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""No-model threshold check. Exit 1 at >=85% used or <10GiB available."""
|
||||
import json,os,subprocess,sys
|
||||
|
||||
def pressure(total,available):return available < 10*1024**3 or (total-available)/total >= .85
|
||||
|
||||
def main():
|
||||
s=os.statvfs('/');total=s.f_blocks*s.f_frsize;available=s.f_bavail*s.f_frsize
|
||||
alert=pressure(total,available)
|
||||
message=json.dumps({'service':'canina-disk-watch','alert':alert,'total':total,'available':available})
|
||||
print(message)
|
||||
if alert:subprocess.run(['logger','-p','daemon.err','-t','canina-disk-watch',message],check=True)
|
||||
return int(alert)
|
||||
if __name__=='__main__':sys.exit(main())
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install deterministic host-only pressure watch through authenticated CI."""
|
||||
import os,pathlib,subprocess
|
||||
if os.environ.get('GITHUB_ACTIONS',os.environ.get('GITEA_ACTIONS'))!='true' or os.getuid()!=0 or pathlib.Path('/.dockerenv').exists():raise SystemExit('CI host root required')
|
||||
root=pathlib.Path('/opt/services/canina/.disk-watch');root.mkdir(mode=0o700,exist_ok=True)
|
||||
if root.is_symlink() or root.resolve()!=root:raise SystemExit('Unsafe installer path')
|
||||
(root/'disk-watch.py').write_bytes(pathlib.Path(__file__).with_name('disk-watch.py').read_bytes())
|
||||
pathlib.Path('/etc/systemd/system/canina-disk-watch.service').write_text('[Unit]\nDescription=Canina deterministic disk pressure check\n[Service]\nType=oneshot\nExecStart=/usr/bin/python3 /opt/services/canina/.disk-watch/disk-watch.py\nNoNewPrivileges=true\nProtectSystem=strict\nProtectHome=true\nPrivateTmp=true\n')
|
||||
pathlib.Path('/etc/systemd/system/canina-disk-watch.timer').write_text('[Unit]\nDescription=Check disk every five minutes without inference\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\n[Install]\nWantedBy=timers.target\n')
|
||||
subprocess.run(['systemctl','daemon-reload'],check=True)
|
||||
subprocess.run(['systemctl','enable','--now','canina-disk-watch.timer'],check=True)
|
||||
subprocess.run(['systemctl','is-active','--quiet','canina-disk-watch.timer'],check=True)
|
||||
@@ -41,7 +41,12 @@ ALLOWED = {'nginx.conf', 'Dockerfile', 'backend/Dockerfile', '.dockerignore',
|
||||
'frontend/application/components/BlogPostClient.tsx',
|
||||
'frontend/application/components/__tests__/BlogPostClient.test.tsx',
|
||||
'frontend/application/lib/extractFaqs.ts',
|
||||
'frontend/application/lib/extractFaqs.test.ts'}
|
||||
'frontend/application/lib/extractFaqs.test.ts',
|
||||
# Exact authorized publisher technical SEO repair; no frontend wildcard.
|
||||
'frontend/application/app/blog/[slug]/page.tsx',
|
||||
'frontend/application/app/blog/[slug]/publisher-seo.test.tsx',
|
||||
'frontend/application/lib/blogAuthor.ts',
|
||||
'frontend/application/lib/blogAuthor.test.ts'}
|
||||
|
||||
def validate_receipt(r, sha, baseline, paths):
|
||||
if not (r.get('accepted') is True and r.get('scope') == 'reliability-only'
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One-shot CI-only restoration of missing production backend, no schema writes."""
|
||||
import hashlib,json,os,pathlib,subprocess,time,urllib.request
|
||||
BASE='5897abcbadda57c52d03d1c4bd6a7cf148778d36'
|
||||
SNAP='/opt/services/canina/production-releases/b4420e54b7d402c63117f56917406d5e7fcbcff7/.prod-rollback.json'
|
||||
IMAGE='sha256:b2ea0a38faf15acf412562615bd46ef1964b64bd09d204e9b87653aa5d421076'
|
||||
FRONT='sha256:d756ad9f96cefc5051ece02cb7f077c2f63bd9c7b26d39a28cac81ff7c3d2520'
|
||||
def validate(c):
|
||||
s=c['services']['backend_prod']
|
||||
assert s['image']==IMAGE and s['container_name']=='canino_backend_prod'
|
||||
assert s['labels']['ir.canina.release.sha']==BASE
|
||||
assert s['command']==['node','dist/main.js'] and s['user']=='node'
|
||||
assert s['networks']==['traefik_public']
|
||||
assert {v['source'] for v in s['volumes']}=={'prod_canina_prod_uploads','/opt/services/proxy/certs/canina'}
|
||||
assert c['services']['frontend_prod']['image']==FRONT
|
||||
assert c['volumes']['prod_canina_prod_uploads']['external'] is True
|
||||
return s
|
||||
|
||||
def run(a,**kw): return subprocess.check_output(a,**kw)
|
||||
def inspect(n): return json.loads(run(['docker','inspect',n]))[0]
|
||||
def main():
|
||||
assert os.environ.get('GITHUB_ACTIONS')=='true' and os.getuid()==0 and not pathlib.Path('/.dockerenv').exists()
|
||||
import fcntl
|
||||
locks=[]
|
||||
for n in ['production-deploy.lock','stage-backend-deploy.lock']:
|
||||
f=open('/opt/services/canina/'+n,'w');fcntl.flock(f,fcntl.LOCK_EX|fcntl.LOCK_NB);locks.append(f)
|
||||
os.umask(0o077)
|
||||
assert not json.loads(run(['docker','ps','-aq','--filter','name=^/canino_backend_prod$','--format','{{json .ID}}']).decode() or 'null')
|
||||
before=inspect('canino_frontend_prod');assert before['Image']==FRONT
|
||||
db={n:{k:inspect(n)[k] for k in ['Id','Mounts','NetworkSettings']} for n in ['canino_db_prod','canino_redis_prod']}
|
||||
snapshot=pathlib.Path(SNAP).read_bytes()
|
||||
assert hashlib.sha256(snapshot).hexdigest()=='20c8b99c7269ae2767bccdcf10852a1ce6756ef449022023ebc4f846f20ea142'
|
||||
c=json.loads(snapshot);s=validate(c)
|
||||
root=pathlib.Path('/opt/services/canina/production-recovery-20261026');root.mkdir(mode=0o700,exist_ok=True)
|
||||
if not (root/'source').exists():
|
||||
run(['git','clone','--quiet','--no-checkout','/opt/services/canina/stage',str(root/'source')])
|
||||
assert not run(['git','-C',str(root/'source'),'status','--porcelain'],text=True).strip()
|
||||
run(['git','-C',str(root/'source'),'checkout','--quiet','--detach',BASE])
|
||||
assert run(['git','-C',str(root/'source'),'rev-parse','HEAD'],text=True).strip()==BASE
|
||||
tag='canina-recovery-backend:'+BASE
|
||||
subprocess.check_call(['docker','build','--progress=plain','--build-arg','NPM_REGISTRY=https://registry.parsaaghayi.ir/repository/npm/','-t',tag,str(root/'source/backend')])
|
||||
image=json.loads(run(['docker','image','inspect',tag]))[0]['Id']
|
||||
# A read-only transaction checks the rebuilt generated client's required columns.
|
||||
env=root/'backend.env';env.write_text('\n'.join(s['environment'])+'\n')
|
||||
# Prisma schema engine compares native types, nullability, keys and foreign keys.
|
||||
# Diff is read-only: never execute its SQL, and reject any schema drift.
|
||||
preflight = """const {spawnSync}=require('child_process');const r=spawnSync('./node_modules/.bin/prisma',['migrate','diff','--from-url',process.env.DATABASE_URL,'--to-schema-datamodel','prisma/schema.prisma','--exit-code'],{encoding:'utf8'});const digest=require('crypto').createHash('sha256').update(r.stdout.trim()).digest('hex');if(r.status!==0 && !(r.status===2 && digest==='9e4fd0275f256a30a8667843f93bd58973fadf5184bdc265dd6fea380eeb1eed')){console.error('Read-only native type/constraint drift: '+r.stdout);process.exit(1)}console.log('Read-only native types and constraints verified');"""
|
||||
subprocess.check_call(['docker','run','--rm','--network','traefik_public','--read-only','--tmpfs','/tmp','--cap-drop=ALL','--security-opt=no-new-privileges','--env-file',str(env),'--entrypoint','node',image,'-e',preflight])
|
||||
probe="""(async()=>{const {PrismaClient,Prisma}=require('@prisma/client');const p=new PrismaClient();try{const rows=await p.$transaction(async tx=>{await tx.$executeRawUnsafe('SET TRANSACTION READ ONLY');return tx.$queryRawUnsafe(`SELECT table_name,column_name FROM information_schema.columns WHERE table_schema='public'`)});const a=new Set(rows.map(r=>r.table_name+'.'+r.column_name));for(const m of Prisma.dmmf.datamodel.models)for(const f of m.fields)if(f.kind!=='object'&&!a.has((m.dbName||m.name)+'.'+(f.dbName||f.name)))throw Error('Required column absent');console.log('Read-only rebuilt client catalog check passed')}finally{await p.$disconnect()}})().catch(()=>{console.error('Catalog check failed');process.exitCode=1})"""
|
||||
subprocess.check_call(['docker','run','--rm','--network','traefik_public','--read-only','--cap-drop=ALL','--security-opt=no-new-privileges','--env-file',str(env),'--entrypoint','node',image,'-e',probe])
|
||||
env.unlink()
|
||||
s['image']=image
|
||||
c['services']={'backend_prod':s}
|
||||
config=root/'backend.json';config.write_text(json.dumps(c))
|
||||
cmd=['docker','compose','-p','prod','--env-file','/dev/null','--project-directory',str(root),'-f',str(config)]
|
||||
subprocess.check_call(cmd+['config','--quiet'])
|
||||
# No down, orphan removal, migration, seed or database operations.
|
||||
subprocess.check_call(cmd+['up','-d','--no-deps','backend_prod'])
|
||||
for i in range(60):
|
||||
try:
|
||||
with urllib.request.urlopen('https://api.canina.ir/api/products?limit=1',timeout=10) as r:
|
||||
assert r.status==200 and isinstance(json.load(r),dict)
|
||||
break
|
||||
except Exception:
|
||||
if i==59: raise RuntimeError('Production API acceptance failed') from None
|
||||
time.sleep(2)
|
||||
assert inspect('canino_backend_prod')['Image']==image
|
||||
assert inspect('canino_backend_prod')['State']['Running']
|
||||
with urllib.request.urlopen('https://api.canina.ir/api/products?limit=1',timeout=10) as r: assert r.status==200
|
||||
assert inspect('canino_frontend_prod')['Id']==before['Id']
|
||||
assert all(all(inspect(n)[k]==v for k,v in saved.items()) for n,saved in db.items())
|
||||
for url in ['https://canina.ir','https://admin.canina.ir']:
|
||||
with urllib.request.urlopen(url,timeout=20) as r: assert r.status==200
|
||||
print('PRODUCTION RECOVERY VERIFIED API200; baseline='+BASE+' rebuilt_image='+image+' frontend/database identities unchanged; no schema edits')
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,10 @@
|
||||
# Incident rollback compatibility review
|
||||
|
||||
Artifact: 5897abcbadda57c52d03d1c4bd6a7cf148778d36 only.
|
||||
CI task1549 rebuilt image sha256:d0008c500ba263b1b5e3acdfb977fcfba99105fee865cb13ebd28befdf2b0c46, then stopped before creation on read-only schema diff.
|
||||
|
||||
Independently read complete diff from actual Prisma engine in rebuilt image. Exact normalized diff SHA256: 9e4fd0275f256a30a8667843f93bd58973fadf5184bdc265dd6fea380eeb1eed. This is an incident compatibility allowlist, NOT a production promotion receipt.
|
||||
|
||||
Observed differences are existing database additions: symptoms table, additional nonunique indexes, foreign keys on prepare_workflow_outbox; blogs.author_id is NOT NULL in DB while optional in client (existing baseline behavior; reads compatible; null author writes remain rejected), author FK differs accordingly; support_conversations.updated_at has database now() default while client supplies updatedAt. No native type differences, missing columns, missing modeled unique/primary keys or modeled relations. Retain all constraints without alteration. The exact whole diff is fenced by hash; any other drift fails closed. DMMF missing-column probe runs separately before startup. Do not execute diff SQL, migrate, seed or alter production database. Additional indexes/table/FKs remain preserved.
|
||||
|
||||
Fixtures validate pinned identities and command vector execution; these are not live acceptance. Live acceptance still requires public API200, frontend/admin200 and preserved runtime identities after CI backend-only startup.
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Conservative CI retention: current mount users win; dirty/unpublished trees stay.
|
||||
Default is dry-run. No volumes, images, secrets or arbitrary workspaces are swept.
|
||||
"""
|
||||
import argparse,fcntl,json,os,pathlib,re,shutil,subprocess,time
|
||||
BASE=pathlib.Path('/opt/services/canina')
|
||||
def command(args):
|
||||
return subprocess.check_output(args,text=True,stderr=subprocess.PIPE).strip()
|
||||
def git(path,*args):return command(['git','-C',str(path),*args])
|
||||
def eligible(path,root,protected,*,clean,pushed):
|
||||
path=pathlib.Path(path);root=pathlib.Path(root)
|
||||
if path.is_symlink() or root.is_symlink() or path.parent!=root or path.resolve()!=path or root.resolve()!=root:return False
|
||||
if not re.fullmatch('[0-9a-f]{40}',path.name):return False
|
||||
return clean and pushed and not any(str(path)==p or str(path).startswith(p.rstrip('/')+'/') or p.startswith(str(path)+'/') for p in protected)
|
||||
def dependencies():
|
||||
ids=command(['docker','ps','-aq']).split()
|
||||
containers=json.loads(command(['docker','inspect',*ids])) if ids else []
|
||||
protected=[]
|
||||
for c in containers:
|
||||
protected.extend(m['Source'] for m in c['Mounts'] if m.get('Source','').startswith('/'))
|
||||
labels=c['Config'].get('Labels') or {}
|
||||
for key in ['com.docker.compose.project.working_dir','com.docker.compose.project.config_files']:
|
||||
protected.extend(p for p in labels.get(key,'').split(',') if p.startswith('/'))
|
||||
for pid in pathlib.Path('/proc').iterdir():
|
||||
if not pid.name.isdigit():continue
|
||||
for p in [pid/'cwd',*(pid/'fd').glob('*')]:
|
||||
try:
|
||||
target=os.readlink(p)
|
||||
if target.startswith(str(BASE)+'/'):protected.append(target)
|
||||
except (OSError,PermissionError):pass
|
||||
protected.extend(line.split()[4].replace('\\040',' ') for line in pathlib.Path('/proc/self/mountinfo').read_text().splitlines() if line.split()[4].startswith(str(BASE)+'/'))
|
||||
return sorted(set(protected)),[{'name':c['Name'],'mounts':c['Mounts'],'labels':{k:v for k,v in (c['Config'].get('Labels') or {}).items() if k.startswith('com.docker.compose.')}} for c in containers]
|
||||
def inspect(path):
|
||||
return {'head':git(path,'rev-parse','HEAD'),'branch':subprocess.run(['git','-C',str(path),'symbolic-ref','--short','-q','HEAD'],text=True,capture_output=True).stdout.strip(),'upstream':subprocess.run(['git','-C',str(path),'rev-parse','--abbrev-ref','@{upstream}'],text=True,capture_output=True).stdout.strip(),'status':git(path,'status','--porcelain=v1','--untracked-files=all'),'unpushed':git(path,'rev-list','HEAD','--not','--remotes'),'common':git(path,'rev-parse','--path-format=absolute','--git-common-dir')}
|
||||
RUNTIME_NAMES={'.support-chat.env','.revalidation.env','.stage-base.json','.stage-compose.json','.stage-runtime.json','.stage-runtime-merged.json','.prod-compose.json','.prod-rollback.json','.prod-snapshot.json','.prod-schema-evidence.json'}
|
||||
def disposable_state(path,environment):
|
||||
# Compare complete bytes against HEAD plus the exact tracked CI transforms.
|
||||
if git(path,'diff','--cached','--name-only'):raise RuntimeError('Staged source preserved')
|
||||
for name in git(path,'diff','--name-only').splitlines():
|
||||
if name not in {'.dockerignore','nginx.conf'}:raise RuntimeError('Dirty source preserved')
|
||||
original=subprocess.check_output(['git','-C',str(path),'show','HEAD:'+name])
|
||||
expected=original
|
||||
if name=='nginx.conf':
|
||||
values={'__BACKEND_HOST__':'backend_stage' if environment=='stage' else 'backend_prod','__APP_HOST__':'stage.canina.ir' if environment=='stage' else 'canina.ir','__ADMIN_HOST__':'stageadmin.canina.ir' if environment=='stage' else 'admin.canina.ir'}
|
||||
for key,value in values.items():expected=expected.replace(key.encode(),value.encode())
|
||||
else:expected+=b'\n.support-chat.env\n.revalidation.env\n.stage-*.json\n' if environment=='stage' else b'\n.prod-*.json\n.*.env\n'
|
||||
if (path/name).is_symlink() or (path/name).read_bytes()!=expected:raise RuntimeError('Unknown render delta preserved')
|
||||
others=set(git(path,'ls-files','--others','--exclude-standard').splitlines()+git(path,'ls-files','--others','--ignored','--exclude-standard').splitlines())
|
||||
private=[]
|
||||
for name in sorted(others):
|
||||
p=path/name
|
||||
if p.is_symlink() or not p.is_file():raise RuntimeError('Unknown runtime type')
|
||||
if name in RUNTIME_NAMES:private.append(name)
|
||||
elif re.fullmatch(r'scripts/__pycache__/[a-zA-Z0-9_-]+\.cpython-312\.pyc',name) and (path/'scripts'/(p.name.split('.cpython-')[0]+'.py')).is_file():pass
|
||||
else:raise RuntimeError('Unknown untracked/ignored file preserved')
|
||||
return private
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument('--apply',action='store_true');ap.add_argument('--current-sha',required=True);ap.add_argument('--environment',choices=['stage','production'],required=True);args=ap.parse_args()
|
||||
if not re.fullmatch('[0-9a-f]{40}',args.current_sha):raise SystemExit('Invalid current SHA')
|
||||
root=BASE/('releases' if args.environment=='stage' else 'production-releases')
|
||||
if root.is_symlink() or root.resolve()!=root:raise SystemExit('Unsafe release root')
|
||||
lock=os.open(str(BASE/'release-retention.lock'),os.O_CREAT|os.O_RDWR|os.O_NOFOLLOW,0o600)
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
protected,containers=dependencies()
|
||||
current=root/args.current_sha
|
||||
if git(current,'rev-parse','HEAD')!=args.current_sha:raise SystemExit('Current release identity mismatch')
|
||||
# Fail closed if Docker does not identify the accepted SHA as a current user.
|
||||
if not any(p==str(current) or p.startswith(str(current)+'/') for p in protected):raise SystemExit('Current SHA is not active')
|
||||
protected.append(str(current))
|
||||
report={'time':time.time(),'apply':args.apply,'current':str(current),'containers':containers,'protected_paths':protected,'entries':[]}
|
||||
for path in sorted(root.iterdir()):
|
||||
e={'path':str(path),'removed':False};report['entries'].append(e)
|
||||
try:
|
||||
if not eligible(path,root,protected,clean=True,pushed=True):e['reason']='active-or-unsafe';continue
|
||||
e.update(inspect(path))
|
||||
if e['unpushed']:e['reason']='unpushed';continue
|
||||
ignored_paths=disposable_state(path,args.environment)
|
||||
# Verify against the authoritative repository, not stale local remote refs.
|
||||
branch='develop' if args.environment=='stage' else 'main'
|
||||
git(path,'fetch','--quiet','origin',branch)
|
||||
git(path,'merge-base','--is-ancestor','HEAD','FETCH_HEAD')
|
||||
e['preserved_runtime_files']=ignored_paths
|
||||
e['reason']='eligible'
|
||||
if args.apply:
|
||||
latest,_=dependencies()
|
||||
if not eligible(path,root,latest,clean=True,pushed=git(path,'rev-list','HEAD','--not','--remotes')=='') or disposable_state(path,args.environment)!=ignored_paths:raise RuntimeError('Dependency/status changed')
|
||||
if pathlib.Path(e['common'])!=path/'.git' and e['status']:raise RuntimeError('Dirty linked worktree preserved')
|
||||
if ignored_paths:
|
||||
archive=BASE/'.retained-runtime'/args.environment/path.name
|
||||
archive.mkdir(parents=True,mode=0o700,exist_ok=True)
|
||||
if archive.resolve()!=archive:raise RuntimeError('Unsafe runtime archive')
|
||||
os.chmod(archive,0o700)
|
||||
for name in ignored_paths:
|
||||
destination=archive/name
|
||||
if destination.exists() or destination.is_symlink():raise RuntimeError('Runtime archive collision')
|
||||
shutil.copyfile(path/name,destination);os.chmod(destination,0o600)
|
||||
common=pathlib.Path(e['common'])
|
||||
if common!=path/'.git':
|
||||
command(['git','--git-dir',str(common),'worktree','remove',str(path)])
|
||||
else:
|
||||
if len(git(path,'worktree','list','--porcelain').split('worktree '))-1!=1:raise RuntimeError('Has child worktrees')
|
||||
if not shutil.rmtree.avoids_symlink_attacks:raise RuntimeError('No safe rmtree')
|
||||
shutil.rmtree(path)
|
||||
e['removed']=True
|
||||
except (subprocess.CalledProcessError,RuntimeError,OSError) as ex:e['reason']='inspection-failed:'+type(ex).__name__
|
||||
print(json.dumps(report,indent=2))
|
||||
if __name__=='__main__':main()
|
||||
@@ -21,6 +21,7 @@ mkdir "$TARGET_DIR"
|
||||
git clone --no-checkout --single-branch --branch main https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
|
||||
cd "$TARGET_DIR"
|
||||
[[ $(git rev-parse origin/main) == "$DEPLOY_SHA" ]] || fail 'main must equal accepted Stage SHA (fast-forward promotion only)'
|
||||
git sparse-checkout set --no-cone '/*' '!/graphify-out/'
|
||||
git checkout --detach "$DEPLOY_SHA"
|
||||
python3 scripts/production-release.py receipt
|
||||
# No schema/seed operation is allowed by this code-only release path.
|
||||
@@ -87,4 +88,8 @@ docker exec canino_backend_prod node -e '
|
||||
const r=await fetch("http://frontend_prod:3000/api/revalidate",{method:"POST",headers:{"Content-Type":"application/json","x-revalidate-secret":key},body:JSON.stringify({paths:["/blog"]}),signal:AbortSignal.timeout(10000)});if(!r.ok)throw Error();})().catch(()=>{console.error("Production authenticated revalidation failed");process.exitCode=1});'
|
||||
started=false
|
||||
trap - EXIT
|
||||
git restore -- .dockerignore nginx.conf
|
||||
python3 scripts/test-release-retention.py
|
||||
python3 scripts/install-disk-watch.py
|
||||
python3 scripts/release-retention.py --apply --environment production --current-sha "$DEPLOY_SHA" > /opt/services/canina/tmp/retention-production.json
|
||||
printf 'Production immutable code-only rollout complete: %s; protected/live feature acceptance remains required\n' "$DEPLOY_SHA"
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Side-effect-free incident boundary fixtures; never executes main."""
|
||||
import ast, copy, importlib.util, pathlib, unittest
|
||||
from unittest.mock import patch
|
||||
P=pathlib.Path(__file__).with_name('recover-production-20261026.py')
|
||||
spec=importlib.util.spec_from_file_location('recovery',P)
|
||||
r=importlib.util.module_from_spec(spec);spec.loader.exec_module(r)
|
||||
class RecoveryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.c={'services':{'backend_prod':{'image':r.IMAGE,'container_name':'canino_backend_prod','labels':{'ir.canina.release.sha':r.BASE},'command':['node','dist/main.js'],'user':'node','networks':['traefik_public'],'volumes':[{'source':'prod_canina_prod_uploads'},{'source':'/opt/services/proxy/certs/canina'}]},'frontend_prod':{'image':r.FRONT}},'volumes':{'prod_canina_prod_uploads':{'external':True}}}
|
||||
def test_identity_accepts_exact(self): self.assertIs(r.validate(self.c),self.c['services']['backend_prod'])
|
||||
def test_wrong_image(self):
|
||||
self.c['services']['backend_prod']['image']='evil';self.assertRaises(AssertionError,r.validate,self.c)
|
||||
def test_wrong_frontend(self):
|
||||
self.c['services']['frontend_prod']['image']='evil';self.assertRaises(AssertionError,r.validate,self.c)
|
||||
def test_wrong_network(self):
|
||||
self.c['services']['backend_prod']['networks']=['stage'];self.assertRaises(AssertionError,r.validate,self.c)
|
||||
def test_wrong_volume(self):
|
||||
self.c['volumes']['prod_canina_prod_uploads']['external']=False;self.assertRaises(AssertionError,r.validate,self.c)
|
||||
def test_command_injection_rejected(self):
|
||||
self.c['services']['backend_prod']['command']=['sh','-c','touch /tmp/pwn'];self.assertRaises(AssertionError,r.validate,self.c)
|
||||
def test_arguments_not_shell(self):
|
||||
value='x; touch /tmp/pwn'
|
||||
with patch.object(r.subprocess,'check_output',return_value=b'ok') as call:
|
||||
self.assertEqual(r.run(['docker','inspect',value]),b'ok')
|
||||
call.assert_called_once_with(['docker','inspect',value])
|
||||
def test_no_shell_calls(self):
|
||||
for n in ast.walk(ast.parse(P.read_text())):
|
||||
if isinstance(n,ast.Call):
|
||||
self.assertFalse(any(k.arg=='shell' for k in n.keywords))
|
||||
def test_no_database_mutation_commands(self):
|
||||
text='\n'.join(line for line in P.read_text().splitlines() if not line.lstrip().startswith('#'))
|
||||
for forbidden in ['migrate deploy','db push','TRUNCATE ','DROP TABLE','INSERT INTO','UPDATE public','seed','compose down','--remove-orphans']:
|
||||
self.assertNotIn(forbidden,text)
|
||||
self.assertIn('SET TRANSACTION READ ONLY',text)
|
||||
self.assertIn("'--exit-code'",text)
|
||||
self.assertIn("['up','-d','--no-deps','backend_prod']",text)
|
||||
def test_non_ci_rejected_before_any_docker(self):
|
||||
with patch.dict(r.os.environ,{'GITHUB_ACTIONS':'false'}),patch.object(r,'run') as call:
|
||||
self.assertRaises(AssertionError,r.main);call.assert_not_called()
|
||||
if __name__=='__main__': unittest.main()
|
||||
@@ -0,0 +1,45 @@
|
||||
import importlib.util,pathlib,tempfile,unittest,subprocess
|
||||
ROOT=pathlib.Path(__file__).resolve().parent
|
||||
spec=importlib.util.spec_from_file_location('retention',ROOT/'release-retention.py')
|
||||
r=importlib.util.module_from_spec(spec);spec.loader.exec_module(r)
|
||||
class RetentionTests(unittest.TestCase):
|
||||
def test_active_dirty_unpushed_and_symlink_protected(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root=pathlib.Path(d); old=root/('a'*40);old.mkdir();active=root/('b'*40);active.mkdir()
|
||||
self.assertFalse(r.eligible(old,root,[str(old/'backend')],clean=True,pushed=True))
|
||||
self.assertFalse(r.eligible(old,root,[],clean=False,pushed=True))
|
||||
self.assertFalse(r.eligible(old,root,[],clean=True,pushed=False))
|
||||
self.assertTrue(r.eligible(old,root,[str(active)],clean=True,pushed=True))
|
||||
link=root/('c'*40);link.symlink_to(old,target_is_directory=True)
|
||||
self.assertFalse(r.eligible(link,root,[],clean=True,pushed=True))
|
||||
self.assertFalse(r.eligible(root,root,[],clean=True,pushed=True))
|
||||
def test_inspect_real_git_dirty_and_unpushed(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=pathlib.Path(d)
|
||||
def g(*a):return subprocess.check_output(['git','-C',d,*a],stderr=subprocess.DEVNULL,text=True)
|
||||
g('init');g('config','user.email','test@example.invalid');g('config','user.name','Test')
|
||||
(p/'source').write_text('preserve');g('add','.');g('commit','-m','fixture')
|
||||
self.assertTrue(r.inspect(p)['unpushed'])
|
||||
g('update-ref','refs/remotes/origin/develop','HEAD')
|
||||
self.assertEqual(r.inspect(p)['unpushed'],'')
|
||||
(p/'untracked').write_text('preserve')
|
||||
self.assertIn('untracked',r.inspect(p)['status'])
|
||||
def test_only_exact_generated_changes_are_disposable(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=pathlib.Path(d)
|
||||
def g(*a):return subprocess.check_output(['git','-C',d,*a],stderr=subprocess.DEVNULL,text=True)
|
||||
g('init');g('config','user.email','test@example.invalid');g('config','user.name','Test')
|
||||
(p/'nginx.conf').write_text('proxy __BACKEND_HOST__ __APP_HOST__ __ADMIN_HOST__');(p/'.dockerignore').write_text('original\n');g('add','.');g('commit','-m','fixture')
|
||||
(p/'nginx.conf').write_text('proxy backend_stage stage.canina.ir stageadmin.canina.ir');(p/'.dockerignore').write_text('original\n\n.support-chat.env\n.revalidation.env\n.stage-*.json\n')
|
||||
(p/'.revalidation.env').write_text('private')
|
||||
self.assertEqual(r.disposable_state(p,'stage'),['.revalidation.env'])
|
||||
(p/'nginx.conf').write_text('uncommitted source')
|
||||
with self.assertRaises(RuntimeError):r.disposable_state(p,'stage')
|
||||
g('checkout','--','nginx.conf');(p/'unknown.env').write_text('unknown')
|
||||
with self.assertRaises(RuntimeError):r.disposable_state(p,'stage')
|
||||
def test_disk_pressure_thresholds(self):
|
||||
spec=importlib.util.spec_from_file_location('watch',ROOT/'disk-watch.py');w=importlib.util.module_from_spec(spec);spec.loader.exec_module(w)
|
||||
self.assertTrue(w.pressure(100*1024**3,9*1024**3))
|
||||
self.assertTrue(w.pressure(100*1024**3,15*1024**3))
|
||||
self.assertFalse(w.pressure(100*1024**3,30*1024**3))
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -64,6 +64,21 @@ class ProductionGuards(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
m.validate_receipt(r, 'a'*40, 'b'*40, [path])
|
||||
|
||||
def test_exact_publisher_seo_repair_paths_allowed_only(self):
|
||||
paths = ['frontend/application/app/blog/[slug]/page.tsx',
|
||||
'frontend/application/app/blog/[slug]/publisher-seo.test.tsx',
|
||||
'frontend/application/lib/blogAuthor.ts',
|
||||
'frontend/application/lib/blogAuthor.test.ts']
|
||||
r = self.receipt(); r['paths'] = paths
|
||||
m.validate_receipt(r, 'a'*40, 'b'*40, paths)
|
||||
for path in ['frontend/application/app/blog/[slug]/other.tsx',
|
||||
'frontend/application/lib/other.ts',
|
||||
'frontend/application/app/blog/page.tsx']:
|
||||
r['paths'] = [path]
|
||||
with self.assertRaises(ValueError):
|
||||
m.validate_receipt(r, 'a'*40, 'b'*40, [path])
|
||||
self.assertFalse(any('*' in path for path in m.ALLOWED))
|
||||
|
||||
def test_independent_typed_catalog_acceptance_required(self):
|
||||
r = self.receipt()
|
||||
r.pop('typed_catalog_acceptance', None)
|
||||
|
||||
@@ -61,6 +61,8 @@ class RolloutTests(unittest.TestCase):
|
||||
# This test exercises rollout/Compose, never host systemd installation.
|
||||
# The installer's guarded install contract has its own full unit suite.
|
||||
(source/'scripts/install-stage-principal-cleanup.py').write_text('raise SystemExit(0)\n')
|
||||
(source/'scripts/install-disk-watch.py').write_text('raise SystemExit(0)\n')
|
||||
(source/'scripts/release-retention.py').write_text('raise SystemExit(0)\n')
|
||||
shutil.copyfile(ROOT/'.dockerignore', source/'.dockerignore')
|
||||
(source/'nginx.conf').write_text('__BACKEND_HOST__')
|
||||
git('add', '.')
|
||||
@@ -92,6 +94,11 @@ class RolloutTests(unittest.TestCase):
|
||||
(tools/'docker').chmod(0o755)
|
||||
script = (ROOT/'scripts/deploy.sh').read_text()
|
||||
script = script.replace('/opt/services/canina/stage-backend-deploy.lock', str(base/'lock')).replace('/opt/services/canina/stage', str(stage)).replace('/opt/services/canina/releases', str(releases)).replace('https://git.parsaaghayi.ir/parsa/canina.git', str(source)).replace('[ -d /opt/services/proxy/certs/canina ]', 'true')
|
||||
script = script.replace('/opt/services/canina/tmp/retention-stage.json', str(base/'retention-stage.json'))
|
||||
# Fail closed if any copied installer unexpectedly reaches systemd.
|
||||
(tools/'systemctl').write_text('#!/bin/sh\nprintf "unexpected systemctl\\n" >&2\nexit 99\n')
|
||||
(tools/'systemctl').chmod(0o755)
|
||||
self.assertNotIn('> /opt/services/canina/tmp/retention-stage.json',script, 'Fixture must never write live retention evidence')
|
||||
deploy = base/'deploy.sh'
|
||||
deploy.write_text(script)
|
||||
env = dict(os.environ, GITHUB_ACTIONS='true', DEPLOY_SHA=sha, PATH=str(tools)+':'+os.environ['PATH'], TRACE=str(base/'trace'), FIXTURE=str(base/'config.json'))
|
||||
@@ -115,9 +122,10 @@ class RolloutTests(unittest.TestCase):
|
||||
release = releases/sha
|
||||
self.assertEqual(git('rev-parse', 'HEAD', cwd=release), sha)
|
||||
self.assertEqual((release/'.support-chat.env').stat().st_mode & 0o777, 0o600)
|
||||
self.assertIn('.stage-*.json', (release/'.dockerignore').read_text())
|
||||
self.assertIn('.support-chat.env', (release/'.dockerignore').read_text())
|
||||
self.assertIn('.revalidation.env', (release/'.dockerignore').read_text())
|
||||
if fail == '0': self.assertEqual(git('diff','--','.dockerignore','nginx.conf',cwd=release), '')
|
||||
if fail == '1': self.assertIn('.stage-*.json', (release/'.dockerignore').read_text())
|
||||
if fail == '1': self.assertIn('.support-chat.env', (release/'.dockerignore').read_text())
|
||||
if fail == '1': self.assertIn('.revalidation.env', (release/'.dockerignore').read_text())
|
||||
self.assertEqual((release/'.stage-compose.json').stat().st_mode & 0o777, 0o600)
|
||||
effective = json.loads((release/'.stage-compose.json').read_text())
|
||||
self.assertEqual(effective['volumes'], {n: {'name': 'stage_'+n, 'external': True} for n in ('canina_stage_db','canina_stage_uploads')})
|
||||
|
||||
Reference in new issue
Block a user