From 366bc9fc88e6ac452f3dd22addcadff95288acc1 Mon Sep 17 00:00:00 2001 From: devops Date: Mon, 26 Oct 2026 04:57:33 +0000 Subject: [PATCH] fix(ops): conservative post-rollout retention and deterministic disk watch --- .gitignore | 4 ++ docs/disk-retention.md | 9 ++++ scripts/deploy.sh | 6 +++ scripts/disk-watch.py | 14 +++++ scripts/install-disk-watch.py | 12 +++++ scripts/release-retention.py | 85 +++++++++++++++++++++++++++++++ scripts/rollout-production.sh | 5 ++ scripts/test-release-retention.py | 32 ++++++++++++ 8 files changed, 167 insertions(+) create mode 100644 docs/disk-retention.md create mode 100644 scripts/disk-watch.py create mode 100644 scripts/install-disk-watch.py create mode 100644 scripts/release-retention.py create mode 100644 scripts/test-release-retention.py diff --git a/.gitignore b/.gitignore index a82c5639..77de8897 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,7 @@ playwright-report/ playwright/.auth/ blob-report/ + +# Python CI helpers must not dirty immutable release checkouts. +__pycache__/ +*.pyc diff --git a/docs/disk-retention.md b/docs/disk-retention.md new file mode 100644 index 00000000..3fbb8c41 --- /dev/null +++ b/docs/disk-retention.md @@ -0,0 +1,9 @@ +# Host disk retention + +Successful authenticated CI rollout runs release-retention.py. Default keep policy is only the current active release per environment; Docker mounts and Compose working directories always protect older still-used artifacts. Dry/unpublished source and unknown ignored files are preserved, so this is not a hard quota. CI restores only its own tracked nginx/dockerignore render changes after successful acceptance. New clones use sparse checkout to omit the 85 MiB tracked graphify-out history. No node_modules were bundled in the audited releases: their ~218 MiB is tracked source/assets plus an independent ~44 MiB Git store. Historical dirty releases require individual source review; never reset them for cleanup. + +Retained small private runtime configuration is copied mode0600 under .retained-runtime before deleting a clean obsolete release. This avoids secret deletion; this archive is not a release rollback and requires explicit secret retention review separately. No volumes, databases, sessions, evidence, images or build cache are deleted. + +The CI-installed canina-disk-watch.timer checks every five minutes with stdlib Python, no model calls. It logs daemon.err and returns nonzero at >=85% usage or <10GiB free. This is local detection, NOT off-host paging. systemctl status canina-disk-watch.service and journalctl -t canina-disk-watch expose alarms. Monitoring does not guarantee the disk cannot fill. + +Audit 2026-10-26: Docker daemon already bounds new json-file containers to 3x10MiB and BuildKit GC to 6GB. Fifteen pre-existing containers (including Nexus, Gitea, databases and proxy) have empty LogConfig and remain unbounded until their owners roll them through their own Git/CI. Journald has no explicit SystemMaxUse and currently occupies 1.3GB. Nexus data is real persistent storage; retention tasks were not authenticated/audited and no blobs were deleted. Neither unrelated service config nor daemon/journal configuration is changed by Canina CI. Worktree branches with unpushed commits remain intentionally retained. These residual growth mechanisms prevent any honest claim of a globally bounded host or <50GB total. diff --git a/scripts/deploy.sh b/scripts/deploy.sh index c03643a8..dc7268f3 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -35,6 +35,7 @@ git clone --no-checkout --single-branch --branch develop https://git.parsaaghayi cd "$TARGET_DIR" git cat-file -e "${DEPLOY_SHA}^{commit}" git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop +git sparse-checkout set --no-cone '/*' '!/graphify-out/' git checkout --detach "$DEPLOY_SHA" [ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ] # Provision against the pristine approved SHA, before any tracked render edits. @@ -125,4 +126,9 @@ source scripts/rollout-stage.sh # This preserves the root-private intent journal and fails closed on timer failure. python3 scripts/install-stage-principal-cleanup.py stage_rollout +python3 scripts/test-release-retention.py +# Restore only CI-owned render edits after images and acceptance succeed. +git restore -- .dockerignore nginx.conf +python3 scripts/install-disk-watch.py +python3 scripts/release-retention.py --apply --environment stage --current-sha "$DEPLOY_SHA" > /opt/services/canina/tmp/retention-stage.json echo 'Deployment for develop completed successfully!' diff --git a/scripts/disk-watch.py b/scripts/disk-watch.py new file mode 100644 index 00000000..9d2aefaa --- /dev/null +++ b/scripts/disk-watch.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""No-model threshold check. Exit 1 at >=85% used or <10GiB available.""" +import json,os,subprocess,sys + +def pressure(total,available):return available < 10*1024**3 or (total-available)/total >= .85 + +def main(): + s=os.statvfs('/');total=s.f_blocks*s.f_frsize;available=s.f_bavail*s.f_frsize + alert=pressure(total,available) + message=json.dumps({'service':'canina-disk-watch','alert':alert,'total':total,'available':available}) + print(message) + if alert:subprocess.run(['logger','-p','daemon.err','-t','canina-disk-watch',message],check=True) + return int(alert) +if __name__=='__main__':sys.exit(main()) diff --git a/scripts/install-disk-watch.py b/scripts/install-disk-watch.py new file mode 100644 index 00000000..aaea19dd --- /dev/null +++ b/scripts/install-disk-watch.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +"""Install deterministic host-only pressure watch through authenticated CI.""" +import os,pathlib,subprocess +if os.environ.get('GITHUB_ACTIONS',os.environ.get('GITEA_ACTIONS'))!='true' or os.getuid()!=0 or pathlib.Path('/.dockerenv').exists():raise SystemExit('CI host root required') +root=pathlib.Path('/opt/services/canina/.disk-watch');root.mkdir(mode=0o700,exist_ok=True) +if root.is_symlink() or root.resolve()!=root:raise SystemExit('Unsafe installer path') +(root/'disk-watch.py').write_bytes(pathlib.Path(__file__).with_name('disk-watch.py').read_bytes()) +pathlib.Path('/etc/systemd/system/canina-disk-watch.service').write_text('[Unit]\nDescription=Canina deterministic disk pressure check\n[Service]\nType=oneshot\nExecStart=/usr/bin/python3 /opt/services/canina/.disk-watch/disk-watch.py\nNoNewPrivileges=true\nProtectSystem=strict\nProtectHome=true\nPrivateTmp=true\n') +pathlib.Path('/etc/systemd/system/canina-disk-watch.timer').write_text('[Unit]\nDescription=Check disk every five minutes without inference\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\n[Install]\nWantedBy=timers.target\n') +subprocess.run(['systemctl','daemon-reload'],check=True) +subprocess.run(['systemctl','enable','--now','canina-disk-watch.timer'],check=True) +subprocess.run(['systemctl','is-active','--quiet','canina-disk-watch.timer'],check=True) diff --git a/scripts/release-retention.py b/scripts/release-retention.py new file mode 100644 index 00000000..34d3c0db --- /dev/null +++ b/scripts/release-retention.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Conservative CI retention: current mount users win; dirty/unpublished trees stay. +Default is dry-run. No volumes, images, secrets or arbitrary workspaces are swept. +""" +import argparse,fcntl,json,os,pathlib,re,shutil,subprocess,time +BASE=pathlib.Path('/opt/services/canina') +def command(args): + return subprocess.check_output(args,text=True,stderr=subprocess.PIPE).strip() +def git(path,*args):return command(['git','-C',str(path),*args]) +def eligible(path,root,protected,*,clean,pushed): + path=pathlib.Path(path);root=pathlib.Path(root) + if path.is_symlink() or root.is_symlink() or path.parent!=root or path.resolve()!=path or root.resolve()!=root:return False + if not re.fullmatch('[0-9a-f]{40}',path.name):return False + return clean and pushed and not any(str(path)==p or str(path).startswith(p.rstrip('/')+'/') or p.startswith(str(path)+'/') for p in protected) +def dependencies(): + ids=command(['docker','ps','-aq']).split() + containers=json.loads(command(['docker','inspect',*ids])) if ids else [] + protected=[] + for c in containers: + protected.extend(m['Source'] for m in c['Mounts'] if m.get('Source','').startswith('/')) + labels=c['Config'].get('Labels') or {} + for key in ['com.docker.compose.project.working_dir','com.docker.compose.project.config_files']: + protected.extend(p for p in labels.get(key,'').split(',') if p.startswith('/')) + for pid in pathlib.Path('/proc').iterdir(): + if not pid.name.isdigit():continue + for p in [pid/'cwd',*(pid/'fd').glob('*')]: + try: + target=os.readlink(p) + if target.startswith(str(BASE)+'/'):protected.append(target) + except (OSError,PermissionError):pass + protected.extend(line.split()[4].replace('\\040',' ') for line in pathlib.Path('/proc/self/mountinfo').read_text().splitlines() if line.split()[4].startswith(str(BASE)+'/')) + return sorted(set(protected)),[{'name':c['Name'],'mounts':c['Mounts'],'labels':{k:v for k,v in (c['Config'].get('Labels') or {}).items() if k.startswith('com.docker.compose.')}} for c in containers] +def inspect(path): + return {'head':git(path,'rev-parse','HEAD'),'branch':subprocess.run(['git','-C',str(path),'symbolic-ref','--short','-q','HEAD'],text=True,capture_output=True).stdout.strip(),'upstream':subprocess.run(['git','-C',str(path),'rev-parse','--abbrev-ref','@{upstream}'],text=True,capture_output=True).stdout.strip(),'status':git(path,'status','--porcelain=v1','--untracked-files=all'),'unpushed':git(path,'rev-list','HEAD','--not','--remotes'),'common':git(path,'rev-parse','--path-format=absolute','--git-common-dir')} +def main(): + ap=argparse.ArgumentParser();ap.add_argument('--apply',action='store_true');ap.add_argument('--current-sha',required=True);ap.add_argument('--environment',choices=['stage','production'],required=True);args=ap.parse_args() + if not re.fullmatch('[0-9a-f]{40}',args.current_sha):raise SystemExit('Invalid current SHA') + root=BASE/('releases' if args.environment=='stage' else 'production-releases') + if root.is_symlink() or root.resolve()!=root:raise SystemExit('Unsafe release root') + lock=os.open(str(BASE/'release-retention.lock'),os.O_CREAT|os.O_RDWR|os.O_NOFOLLOW,0o600) + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + protected,containers=dependencies() + current=root/args.current_sha + if git(current,'rev-parse','HEAD')!=args.current_sha:raise SystemExit('Current release identity mismatch') + # Fail closed if Docker does not identify the accepted SHA as a current user. + if not any(p==str(current) or p.startswith(str(current)+'/') for p in protected):raise SystemExit('Current SHA is not active') + protected.append(str(current)) + report={'time':time.time(),'apply':args.apply,'current':str(current),'containers':containers,'protected_paths':protected,'entries':[]} + for path in sorted(root.iterdir()): + e={'path':str(path),'removed':False};report['entries'].append(e) + try: + if not eligible(path,root,protected,clean=True,pushed=True):e['reason']='active-or-unsafe';continue + e.update(inspect(path)) + if not eligible(path,root,protected,clean=e['status']=='',pushed=e['unpushed']==''):e['reason']='dirty-or-unpushed';continue + # Never remove ignored runtime secrets/config even from an otherwise clean tree. + ignored=git(path,'ls-files','--others','--ignored','--exclude-standard') + runtime_names={'.support-chat.env','.revalidation.env','.stage-compose.json','.stage-runtime.json','.stage-runtime-merged.json','.prod-compose.json','.prod-rollback.json','.prod-snapshot.json'} + ignored_paths=ignored.splitlines() + if any(name not in runtime_names or (path/name).is_symlink() or not (path/name).is_file() for name in ignored_paths): + e['reason']='unknown-ignored-files-preserved';continue + e['preserved_runtime_files']=ignored_paths + e['reason']='eligible' + if args.apply: + latest,_=dependencies() + if not eligible(path,root,latest,clean=git(path,'status','--porcelain=v1','--untracked-files=all')=='',pushed=git(path,'rev-list','HEAD','--not','--remotes')==''):raise RuntimeError('Dependency/status changed') + if ignored_paths: + archive=BASE/'.retained-runtime'/args.environment/path.name + archive.mkdir(parents=True,mode=0o700,exist_ok=True) + if archive.resolve()!=archive:raise RuntimeError('Unsafe runtime archive') + os.chmod(archive,0o700) + for name in ignored_paths: + destination=archive/name + if destination.exists() or destination.is_symlink():raise RuntimeError('Runtime archive collision') + shutil.copyfile(path/name,destination);os.chmod(destination,0o600) + common=pathlib.Path(e['common']) + if common!=path/'.git': + command(['git','--git-dir',str(common),'worktree','remove',str(path)]) + else: + if len(git(path,'worktree','list','--porcelain').split('worktree '))-1!=1:raise RuntimeError('Has child worktrees') + if not shutil.rmtree.avoids_symlink_attacks:raise RuntimeError('No safe rmtree') + shutil.rmtree(path) + e['removed']=True + except (subprocess.CalledProcessError,RuntimeError,OSError) as ex:e['reason']='inspection-failed:'+type(ex).__name__ + print(json.dumps(report,indent=2)) +if __name__=='__main__':main() diff --git a/scripts/rollout-production.sh b/scripts/rollout-production.sh index a10e60d0..02b44bfb 100644 --- a/scripts/rollout-production.sh +++ b/scripts/rollout-production.sh @@ -21,6 +21,7 @@ mkdir "$TARGET_DIR" git clone --no-checkout --single-branch --branch main https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR" cd "$TARGET_DIR" [[ $(git rev-parse origin/main) == "$DEPLOY_SHA" ]] || fail 'main must equal accepted Stage SHA (fast-forward promotion only)' +git sparse-checkout set --no-cone '/*' '!/graphify-out/' git checkout --detach "$DEPLOY_SHA" python3 scripts/production-release.py receipt # No schema/seed operation is allowed by this code-only release path. @@ -87,4 +88,8 @@ docker exec canino_backend_prod node -e ' const r=await fetch("http://frontend_prod:3000/api/revalidate",{method:"POST",headers:{"Content-Type":"application/json","x-revalidate-secret":key},body:JSON.stringify({paths:["/blog"]}),signal:AbortSignal.timeout(10000)});if(!r.ok)throw Error();})().catch(()=>{console.error("Production authenticated revalidation failed");process.exitCode=1});' started=false trap - EXIT +git restore -- .dockerignore nginx.conf +python3 scripts/test-release-retention.py +python3 scripts/install-disk-watch.py +python3 scripts/release-retention.py --apply --environment production --current-sha "$DEPLOY_SHA" > /opt/services/canina/tmp/retention-production.json printf 'Production immutable code-only rollout complete: %s; protected/live feature acceptance remains required\n' "$DEPLOY_SHA" diff --git a/scripts/test-release-retention.py b/scripts/test-release-retention.py new file mode 100644 index 00000000..1d0952d8 --- /dev/null +++ b/scripts/test-release-retention.py @@ -0,0 +1,32 @@ +import importlib.util,pathlib,tempfile,unittest,subprocess +ROOT=pathlib.Path(__file__).resolve().parent +spec=importlib.util.spec_from_file_location('retention',ROOT/'release-retention.py') +r=importlib.util.module_from_spec(spec);spec.loader.exec_module(r) +class RetentionTests(unittest.TestCase): + def test_active_dirty_unpushed_and_symlink_protected(self): + with tempfile.TemporaryDirectory() as d: + root=pathlib.Path(d); old=root/('a'*40);old.mkdir();active=root/('b'*40);active.mkdir() + self.assertFalse(r.eligible(old,root,[str(old/'backend')],clean=True,pushed=True)) + self.assertFalse(r.eligible(old,root,[],clean=False,pushed=True)) + self.assertFalse(r.eligible(old,root,[],clean=True,pushed=False)) + self.assertTrue(r.eligible(old,root,[str(active)],clean=True,pushed=True)) + link=root/('c'*40);link.symlink_to(old,target_is_directory=True) + self.assertFalse(r.eligible(link,root,[],clean=True,pushed=True)) + self.assertFalse(r.eligible(root,root,[],clean=True,pushed=True)) + def test_inspect_real_git_dirty_and_unpushed(self): + with tempfile.TemporaryDirectory() as d: + p=pathlib.Path(d) + def g(*a):return subprocess.check_output(['git','-C',d,*a],stderr=subprocess.DEVNULL,text=True) + g('init');g('config','user.email','test@example.invalid');g('config','user.name','Test') + (p/'source').write_text('preserve');g('add','.');g('commit','-m','fixture') + self.assertTrue(r.inspect(p)['unpushed']) + g('update-ref','refs/remotes/origin/develop','HEAD') + self.assertEqual(r.inspect(p)['unpushed'],'') + (p/'untracked').write_text('preserve') + self.assertIn('untracked',r.inspect(p)['status']) + def test_disk_pressure_thresholds(self): + spec=importlib.util.spec_from_file_location('watch',ROOT/'disk-watch.py');w=importlib.util.module_from_spec(spec);spec.loader.exec_module(w) + self.assertTrue(w.pressure(100*1024**3,9*1024**3)) + self.assertTrue(w.pressure(100*1024**3,15*1024**3)) + self.assertFalse(w.pressure(100*1024**3,30*1024**3)) +if __name__=='__main__':unittest.main()