fix(ops): conservative post-rollout retention and deterministic disk watch
Deploy Canina / deploy (push) Has been cancelled
Deploy Canina / deploy (push) Has been cancelled
This commit is contained in:
1 parent
b4420e54b7
commit
366bc9fc88
8 files changed
+167
No files matched your search
@@ -16,3 +16,7 @@ playwright-report/
|
||||
playwright/.auth/
|
||||
blob-report/
|
||||
|
||||
|
||||
# Python CI helpers must not dirty immutable release checkouts.
|
||||
__pycache__/
|
||||
*.pyc
|
||||
@@ -0,0 +1,9 @@
|
||||
# Host disk retention
|
||||
|
||||
Successful authenticated CI rollout runs release-retention.py. Default keep policy is only the current active release per environment; Docker mounts and Compose working directories always protect older still-used artifacts. Dry/unpublished source and unknown ignored files are preserved, so this is not a hard quota. CI restores only its own tracked nginx/dockerignore render changes after successful acceptance. New clones use sparse checkout to omit the 85 MiB tracked graphify-out history. No node_modules were bundled in the audited releases: their ~218 MiB is tracked source/assets plus an independent ~44 MiB Git store. Historical dirty releases require individual source review; never reset them for cleanup.
|
||||
|
||||
Retained small private runtime configuration is copied mode0600 under .retained-runtime before deleting a clean obsolete release. This avoids secret deletion; this archive is not a release rollback and requires explicit secret retention review separately. No volumes, databases, sessions, evidence, images or build cache are deleted.
|
||||
|
||||
The CI-installed canina-disk-watch.timer checks every five minutes with stdlib Python, no model calls. It logs daemon.err and returns nonzero at >=85% usage or <10GiB free. This is local detection, NOT off-host paging. systemctl status canina-disk-watch.service and journalctl -t canina-disk-watch expose alarms. Monitoring does not guarantee the disk cannot fill.
|
||||
|
||||
Audit 2026-10-26: Docker daemon already bounds new json-file containers to 3x10MiB and BuildKit GC to 6GB. Fifteen pre-existing containers (including Nexus, Gitea, databases and proxy) have empty LogConfig and remain unbounded until their owners roll them through their own Git/CI. Journald has no explicit SystemMaxUse and currently occupies 1.3GB. Nexus data is real persistent storage; retention tasks were not authenticated/audited and no blobs were deleted. Neither unrelated service config nor daemon/journal configuration is changed by Canina CI. Worktree branches with unpushed commits remain intentionally retained. These residual growth mechanisms prevent any honest claim of a globally bounded host or <50GB total.
|
||||
@@ -35,6 +35,7 @@ git clone --no-checkout --single-branch --branch develop https://git.parsaaghayi
|
||||
cd "$TARGET_DIR"
|
||||
git cat-file -e "${DEPLOY_SHA}^{commit}"
|
||||
git merge-base --is-ancestor "$DEPLOY_SHA" origin/develop
|
||||
git sparse-checkout set --no-cone '/*' '!/graphify-out/'
|
||||
git checkout --detach "$DEPLOY_SHA"
|
||||
[ "$(git rev-parse HEAD)" = "$DEPLOY_SHA" ]
|
||||
# Provision against the pristine approved SHA, before any tracked render edits.
|
||||
@@ -125,4 +126,9 @@ source scripts/rollout-stage.sh
|
||||
# This preserves the root-private intent journal and fails closed on timer failure.
|
||||
python3 scripts/install-stage-principal-cleanup.py
|
||||
stage_rollout
|
||||
python3 scripts/test-release-retention.py
|
||||
# Restore only CI-owned render edits after images and acceptance succeed.
|
||||
git restore -- .dockerignore nginx.conf
|
||||
python3 scripts/install-disk-watch.py
|
||||
python3 scripts/release-retention.py --apply --environment stage --current-sha "$DEPLOY_SHA" > /opt/services/canina/tmp/retention-stage.json
|
||||
echo 'Deployment for develop completed successfully!'
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""No-model threshold check. Exit 1 at >=85% used or <10GiB available."""
|
||||
import json,os,subprocess,sys
|
||||
|
||||
def pressure(total,available):return available < 10*1024**3 or (total-available)/total >= .85
|
||||
|
||||
def main():
|
||||
s=os.statvfs('/');total=s.f_blocks*s.f_frsize;available=s.f_bavail*s.f_frsize
|
||||
alert=pressure(total,available)
|
||||
message=json.dumps({'service':'canina-disk-watch','alert':alert,'total':total,'available':available})
|
||||
print(message)
|
||||
if alert:subprocess.run(['logger','-p','daemon.err','-t','canina-disk-watch',message],check=True)
|
||||
return int(alert)
|
||||
if __name__=='__main__':sys.exit(main())
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Install deterministic host-only pressure watch through authenticated CI."""
|
||||
import os,pathlib,subprocess
|
||||
if os.environ.get('GITHUB_ACTIONS',os.environ.get('GITEA_ACTIONS'))!='true' or os.getuid()!=0 or pathlib.Path('/.dockerenv').exists():raise SystemExit('CI host root required')
|
||||
root=pathlib.Path('/opt/services/canina/.disk-watch');root.mkdir(mode=0o700,exist_ok=True)
|
||||
if root.is_symlink() or root.resolve()!=root:raise SystemExit('Unsafe installer path')
|
||||
(root/'disk-watch.py').write_bytes(pathlib.Path(__file__).with_name('disk-watch.py').read_bytes())
|
||||
pathlib.Path('/etc/systemd/system/canina-disk-watch.service').write_text('[Unit]\nDescription=Canina deterministic disk pressure check\n[Service]\nType=oneshot\nExecStart=/usr/bin/python3 /opt/services/canina/.disk-watch/disk-watch.py\nNoNewPrivileges=true\nProtectSystem=strict\nProtectHome=true\nPrivateTmp=true\n')
|
||||
pathlib.Path('/etc/systemd/system/canina-disk-watch.timer').write_text('[Unit]\nDescription=Check disk every five minutes without inference\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\n[Install]\nWantedBy=timers.target\n')
|
||||
subprocess.run(['systemctl','daemon-reload'],check=True)
|
||||
subprocess.run(['systemctl','enable','--now','canina-disk-watch.timer'],check=True)
|
||||
subprocess.run(['systemctl','is-active','--quiet','canina-disk-watch.timer'],check=True)
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Conservative CI retention: current mount users win; dirty/unpublished trees stay.
|
||||
Default is dry-run. No volumes, images, secrets or arbitrary workspaces are swept.
|
||||
"""
|
||||
import argparse,fcntl,json,os,pathlib,re,shutil,subprocess,time
|
||||
BASE=pathlib.Path('/opt/services/canina')
|
||||
def command(args):
|
||||
return subprocess.check_output(args,text=True,stderr=subprocess.PIPE).strip()
|
||||
def git(path,*args):return command(['git','-C',str(path),*args])
|
||||
def eligible(path,root,protected,*,clean,pushed):
|
||||
path=pathlib.Path(path);root=pathlib.Path(root)
|
||||
if path.is_symlink() or root.is_symlink() or path.parent!=root or path.resolve()!=path or root.resolve()!=root:return False
|
||||
if not re.fullmatch('[0-9a-f]{40}',path.name):return False
|
||||
return clean and pushed and not any(str(path)==p or str(path).startswith(p.rstrip('/')+'/') or p.startswith(str(path)+'/') for p in protected)
|
||||
def dependencies():
|
||||
ids=command(['docker','ps','-aq']).split()
|
||||
containers=json.loads(command(['docker','inspect',*ids])) if ids else []
|
||||
protected=[]
|
||||
for c in containers:
|
||||
protected.extend(m['Source'] for m in c['Mounts'] if m.get('Source','').startswith('/'))
|
||||
labels=c['Config'].get('Labels') or {}
|
||||
for key in ['com.docker.compose.project.working_dir','com.docker.compose.project.config_files']:
|
||||
protected.extend(p for p in labels.get(key,'').split(',') if p.startswith('/'))
|
||||
for pid in pathlib.Path('/proc').iterdir():
|
||||
if not pid.name.isdigit():continue
|
||||
for p in [pid/'cwd',*(pid/'fd').glob('*')]:
|
||||
try:
|
||||
target=os.readlink(p)
|
||||
if target.startswith(str(BASE)+'/'):protected.append(target)
|
||||
except (OSError,PermissionError):pass
|
||||
protected.extend(line.split()[4].replace('\\040',' ') for line in pathlib.Path('/proc/self/mountinfo').read_text().splitlines() if line.split()[4].startswith(str(BASE)+'/'))
|
||||
return sorted(set(protected)),[{'name':c['Name'],'mounts':c['Mounts'],'labels':{k:v for k,v in (c['Config'].get('Labels') or {}).items() if k.startswith('com.docker.compose.')}} for c in containers]
|
||||
def inspect(path):
|
||||
return {'head':git(path,'rev-parse','HEAD'),'branch':subprocess.run(['git','-C',str(path),'symbolic-ref','--short','-q','HEAD'],text=True,capture_output=True).stdout.strip(),'upstream':subprocess.run(['git','-C',str(path),'rev-parse','--abbrev-ref','@{upstream}'],text=True,capture_output=True).stdout.strip(),'status':git(path,'status','--porcelain=v1','--untracked-files=all'),'unpushed':git(path,'rev-list','HEAD','--not','--remotes'),'common':git(path,'rev-parse','--path-format=absolute','--git-common-dir')}
|
||||
def main():
|
||||
ap=argparse.ArgumentParser();ap.add_argument('--apply',action='store_true');ap.add_argument('--current-sha',required=True);ap.add_argument('--environment',choices=['stage','production'],required=True);args=ap.parse_args()
|
||||
if not re.fullmatch('[0-9a-f]{40}',args.current_sha):raise SystemExit('Invalid current SHA')
|
||||
root=BASE/('releases' if args.environment=='stage' else 'production-releases')
|
||||
if root.is_symlink() or root.resolve()!=root:raise SystemExit('Unsafe release root')
|
||||
lock=os.open(str(BASE/'release-retention.lock'),os.O_CREAT|os.O_RDWR|os.O_NOFOLLOW,0o600)
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
protected,containers=dependencies()
|
||||
current=root/args.current_sha
|
||||
if git(current,'rev-parse','HEAD')!=args.current_sha:raise SystemExit('Current release identity mismatch')
|
||||
# Fail closed if Docker does not identify the accepted SHA as a current user.
|
||||
if not any(p==str(current) or p.startswith(str(current)+'/') for p in protected):raise SystemExit('Current SHA is not active')
|
||||
protected.append(str(current))
|
||||
report={'time':time.time(),'apply':args.apply,'current':str(current),'containers':containers,'protected_paths':protected,'entries':[]}
|
||||
for path in sorted(root.iterdir()):
|
||||
e={'path':str(path),'removed':False};report['entries'].append(e)
|
||||
try:
|
||||
if not eligible(path,root,protected,clean=True,pushed=True):e['reason']='active-or-unsafe';continue
|
||||
e.update(inspect(path))
|
||||
if not eligible(path,root,protected,clean=e['status']=='',pushed=e['unpushed']==''):e['reason']='dirty-or-unpushed';continue
|
||||
# Never remove ignored runtime secrets/config even from an otherwise clean tree.
|
||||
ignored=git(path,'ls-files','--others','--ignored','--exclude-standard')
|
||||
runtime_names={'.support-chat.env','.revalidation.env','.stage-compose.json','.stage-runtime.json','.stage-runtime-merged.json','.prod-compose.json','.prod-rollback.json','.prod-snapshot.json'}
|
||||
ignored_paths=ignored.splitlines()
|
||||
if any(name not in runtime_names or (path/name).is_symlink() or not (path/name).is_file() for name in ignored_paths):
|
||||
e['reason']='unknown-ignored-files-preserved';continue
|
||||
e['preserved_runtime_files']=ignored_paths
|
||||
e['reason']='eligible'
|
||||
if args.apply:
|
||||
latest,_=dependencies()
|
||||
if not eligible(path,root,latest,clean=git(path,'status','--porcelain=v1','--untracked-files=all')=='',pushed=git(path,'rev-list','HEAD','--not','--remotes')==''):raise RuntimeError('Dependency/status changed')
|
||||
if ignored_paths:
|
||||
archive=BASE/'.retained-runtime'/args.environment/path.name
|
||||
archive.mkdir(parents=True,mode=0o700,exist_ok=True)
|
||||
if archive.resolve()!=archive:raise RuntimeError('Unsafe runtime archive')
|
||||
os.chmod(archive,0o700)
|
||||
for name in ignored_paths:
|
||||
destination=archive/name
|
||||
if destination.exists() or destination.is_symlink():raise RuntimeError('Runtime archive collision')
|
||||
shutil.copyfile(path/name,destination);os.chmod(destination,0o600)
|
||||
common=pathlib.Path(e['common'])
|
||||
if common!=path/'.git':
|
||||
command(['git','--git-dir',str(common),'worktree','remove',str(path)])
|
||||
else:
|
||||
if len(git(path,'worktree','list','--porcelain').split('worktree '))-1!=1:raise RuntimeError('Has child worktrees')
|
||||
if not shutil.rmtree.avoids_symlink_attacks:raise RuntimeError('No safe rmtree')
|
||||
shutil.rmtree(path)
|
||||
e['removed']=True
|
||||
except (subprocess.CalledProcessError,RuntimeError,OSError) as ex:e['reason']='inspection-failed:'+type(ex).__name__
|
||||
print(json.dumps(report,indent=2))
|
||||
if __name__=='__main__':main()
|
||||
@@ -21,6 +21,7 @@ mkdir "$TARGET_DIR"
|
||||
git clone --no-checkout --single-branch --branch main https://git.parsaaghayi.ir/parsa/canina.git "$TARGET_DIR"
|
||||
cd "$TARGET_DIR"
|
||||
[[ $(git rev-parse origin/main) == "$DEPLOY_SHA" ]] || fail 'main must equal accepted Stage SHA (fast-forward promotion only)'
|
||||
git sparse-checkout set --no-cone '/*' '!/graphify-out/'
|
||||
git checkout --detach "$DEPLOY_SHA"
|
||||
python3 scripts/production-release.py receipt
|
||||
# No schema/seed operation is allowed by this code-only release path.
|
||||
@@ -87,4 +88,8 @@ docker exec canino_backend_prod node -e '
|
||||
const r=await fetch("http://frontend_prod:3000/api/revalidate",{method:"POST",headers:{"Content-Type":"application/json","x-revalidate-secret":key},body:JSON.stringify({paths:["/blog"]}),signal:AbortSignal.timeout(10000)});if(!r.ok)throw Error();})().catch(()=>{console.error("Production authenticated revalidation failed");process.exitCode=1});'
|
||||
started=false
|
||||
trap - EXIT
|
||||
git restore -- .dockerignore nginx.conf
|
||||
python3 scripts/test-release-retention.py
|
||||
python3 scripts/install-disk-watch.py
|
||||
python3 scripts/release-retention.py --apply --environment production --current-sha "$DEPLOY_SHA" > /opt/services/canina/tmp/retention-production.json
|
||||
printf 'Production immutable code-only rollout complete: %s; protected/live feature acceptance remains required\n' "$DEPLOY_SHA"
|
||||
@@ -0,0 +1,32 @@
|
||||
import importlib.util,pathlib,tempfile,unittest,subprocess
|
||||
ROOT=pathlib.Path(__file__).resolve().parent
|
||||
spec=importlib.util.spec_from_file_location('retention',ROOT/'release-retention.py')
|
||||
r=importlib.util.module_from_spec(spec);spec.loader.exec_module(r)
|
||||
class RetentionTests(unittest.TestCase):
|
||||
def test_active_dirty_unpushed_and_symlink_protected(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root=pathlib.Path(d); old=root/('a'*40);old.mkdir();active=root/('b'*40);active.mkdir()
|
||||
self.assertFalse(r.eligible(old,root,[str(old/'backend')],clean=True,pushed=True))
|
||||
self.assertFalse(r.eligible(old,root,[],clean=False,pushed=True))
|
||||
self.assertFalse(r.eligible(old,root,[],clean=True,pushed=False))
|
||||
self.assertTrue(r.eligible(old,root,[str(active)],clean=True,pushed=True))
|
||||
link=root/('c'*40);link.symlink_to(old,target_is_directory=True)
|
||||
self.assertFalse(r.eligible(link,root,[],clean=True,pushed=True))
|
||||
self.assertFalse(r.eligible(root,root,[],clean=True,pushed=True))
|
||||
def test_inspect_real_git_dirty_and_unpushed(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p=pathlib.Path(d)
|
||||
def g(*a):return subprocess.check_output(['git','-C',d,*a],stderr=subprocess.DEVNULL,text=True)
|
||||
g('init');g('config','user.email','test@example.invalid');g('config','user.name','Test')
|
||||
(p/'source').write_text('preserve');g('add','.');g('commit','-m','fixture')
|
||||
self.assertTrue(r.inspect(p)['unpushed'])
|
||||
g('update-ref','refs/remotes/origin/develop','HEAD')
|
||||
self.assertEqual(r.inspect(p)['unpushed'],'')
|
||||
(p/'untracked').write_text('preserve')
|
||||
self.assertIn('untracked',r.inspect(p)['status'])
|
||||
def test_disk_pressure_thresholds(self):
|
||||
spec=importlib.util.spec_from_file_location('watch',ROOT/'disk-watch.py');w=importlib.util.module_from_spec(spec);spec.loader.exec_module(w)
|
||||
self.assertTrue(w.pressure(100*1024**3,9*1024**3))
|
||||
self.assertTrue(w.pressure(100*1024**3,15*1024**3))
|
||||
self.assertFalse(w.pressure(100*1024**3,30*1024**3))
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user