ops: install narrowly scoped bounded disk maintenance via CI
Owned disk maintenance only / maintenance (push) Successful in 11s
Owned disk maintenance only / maintenance (push) Successful in 11s
This commit is contained in:
1 parent
dc34c57a0c
commit
6dec2c1469
4 files changed
+115
No files matched your search
@@ -0,0 +1,25 @@
|
||||
name: Owned disk maintenance only
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- ops/bounded-disk-20261119
|
||||
jobs:
|
||||
maintenance:
|
||||
runs-on: canina
|
||||
steps:
|
||||
- name: Test and install bounded maintenance without app deployment
|
||||
env:
|
||||
SHA: ${{ github.sha }}
|
||||
run: |
|
||||
set -eu
|
||||
umask 077
|
||||
case "$SHA" in *[!0-9a-f]*|'') exit 2;; esac
|
||||
[ "${#SHA}" -eq 40 ]
|
||||
mkdir -p disk-maintenance
|
||||
cd disk-maintenance
|
||||
export no_proxy=git.parsaaghayi.ir,localhost,127.0.0.1
|
||||
for name in managed-disk.py test-managed-disk.py install-managed-disk.py; do
|
||||
wget --no-proxy -qO "$name" "https://git.parsaaghayi.ir/parsa/canina/raw/commit/$SHA/scripts/$name"
|
||||
done
|
||||
python3 -B test-managed-disk.py
|
||||
tar -cf - managed-disk.py test-managed-disk.py install-managed-disk.py | ssh -F /dev/null -i /root/.ssh/id_ed25519 -p 2234 -o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=15 devops@172.17.0.1 'sudo -n bash -c '\''set -eu; umask 077; d=$(mktemp -d /opt/services/canina/tmp/managed-disk-ci.XXXXXXXX); trap "rm -rf -- $d" EXIT; tar -xf - -C "$d"; GITHUB_ACTIONS=true python3 -B "$d/install-managed-disk.py"; python3 -B /opt/services/canina/.managed-disk/managed-disk.py; python3 -B /opt/services/canina/.managed-disk/managed-disk.py'\'''
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Host maintenance only; never deploy/recreate an application."""
|
||||
import os,pathlib,subprocess
|
||||
if os.environ.get('GITHUB_ACTIONS')!='true' or os.getuid()!=0 or pathlib.Path('/.dockerenv').exists():raise SystemExit('Authenticated host CI required')
|
||||
r=pathlib.Path('/opt/services/canina/.managed-disk')
|
||||
if r.is_symlink() or r.resolve()!=r:raise SystemExit('unsafe destination')
|
||||
r.mkdir(mode=0o700,exist_ok=True);(r/'tests').mkdir(mode=0o700,exist_ok=True)
|
||||
for n in ['managed-disk.py','test-managed-disk.py']:
|
||||
p=r/n
|
||||
if p.is_symlink():raise SystemExit('unsafe destination file')
|
||||
p.write_bytes(pathlib.Path(__file__).with_name(n).read_bytes());p.chmod(0o600)
|
||||
(r/'heavy.lock').touch(mode=0o600,exist_ok=True)
|
||||
subprocess.run(['python3','-B',str(r/'test-managed-disk.py')],check=True)
|
||||
u=pathlib.Path('/etc/systemd/system')
|
||||
(u/'canina-managed-disk.service').write_text('[Unit]\nDescription=Bounded owned disposable disk retention\n[Service]\nType=oneshot\nExecStart=/usr/bin/python3 -B /opt/services/canina/.managed-disk/managed-disk.py\nNice=15\nIOSchedulingClass=idle\nNoNewPrivileges=true\n')
|
||||
(u/'canina-managed-disk.timer').write_text('[Unit]\nDescription=Daily conservative disposable retention\n[Timer]\nOnCalendar=*-*-* 07:30:00 UTC\nRandomizedDelaySec=15min\nPersistent=true\n[Install]\nWantedBy=timers.target\n')
|
||||
subprocess.run(['systemctl','daemon-reload'],check=True)
|
||||
subprocess.run(['systemctl','enable','--now','canina-managed-disk.timer'],check=True)
|
||||
subprocess.run(['systemctl','is-active','--quiet','canina-managed-disk.timer'],check=True)
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Owned opt-in test expiry, conservative unused cache, and shared build guard."""
|
||||
import pathlib,os,time,shutil,fcntl,subprocess,json,sys
|
||||
BASE=pathlib.Path('/opt/services/canina/.managed-disk')
|
||||
RESERVE=20*1024**3
|
||||
|
||||
def retire(root,now=None):
|
||||
root=pathlib.Path(root);now=time.time() if now is None else now
|
||||
if root.is_symlink() or root.resolve()!=root:raise RuntimeError('unsafe root')
|
||||
removed=[]
|
||||
for p in sorted(root.iterdir()):
|
||||
marker=p/'.disposable-test'
|
||||
if p.is_symlink() or not p.is_dir() or p.resolve()!=p:continue
|
||||
if not marker.is_file() or marker.is_symlink() or marker.read_text()!='canina-managed-test-v1\n':continue
|
||||
if now-p.stat().st_mtime<7*86400:continue
|
||||
# Unknown/source-bearing contents are never eligible; symlinks are never followed.
|
||||
if any(x.name in {'.git','.env','node_modules','vendor'} or x.is_symlink() or os.path.ismount(x) for x in p.rglob('*')):continue
|
||||
users=[]
|
||||
for proc in pathlib.Path('/proc').glob('[0-9]*'):
|
||||
for q in [proc/'cwd',proc/'root',*(proc/'fd').glob('*')]:
|
||||
try:
|
||||
v=os.readlink(q)
|
||||
if v==str(p) or v.startswith(str(p)+'/'):users.append(v)
|
||||
except OSError:pass
|
||||
if users:continue
|
||||
shutil.rmtree(p);removed.append(p.name)
|
||||
return removed
|
||||
|
||||
def busy():
|
||||
for p in pathlib.Path('/proc').glob('[0-9]*'):
|
||||
try:
|
||||
args=(p/'cmdline').read_bytes().replace(b'\0',b' ').decode(errors='replace')
|
||||
if any(x in args for x in ['docker build','compose build','next build','npm ci','npm install','composer install','rclone rcat']):return True
|
||||
except OSError:continue
|
||||
return False
|
||||
|
||||
def main():
|
||||
if BASE.is_symlink() or BASE.resolve()!=BASE:raise SystemExit('unsafe state root')
|
||||
with (BASE/'heavy.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if sys.argv[1:2]==['build']:
|
||||
free=shutil.disk_usage(BASE).free
|
||||
if free<RESERVE:raise SystemExit('Insufficient available capacity: 20GiB required')
|
||||
if not sys.argv[2:]:raise SystemExit('command required')
|
||||
raise SystemExit(subprocess.call(sys.argv[2:]))
|
||||
if busy():raise SystemExit('Active heavy work: retention deferred')
|
||||
before=shutil.disk_usage(BASE).free
|
||||
removed=retire(BASE/'tests')
|
||||
p=subprocess.run(['docker','builder','prune','--force','--filter','until=24h'],capture_output=True,text=True,timeout=300,check=True)
|
||||
print(json.dumps({'removed_tests':removed,'docker_cache_result':p.stdout.strip(),'available_before':before,'available_after':shutil.disk_usage(BASE).free}))
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,20 @@
|
||||
import importlib.util,pathlib,tempfile,unittest,os,time
|
||||
spec=importlib.util.spec_from_file_location('disk',pathlib.Path(__file__).with_name('managed-disk.py'))
|
||||
class Tests(unittest.TestCase):
|
||||
def test_expiry_and_escape(self):
|
||||
self.assertTrue(pathlib.Path(spec.origin).exists(),'managed retention implementation missing')
|
||||
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r=pathlib.Path(d);old=r/'old';old.mkdir();(old/'.disposable-test').write_text('canina-managed-test-v1\n');(old/'result').write_text('fixture');os.utime(old,(1,1))
|
||||
unknown=r/'unknown';unknown.mkdir();os.utime(unknown,(1,1))
|
||||
outside=r/'source';outside.mkdir();(outside/'important').write_text('preserve');os.symlink(outside,r/'escape')
|
||||
self.assertEqual(m.retire(r,now=time.time()),['old']);self.assertEqual(m.retire(r,now=time.time()),[])
|
||||
self.assertTrue((outside/'important').exists());self.assertTrue(unknown.exists());self.assertTrue((r/'escape').is_symlink())
|
||||
def test_open_artifact_is_preserved(self):
|
||||
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
r=pathlib.Path(d);p=r/'busy';p.mkdir();(p/'.disposable-test').write_text('canina-managed-test-v1\n');f=(p/'result').open('w');os.utime(p,(1,1))
|
||||
try:self.assertEqual(m.retire(r),[])
|
||||
finally:f.close()
|
||||
self.assertEqual(m.retire(r),['busy'])
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user