fix(auth): persist generation-fenced Stage principal cleanup retries
This commit is contained in:
1 parent
80a2e188cf
commit
70a5449d35
13 files changed
+323
-44
No files matched your search
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Retry exact owned temporary Stage principal cleanup
|
||||
After=docker.service
|
||||
Wants=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
UMask=0077
|
||||
ExecStart=/usr/bin/python3 /opt/services/canina/stage-principal-cleanup.py sweep
|
||||
TimeoutStartSec=120
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Recover temporary Stage principal cleanup after reboot and failures
|
||||
|
||||
[Timer]
|
||||
OnBootSec=30s
|
||||
OnUnitInactiveSec=30s
|
||||
Unit=canina-stage-principal-cleanup.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Root-only Stage principal intent journal. No generic command/container inputs."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path('/opt/services/canina/stage-principal-cleanup')
|
||||
APPROVAL = 'owner-authorized-stage-editorial'
|
||||
|
||||
def validate(i):
|
||||
if set(i) != {'sha','session','ttl','generation','due'} or not re.fullmatch('[a-f0-9]{40}', i['sha']) or not re.fullmatch('[a-z0-9-]{8,64}', i['session']) or not re.fullmatch('[a-f0-9-]{36}', i['generation']) or not isinstance(i['ttl'],int) or not 60 <= i['ttl'] <= 3600 or not isinstance(i['due'],int):
|
||||
raise ValueError('Invalid cleanup intent')
|
||||
|
||||
class Worker:
|
||||
def __init__(self, root, run):
|
||||
self.root, self.run = root, run
|
||||
def save(self, intent):
|
||||
validate(intent)
|
||||
p = self.root / (intent['session']+'.json')
|
||||
# Exclusive creation forbids reusing a session or extending its deadline.
|
||||
fd = os.open(p, os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW, 0o600)
|
||||
with os.fdopen(fd,'w') as f:
|
||||
json.dump(intent,f); f.flush(); os.fsync(f.fileno())
|
||||
self.sync()
|
||||
def sync(self):
|
||||
fd=os.open(self.root,os.O_RDONLY|os.O_DIRECTORY)
|
||||
try: os.fsync(fd)
|
||||
finally: os.close(fd)
|
||||
def sweep(self, now):
|
||||
with open(self.root/'lock','a') as lock:
|
||||
try: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
except BlockingIOError: return
|
||||
for p in sorted(self.root.glob('*.json')):
|
||||
try:
|
||||
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
with os.fdopen(fd) as f: i=json.load(f)
|
||||
validate(i)
|
||||
if p.name != i['session']+'.json': raise ValueError('Intent name mismatch')
|
||||
if i['due'] <= now and self.run(i,'close'):
|
||||
p.unlink(); self.sync()
|
||||
except Exception:
|
||||
# Keep intent through DB/container/redeploy failures; next tick retries.
|
||||
print('Stage cleanup pending',file=sys.stderr)
|
||||
|
||||
def run(i, action):
|
||||
# Opens remain SHA pinned; cleanup only uses the CURRENT artifact's narrow CLI.
|
||||
# Original ownership is the session plus unpredictable account generation.
|
||||
container=subprocess.check_output(['docker','inspect','--format','{{.Id}}','canino_backend_stage'],text=True,timeout=30).strip()
|
||||
if not re.fullmatch('[a-f0-9]{64}',container): return False
|
||||
sha=subprocess.check_output(['docker','exec',container,'cat','/app/STAGE_TEST_ARTIFACT_SHA'],text=True,timeout=30).strip()
|
||||
if not re.fullmatch('[a-f0-9]{40}',sha) or (action=='open' and sha!=i['sha']):
|
||||
return False
|
||||
env={'APP_ENV':'stage','CI':'true','STAGE_TEST_PRINCIPAL_ENABLED':'true','STAGE_TEST_APPROVAL':APPROVAL,'STAGE_TEST_SHA':sha,'STAGE_TEST_SESSION':i['session'],'STAGE_TEST_TTL_SECONDS':str(i['ttl']),'STAGE_TEST_GENERATION':i['generation']}
|
||||
cmd=['docker','exec']
|
||||
for k,v in env.items(): cmd+=['-e',k+'='+v]
|
||||
cli=cmd+[container,'node','dist/auth/stage-test-principal.cli.js']
|
||||
# Never send a generation-fenced cleanup to an older CLI that ignores fences.
|
||||
capability=subprocess.check_output(cli+['cleanup-capability'],text=True,timeout=30)
|
||||
if capability!='stage-generation-cleanup-v1': return False
|
||||
return subprocess.run(cli+[action],timeout=90).returncode==0
|
||||
|
||||
def main():
|
||||
if os.getuid()!=0 or Path('/.dockerenv').exists(): raise ValueError('Host root only')
|
||||
os.umask(0o077)
|
||||
ROOT.mkdir(mode=0o700,exist_ok=True)
|
||||
st=ROOT.lstat()
|
||||
if ROOT.is_symlink() or st.st_uid!=0 or st.st_mode & 0o777 != 0o700: raise ValueError('Unsafe journal')
|
||||
worker=Worker(ROOT,run)
|
||||
if sys.argv[1:]==['sweep']:
|
||||
worker.sweep(int(time.time())); return
|
||||
if len(sys.argv)!=5 or os.environ.get('CI')!='true' or os.environ.get('STAGE_TEST_APPROVAL')!=APPROVAL: raise ValueError('CI grant required')
|
||||
sha,session,ttl,action=sys.argv[1:]
|
||||
i={'sha':sha,'session':session,'ttl':int(ttl),'generation':str(uuid.uuid4()),'due':int(time.time())+int(ttl)}
|
||||
validate(i)
|
||||
if action not in ('open','close'): raise ValueError('Action denied')
|
||||
with open(ROOT/'lock','a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX) # contention waits, never discards intent
|
||||
if action=='open':
|
||||
# Repeated opens cannot mint a new generation or renew TTL.
|
||||
worker.save(i)
|
||||
# Journal is durable before the first possible DB mutation.
|
||||
if not run(i,'open'):
|
||||
raise ValueError('Open failed; cleanup remains pending')
|
||||
else:
|
||||
p=ROOT/(session+'.json')
|
||||
fd=os.open(p,os.O_RDONLY|os.O_NOFOLLOW)
|
||||
with os.fdopen(fd) as f: i=json.load(f)
|
||||
validate(i)
|
||||
if run(i,'close'): p.unlink(); worker.sync()
|
||||
else: raise ValueError('Cleanup pending')
|
||||
if __name__=='__main__':
|
||||
try: main()
|
||||
except Exception:
|
||||
print('Stage principal denied or pending cleanup',file=sys.stderr); sys.exit(1)
|
||||
@@ -1,27 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# Authenticated CI host-side command only. No production argument or dynamic container name.
|
||||
# CI-only authority: fixed Stage cleanup worker, no dynamic executable/container.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
[[ $# == 4 && $1 =~ ^[0-9a-f]{40}$ && $2 =~ ^[a-z0-9-]{8,64}$ && $3 =~ ^[0-9]+$ && $4 =~ ^(open|close)$ ]] || exit 2
|
||||
[[ $3 -ge 60 && $3 -le 3600 && ${STAGE_TEST_APPROVAL:-} == owner-authorized-stage-editorial && ${CI:-} == true ]] || exit 2
|
||||
sha=$1 session=$2 ttl=$3 action=$4
|
||||
[[ ! -e /.dockerenv && $(id -u) == 0 ]] || exit 2
|
||||
exec 9>/opt/services/canina/stage-test-principal.lock
|
||||
flock -n 9 || exit 2
|
||||
[[ $(docker exec canino_backend_stage cat /app/STAGE_TEST_ARTIFACT_SHA) == "$sha" ]] || exit 2
|
||||
[[ $(docker inspect canino_backend_stage --format '{{.Name}}') == /canino_backend_stage ]] || exit 2
|
||||
run() {
|
||||
docker exec -e APP_ENV=stage -e CI=true -e STAGE_TEST_PRINCIPAL_ENABLED=true \
|
||||
-e STAGE_TEST_APPROVAL=owner-authorized-stage-editorial -e STAGE_TEST_SHA="$sha" \
|
||||
-e STAGE_TEST_SESSION="$session" -e STAGE_TEST_TTL_SECONDS="$ttl" \
|
||||
canino_backend_stage node dist/auth/stage-test-principal.cli.js "$1"
|
||||
}
|
||||
if [[ $action == open ]]; then
|
||||
# Arm cleanup BEFORE creating the account. systemd keeps this independent of CI loss.
|
||||
script="/opt/services/canina/releases/$sha/scripts/stage-test-principal-host.sh"
|
||||
[[ -f $script && ! -L $script ]] || exit 2
|
||||
systemd-run --quiet --unit="canina-stage-principal-$session" --on-active="${ttl}s" \
|
||||
--setenv=CI=true --setenv=STAGE_TEST_APPROVAL=owner-authorized-stage-editorial \
|
||||
/bin/bash "$script" "$sha" "$session" "$ttl" close
|
||||
fi
|
||||
run "$action"
|
||||
# Provisioned through reviewed deployment, not through the credential CI command.
|
||||
# Fail before any DB mutation unless reboot-persistent retry recovery is enabled.
|
||||
systemctl is-enabled --quiet canina-stage-principal-cleanup.timer
|
||||
systemctl is-active --quiet canina-stage-principal-cleanup.timer
|
||||
exec /usr/bin/python3 /opt/services/canina/stage-principal-cleanup.py "$@"
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run ONLY inside a disposable fixture container with Postgres/Redis test network.
|
||||
set -euo pipefail
|
||||
[[ ${STAGE_PRINCIPAL_DISPOSABLE_TEST:-} == true && ${DATABASE_URL:-} == postgresql://postgres@principal-durable-pg:5432/principal_disposable ]] || exit 2
|
||||
python3 -c 'from pathlib import Path; s=Path("prisma/schema.prisma").read_text(); Path("prisma/pre-stage.prisma").write_text("\n".join(x for x in s.splitlines() if not any(k in x for k in ["authExpiresAt","stageTestSession","stageTestGeneration"])))'
|
||||
node node_modules/prisma/build/index.js db push --schema prisma/pre-stage.prisma --force-reset --skip-generate
|
||||
# Executes exact additive SQL, not a final-schema db push substitute.
|
||||
node node_modules/prisma/build/index.js db execute --schema prisma/schema.prisma --file prisma/migrations/20261024090000_stage_test_principal/migration.sql
|
||||
node node_modules/prisma/build/index.js generate
|
||||
node node_modules/jest/bin/jest.js principal --runInBand --testTimeout=240000
|
||||
@@ -0,0 +1,45 @@
|
||||
import importlib.util, pathlib, tempfile, unittest, json
|
||||
spec=importlib.util.spec_from_file_location('cleanup',pathlib.Path(__file__).with_name('stage-principal-cleanup.py'))
|
||||
m=importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
class CleanupTests(unittest.TestCase):
|
||||
def test_redeploy_restart_and_failed_delete_retry(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
calls=[]
|
||||
def run(intent,action):
|
||||
calls.append((intent['generation'],action)); return len(calls)>1
|
||||
worker=m.Worker(pathlib.Path(root),run)
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
worker.save(intent)
|
||||
worker.sweep(100)
|
||||
self.assertTrue((pathlib.Path(root)/'test-session.json').exists())
|
||||
m.Worker(pathlib.Path(root),run).sweep(200)
|
||||
self.assertFalse((pathlib.Path(root)/'test-session.json').exists())
|
||||
self.assertEqual(calls,[(intent['generation'],'close')]*2)
|
||||
def test_contention_preserves_intent(self):
|
||||
import fcntl
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
worker=m.Worker(pathlib.Path(root),lambda *_:self.fail('must not execute while locked'))
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
worker.save(intent)
|
||||
with open(pathlib.Path(root)/'lock','w') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
worker.sweep(100)
|
||||
self.assertTrue((pathlib.Path(root)/'test-session.json').exists())
|
||||
def test_commands_follow_redeploy_but_keep_generation(self):
|
||||
from unittest.mock import patch
|
||||
from types import SimpleNamespace
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40,'stage-generation-cleanup-v1']), patch.object(m.subprocess,'run',return_value=SimpleNamespace(returncode=0)) as execute:
|
||||
self.assertTrue(m.run(intent,'close'))
|
||||
cmd=execute.call_args.args[0]
|
||||
self.assertIn('STAGE_TEST_SHA='+'b'*40,cmd)
|
||||
self.assertIn('STAGE_TEST_GENERATION='+intent['generation'],cmd)
|
||||
self.assertIn('c'*64,cmd)
|
||||
def test_old_cli_and_redeploy_open_fail_closed(self):
|
||||
from unittest.mock import patch
|
||||
intent={'sha':'a'*40,'session':'test-session','ttl':60,'generation':'8311eb41-23b4-4aab-ade2-e97b90debc11','due':0}
|
||||
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40]), patch.object(m.subprocess,'run') as execute:
|
||||
self.assertFalse(m.run(intent,'open')); execute.assert_not_called()
|
||||
with patch.object(m.subprocess,'check_output',side_effect=['c'*64,'b'*40,'old']), patch.object(m.subprocess,'run') as execute:
|
||||
self.assertFalse(m.run(intent,'close')); execute.assert_not_called()
|
||||
if __name__=='__main__': unittest.main()
|
||||
Reference in new issue
Block a user