fix(corporate): restore persisted orchestration APIs with code-only Stage CI
Deploy Canina / deploy (push) Successful in 4m39s
Deploy Canina / deploy (push) Successful in 4m39s
This commit is contained in:
1 parent
4d41a5bbf7
commit
85ef01d337
16 files changed
+2832
-1
No files matched your search
@@ -0,0 +1,219 @@
|
||||
"""Exercise the real release lock shell with a strictly local Docker double."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parents[1] / 'release-corporate-stage.sh'
|
||||
|
||||
|
||||
class StageDeployLockTests(unittest.TestCase):
|
||||
def exercise(self, mode='ok', body='docker compose build; docker compose up', accelerated=False):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
docker = root / 'docker'
|
||||
docker.write_text('''#!/usr/bin/env python3
|
||||
import json, os, sys, time, signal
|
||||
from pathlib import Path
|
||||
args = sys.argv[1:]
|
||||
root = Path(os.environ['FAKE_ROOT'])
|
||||
with (root / 'calls').open('a') as out: out.write(json.dumps(args) + '\\n')
|
||||
mode = os.environ['MODE']
|
||||
if (mode == 'stubborn-operation' and args[-1] == 'gitea_runner'
|
||||
or mode == 'stubborn-cleanup' and args[0] == 'rm'):
|
||||
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
time.sleep(6) # Self-bounded even before the production fix.
|
||||
(root / 'stubborn-completed').touch()
|
||||
if args[0] == 'inspect':
|
||||
denied = False
|
||||
if mode == 'remapped-host' and (root / 'create.json').exists():
|
||||
create = json.loads((root / 'create.json').read_text())
|
||||
# uid0 maps to host100000: directory1001:755/file0:644.
|
||||
# DAC allows read/traversal, not truncation/creation. Model transport
|
||||
# access only; the separate real-shell test proves actual host flock.
|
||||
denied = '9>/host-canina/' in create[-1]
|
||||
print('runner:local' if args[-1] == 'gitea_runner' else ('false' if denied or (root / 'lost').exists() else 'true'))
|
||||
elif args[0] == 'create':
|
||||
if mode == 'contended': sys.exit(1)
|
||||
(root / 'held').touch()
|
||||
(root / 'create.json').write_text(json.dumps(args))
|
||||
print('abc123owned')
|
||||
elif args[0] == 'start':
|
||||
if mode == 'start-failure': sys.exit(1)
|
||||
elif args[0] == 'logs':
|
||||
if mode == 'diagnostic-failure':
|
||||
print('sh: cannot open lock: Permission denied; SECRET_MUST_NOT_LEAK', file=sys.stderr)
|
||||
elif mode != 'host-contended': print('CANINA_STAGE_DEPLOY_LOCK_ACQUIRED')
|
||||
elif args[0] == 'rm':
|
||||
assert args[-1] == 'abc123owned', 'must only remove owned ID'
|
||||
(root / 'held').unlink()
|
||||
elif args[0] == 'compose':
|
||||
assert (root / 'held').exists(), 'deployment ran without lock'
|
||||
if mode == 'loss-during-build' and 'build' in args:
|
||||
(root / 'lost').touch()
|
||||
def stopped(*_):
|
||||
with (root / 'calls').open('a') as out: out.write(json.dumps(['child-stopped']) + '\\n')
|
||||
sys.exit(143)
|
||||
signal.signal(signal.SIGTERM, stopped)
|
||||
time.sleep(5)
|
||||
(root / 'child-completed').touch()
|
||||
else: sys.exit(99)
|
||||
''')
|
||||
docker.chmod(0o755)
|
||||
(root / 'sleep').write_text('#!/bin/sh\nexec /bin/sleep 0.1\n')
|
||||
(root / 'sleep').chmod(0o755)
|
||||
source = SCRIPT.read_text().split('release_stage() {')[0]
|
||||
if accelerated:
|
||||
# Keep GNU timeout real; shorten only test-local deadlines.
|
||||
source = source.replace('--kill-after=2s', '--kill-after=0.1s')
|
||||
source = source.replace('--foreground --kill-after=0.1s 10 ', '--foreground --kill-after=0.1s 2 ')
|
||||
source = source.replace('timeout 15 ', 'timeout 2 ')
|
||||
source = source.replace('--kill-after=0.1s 15 ', '--kill-after=0.1s 2 ')
|
||||
# Only the isolated lock prefix runs; no operational paths are touched.
|
||||
result = subprocess.run(['bash', '-c', source + '\nrelease_stage() {\n' + body + '\n}\nsupervise_release release_stage'],
|
||||
env={**os.environ, 'PATH': str(root) + ':' + os.environ['PATH'],
|
||||
'GITHUB_ACTIONS': 'true', 'DEPLOY_SHA': 'a' * 40,
|
||||
'FAKE_ROOT': str(root), 'MODE': mode},
|
||||
capture_output=True, text=True, timeout=20)
|
||||
calls = [json.loads(line) for line in (root / 'calls').read_text().splitlines()] if (root / 'calls').exists() else []
|
||||
if accelerated:
|
||||
self.assertFalse((root / 'stubborn-completed').exists(),
|
||||
'TERM-ignoring transport outlived its kill deadline')
|
||||
return result, calls, (root / 'held').exists()
|
||||
|
||||
def test_success_holds_host_lock_through_build_and_deploy(self):
|
||||
result, calls, held = self.exercise()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
create = next(c for c in calls if c[0] == 'create')
|
||||
self.assertIn('--pull=never', create)
|
||||
self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina,readonly', create)
|
||||
self.assertIn('runner:local', create)
|
||||
self.assertIn('flock', ' '.join(create))
|
||||
self.assertIn('/host-canina/stage-backend-deploy.lock', ' '.join(create))
|
||||
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
|
||||
self.assertEqual(len([c for c in calls if c[0] == 'compose']), 2)
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_remapped_uid_can_lock_existing_root_owned_readable_host_file(self):
|
||||
result, calls, held = self.exercise('remapped-host')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
create = next(c for c in calls if c[0] == 'create')
|
||||
self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina,readonly', create)
|
||||
self.assertIn('--cap-drop=ALL', create)
|
||||
self.assertIn('--security-opt=no-new-privileges', create)
|
||||
self.assertFalse(any(c.startswith('--userns') or c == '--privileged' for c in create))
|
||||
self.assertNotIn('9>/host-canina/', create[-1])
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_existing_named_helper_fails_closed_without_removal(self):
|
||||
result, calls, _ = self.exercise('contended')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(any(c[0] in ('rm', 'compose', 'start') for c in calls))
|
||||
|
||||
def test_unconfirmed_host_lock_is_bounded_and_removes_only_owned_id(self):
|
||||
result, calls, held = self.exercise('host-contended')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(any(c[0] == 'compose' for c in calls))
|
||||
self.assertEqual(len([c for c in calls if c[:3] == ['logs', '--tail', '1']]), 10)
|
||||
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_failure_diagnostics_are_sanitized_and_captured_before_owned_cleanup(self):
|
||||
result, calls, held = self.exercise('diagnostic-failure')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn('Stage lock helper diagnostic: permission-denied', result.stderr)
|
||||
self.assertNotIn('SECRET_MUST_NOT_LEAK', result.stderr)
|
||||
diagnostic = next(i for i, c in enumerate(calls) if c[:3] == ['logs', '--tail', '10'])
|
||||
self.assertLess(diagnostic, len(calls) - 1)
|
||||
self.assertFalse(any(c[0] == 'compose' for c in calls))
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_start_failure_cleans_owned_helper(self):
|
||||
result, calls, held = self.exercise('start-failure')
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
|
||||
self.assertFalse(held)
|
||||
self.assertFalse(any(c[0] == 'compose' for c in calls))
|
||||
|
||||
def test_deploy_failure_preserves_status_and_releases_owned_helper(self):
|
||||
result, calls, held = self.exercise(body='docker compose build; exit 42')
|
||||
self.assertEqual(result.returncode, 42)
|
||||
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_termination_releases_owned_helper(self):
|
||||
result, calls, held = self.exercise(body='kill -TERM $$')
|
||||
self.assertEqual(result.returncode, 143)
|
||||
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_loss_after_acquisition_before_rollout_prevents_up(self):
|
||||
result, calls, held = self.exercise(body='touch "$FAKE_ROOT/lost"; /bin/sleep 2; docker compose up')
|
||||
self.assertNotEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse(any(c[0] == 'compose' for c in calls))
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_loss_during_build_interrupts_child_and_prevents_up(self):
|
||||
result, calls, held = self.exercise('loss-during-build')
|
||||
self.assertNotEqual(result.returncode, 0, result.stderr)
|
||||
self.assertFalse(any(c[0] == 'compose' and 'up' in c for c in calls))
|
||||
self.assertIn('Stage host lock lost', result.stderr)
|
||||
self.assertIn(['child-stopped'], calls)
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_loss_interrupts_timeout_wrapped_docker_child(self):
|
||||
result, calls, held = self.exercise('loss-during-build', body='timeout --foreground 20 docker compose build; docker compose up')
|
||||
self.assertNotEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn(['child-stopped'], calls)
|
||||
self.assertFalse(any(c[0] == 'compose' and 'up' in c for c in calls))
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_term_ignoring_operational_transport_is_killed(self):
|
||||
result, calls, held = self.exercise('stubborn-operation', accelerated=True)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertFalse(any(c[0] in ('create', 'compose') for c in calls))
|
||||
self.assertFalse(held)
|
||||
|
||||
def test_term_ignoring_cleanup_transport_is_killed_and_fails_closed(self):
|
||||
result, calls, held = self.exercise('stubborn-cleanup', accelerated=True)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn('manual inspection required', result.stderr)
|
||||
self.assertTrue(held, 'failed cleanup must not claim removal')
|
||||
self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned'])
|
||||
|
||||
def test_helper_shell_contends_on_exact_host_inode(self):
|
||||
import fcntl
|
||||
import selectors
|
||||
source = SCRIPT.read_text()
|
||||
helper = source.split('--entrypoint /bin/sh "$runner_image" -c \'')[1].split("\n ')")[0]
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
helper = helper.replace('/host-canina', tmp).replace('sleep 60', 'sleep 0.1')
|
||||
lock_path = Path(tmp) / 'stage-backend-deploy.lock'
|
||||
with lock_path.open('w') as host:
|
||||
fcntl.flock(host, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
blocked = subprocess.run(['sh', '-c', helper], capture_output=True, text=True, timeout=2)
|
||||
self.assertNotEqual(blocked.returncode, 0)
|
||||
self.assertNotIn('ACQUIRED', blocked.stdout)
|
||||
lock_path.write_text('existing-host-inode\n')
|
||||
lock_path.chmod(0o444)
|
||||
with subprocess.Popen(['sh', '-c', helper], stdout=subprocess.PIPE, text=True) as proc:
|
||||
try:
|
||||
assert proc.stdout is not None
|
||||
with selectors.DefaultSelector() as selector:
|
||||
selector.register(proc.stdout, selectors.EVENT_READ)
|
||||
self.assertTrue(selector.select(timeout=2), 'helper did not confirm flock')
|
||||
self.assertEqual(proc.stdout.readline().strip(), 'CANINA_STAGE_DEPLOY_LOCK_ACQUIRED')
|
||||
self.assertEqual(lock_path.read_text(), 'existing-host-inode\n')
|
||||
self.assertEqual(lock_path.stat().st_mode & 0o777, 0o444)
|
||||
with lock_path.open('r') as host:
|
||||
with self.assertRaises(BlockingIOError):
|
||||
fcntl.flock(host, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
finally:
|
||||
proc.terminate()
|
||||
proc.wait(timeout=2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user