From 0fa6abc53d9f67a92116c48ad0ab4856bf46bee4 Mon Sep 17 00:00:00 2001 From: Canina Automation Date: Mon, 26 Oct 2026 18:50:29 +0000 Subject: [PATCH] [verified] fix Stage helper lock under daemon userns remapping --- scripts/release-cache-stage.sh | 25 +++++++++++--- scripts/tests/test_stage_deploy_lock.py | 45 ++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/scripts/release-cache-stage.sh b/scripts/release-cache-stage.sh index 2e1cb465..fcedc1bb 100644 --- a/scripts/release-cache-stage.sh +++ b/scripts/release-cache-stage.sh @@ -46,6 +46,21 @@ cleanup_stage_lock() { wait "$release_pid" 2>/dev/null || : fi if [[ -n $lock_container_id ]]; then + if [[ $status != 0 ]]; then + # Never emit raw helper/daemon output (image entrypoints may contain + # secrets). Classify only a bounded tail before deleting the owned ID. + local helper_log diagnostic=unavailable + helper_log=$(timeout --foreground --kill-after=2s 10 docker logs --tail 10 "$lock_container_id" 2>&1 | head -c 4096) || : + case "$helper_log" in + *'Permission denied'*) diagnostic=permission-denied ;; + *'No such file or directory'*) diagnostic=missing-host-lock ;; + *'Read-only file system'*) diagnostic=read-only-filesystem ;; + *'flock: not found'*) diagnostic=flock-unavailable ;; + *CANINA_STAGE_DEPLOY_LOCK_ACQUIRED*) diagnostic=previously-acquired ;; + ?*) diagnostic=unclassified-output ;; + esac + printf 'Stage lock helper diagnostic: %s\n' "$diagnostic" >&2 + fi timeout --kill-after=2s 15 docker rm -f "$lock_container_id" >/dev/null || { printf 'Stage lock helper cleanup failed; manual inspection required\n' >&2 [[ $status != 0 ]] || status=1 @@ -59,10 +74,12 @@ trap 'exit 143' TERM runner_image=$(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.Config.Image}}' gitea_runner) [[ -n $runner_image ]] || exit 1 lock_container_id=$(timeout --kill-after=2s 15 docker create --name canina-stage-deploy-lock \ - --pull=never --network none \ - --mount type=bind,src=/opt/services/canina,dst=/host-canina \ + --pull=never --network none --cap-drop=ALL --security-opt=no-new-privileges \ + --mount type=bind,src=/opt/services/canina,dst=/host-canina,readonly \ --entrypoint /bin/sh "$runner_image" -c ' - exec 9>/host-canina/stage-backend-deploy.lock + # Linux flock permits an exclusive lock on a read-only descriptor. Preserve + # daemon userns remapping; require the existing host inode, never create it. + exec 9/dev/ lock_acquired=false for lock_attempt in {1..10}; do [[ $(timeout --foreground --kill-after=2s 10 docker inspect -f '{{.State.Running}}' "$lock_container_id") == true ]] || break - if [[ $(timeout --foreground --kill-after=2s 10 docker logs "$lock_container_id") == CANINA_STAGE_DEPLOY_LOCK_ACQUIRED ]]; then + if [[ $(timeout --foreground --kill-after=2s 10 docker logs --tail 1 "$lock_container_id" 2>/dev/null) == CANINA_STAGE_DEPLOY_LOCK_ACQUIRED ]]; then lock_acquired=true break fi diff --git a/scripts/tests/test_stage_deploy_lock.py b/scripts/tests/test_stage_deploy_lock.py index 282584a9..97f694cb 100644 --- a/scripts/tests/test_stage_deploy_lock.py +++ b/scripts/tests/test_stage_deploy_lock.py @@ -27,15 +27,25 @@ if (mode == 'stubborn-operation' and args[-1] == 'gitea_runner' time.sleep(6) # Self-bounded even before the production fix. (root / 'stubborn-completed').touch() if args[0] == 'inspect': - print('runner:local' if args[-1] == 'gitea_runner' else ('false' if (root / 'lost').exists() else 'true')) + denied = False + if mode == 'remapped-host' and (root / 'create.json').exists(): + create = json.loads((root / 'create.json').read_text()) + # uid0 maps to host100000: directory1001:755/file0:644. + # DAC allows read/traversal, not truncation/creation. Model transport + # access only; the separate real-shell test proves actual host flock. + denied = '9>/host-canina/' in create[-1] + print('runner:local' if args[-1] == 'gitea_runner' else ('false' if denied or (root / 'lost').exists() else 'true')) elif args[0] == 'create': if mode == 'contended': sys.exit(1) (root / 'held').touch() + (root / 'create.json').write_text(json.dumps(args)) print('abc123owned') elif args[0] == 'start': if mode == 'start-failure': sys.exit(1) elif args[0] == 'logs': - if mode != 'host-contended': print('CANINA_STAGE_DEPLOY_LOCK_ACQUIRED') + if mode == 'diagnostic-failure': + print('sh: cannot open lock: Permission denied; SECRET_MUST_NOT_LEAK', file=sys.stderr) + elif mode != 'host-contended': print('CANINA_STAGE_DEPLOY_LOCK_ACQUIRED') elif args[0] == 'rm': assert args[-1] == 'abc123owned', 'must only remove owned ID' (root / 'held').unlink() @@ -78,7 +88,7 @@ else: sys.exit(99) self.assertEqual(result.returncode, 0, result.stderr) create = next(c for c in calls if c[0] == 'create') self.assertIn('--pull=never', create) - self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina', create) + self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina,readonly', create) self.assertIn('runner:local', create) self.assertIn('flock', ' '.join(create)) self.assertIn('/host-canina/stage-backend-deploy.lock', ' '.join(create)) @@ -86,6 +96,17 @@ else: sys.exit(99) self.assertEqual(len([c for c in calls if c[0] == 'compose']), 2) self.assertFalse(held) + def test_remapped_uid_can_lock_existing_root_owned_readable_host_file(self): + result, calls, held = self.exercise('remapped-host') + self.assertEqual(result.returncode, 0, result.stderr) + create = next(c for c in calls if c[0] == 'create') + self.assertIn('type=bind,src=/opt/services/canina,dst=/host-canina,readonly', create) + self.assertIn('--cap-drop=ALL', create) + self.assertIn('--security-opt=no-new-privileges', create) + self.assertFalse(any(c.startswith('--userns') or c == '--privileged' for c in create)) + self.assertNotIn('9>/host-canina/', create[-1]) + self.assertFalse(held) + def test_existing_named_helper_fails_closed_without_removal(self): result, calls, _ = self.exercise('contended') self.assertNotEqual(result.returncode, 0) @@ -95,10 +116,20 @@ else: sys.exit(99) result, calls, held = self.exercise('host-contended') self.assertNotEqual(result.returncode, 0) self.assertFalse(any(c[0] == 'compose' for c in calls)) - self.assertEqual(len([c for c in calls if c[0] == 'logs']), 10) + self.assertEqual(len([c for c in calls if c[:3] == ['logs', '--tail', '1']]), 10) self.assertEqual(calls[-1], ['rm', '-f', 'abc123owned']) self.assertFalse(held) + def test_failure_diagnostics_are_sanitized_and_captured_before_owned_cleanup(self): + result, calls, held = self.exercise('diagnostic-failure') + self.assertNotEqual(result.returncode, 0) + self.assertIn('Stage lock helper diagnostic: permission-denied', result.stderr) + self.assertNotIn('SECRET_MUST_NOT_LEAK', result.stderr) + diagnostic = next(i for i, c in enumerate(calls) if c[:3] == ['logs', '--tail', '10']) + self.assertLess(diagnostic, len(calls) - 1) + self.assertFalse(any(c[0] == 'compose' for c in calls)) + self.assertFalse(held) + def test_start_failure_cleans_owned_helper(self): result, calls, held = self.exercise('start-failure') self.assertNotEqual(result.returncode, 0) @@ -165,6 +196,8 @@ else: sys.exit(99) blocked = subprocess.run(['sh', '-c', helper], capture_output=True, text=True, timeout=2) self.assertNotEqual(blocked.returncode, 0) self.assertNotIn('ACQUIRED', blocked.stdout) + lock_path.write_text('existing-host-inode\n') + lock_path.chmod(0o444) with subprocess.Popen(['sh', '-c', helper], stdout=subprocess.PIPE, text=True) as proc: try: assert proc.stdout is not None @@ -172,7 +205,9 @@ else: sys.exit(99) selector.register(proc.stdout, selectors.EVENT_READ) self.assertTrue(selector.select(timeout=2), 'helper did not confirm flock') self.assertEqual(proc.stdout.readline().strip(), 'CANINA_STAGE_DEPLOY_LOCK_ACQUIRED') - with lock_path.open('w') as host: + self.assertEqual(lock_path.read_text(), 'existing-host-inode\n') + self.assertEqual(lock_path.stat().st_mode & 0o777, 0o444) + with lock_path.open('r') as host: with self.assertRaises(BlockingIOError): fcntl.flock(host, fcntl.LOCK_EX | fcntl.LOCK_NB) finally: